From 98d5c682e3ea156a023548f56008d84bab29eb0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ibrahim=20Numanagic=CC=81?= Date: Thu, 26 Sep 2019 08:51:02 -0700 Subject: [PATCH 01/30] Add Python wrapper for JIT --- jupyter/wrapper.py | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 jupyter/wrapper.py diff --git a/jupyter/wrapper.py b/jupyter/wrapper.py new file mode 100644 index 000000000..40ec65a2b --- /dev/null +++ b/jupyter/wrapper.py @@ -0,0 +1,35 @@ +# Exec + +class SeqWrapper: + def __init__(self): + import ctypes + import ctypes.util + + lib = ctypes.util.find_library("seqjit") + self._lib = ctypes.CDLL(lib) + self._init_fn = self._lib.caml_jit_init + self._init_fn.restype = ctypes.c_void_p + self._exec_fn = self._lib.caml_jit_exec + self._exec_fn.argtypes = [ctypes.c_void_p, ctypes.c_char_p] + self.handle = self._init_fn() + + def exec(self, code): + self._exec_fn(self.handle, code.encode('utf-8')) + +s = SeqWrapper() + +s.exec(""" +print 'hello' +x = 1 +print x +def foo(bar): + print 'foo', bar +y = 2 +print x, y, x + y +""") + +s.exec(""" +x += 1 +print x +foo(x+100) +""") \ No newline at end of file From 5bb02d7c5d957ee737afcf2ba656c94756c45a6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ibrahim=20Numanagic=CC=81?= Date: Thu, 26 Sep 2019 08:51:02 -0700 Subject: [PATCH 02/30] Add Python wrapper for JIT --- jupyter/wrapper.py | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 jupyter/wrapper.py diff --git a/jupyter/wrapper.py b/jupyter/wrapper.py new file mode 100644 index 000000000..40ec65a2b --- /dev/null +++ b/jupyter/wrapper.py @@ -0,0 +1,35 @@ +# Exec + +class SeqWrapper: + def __init__(self): + import ctypes + import ctypes.util + + lib = ctypes.util.find_library("seqjit") + self._lib = ctypes.CDLL(lib) + self._init_fn = self._lib.caml_jit_init + self._init_fn.restype = ctypes.c_void_p + self._exec_fn = self._lib.caml_jit_exec + self._exec_fn.argtypes = [ctypes.c_void_p, ctypes.c_char_p] + self.handle = self._init_fn() + + def exec(self, code): + self._exec_fn(self.handle, code.encode('utf-8')) + +s = SeqWrapper() + +s.exec(""" +print 'hello' +x = 1 +print x +def foo(bar): + print 'foo', bar +y = 2 +print x, y, x + y +""") + +s.exec(""" +x += 1 +print x +foo(x+100) +""") \ No newline at end of file From 1fc405be8a143eb9f7f7875deb957fbbbf4b1daa Mon Sep 17 00:00:00 2001 From: jodiew Date: Mon, 18 Nov 2019 15:10:28 -0800 Subject: [PATCH 03/30] Add minimal jupyter kernel --- jupyter/.gitignore | 5 +++ jupyter/README.md | 35 +++++++++++++++ jupyter/pyproject.toml | 15 +++++++ jupyter/seq_kernel/__init__.py | 3 ++ jupyter/seq_kernel/__main__.py | 3 ++ jupyter/seq_kernel/install.py | 67 +++++++++++++++++++++++++++++ jupyter/seq_kernel/kernel.json | 9 ++++ jupyter/seq_kernel/kernel.py | 65 ++++++++++++++++++++++++++++ jupyter/seq_kernel/redirector.py | 64 +++++++++++++++++++++++++++ jupyter/{ => seq_kernel}/wrapper.py | 22 +--------- 10 files changed, 267 insertions(+), 21 deletions(-) create mode 100644 jupyter/.gitignore create mode 100644 jupyter/README.md create mode 100644 jupyter/pyproject.toml create mode 100644 jupyter/seq_kernel/__init__.py create mode 100644 jupyter/seq_kernel/__main__.py create mode 100644 jupyter/seq_kernel/install.py create mode 100644 jupyter/seq_kernel/kernel.json create mode 100644 jupyter/seq_kernel/kernel.py create mode 100644 jupyter/seq_kernel/redirector.py rename jupyter/{ => seq_kernel}/wrapper.py (68%) diff --git a/jupyter/.gitignore b/jupyter/.gitignore new file mode 100644 index 000000000..bd93d1d08 --- /dev/null +++ b/jupyter/.gitignore @@ -0,0 +1,5 @@ +__pycache__ +*.pyc +build/ +dist/ +MANIFEST \ No newline at end of file diff --git a/jupyter/README.md b/jupyter/README.md new file mode 100644 index 000000000..e3911bcc2 --- /dev/null +++ b/jupyter/README.md @@ -0,0 +1,35 @@ +# Seq Kernel + +A seq kernel for Jupyter. + +## To Install + +```bash +python3 -m pip install seq_kernel +python3 -m seq_kernel.install +``` + +## To Use + +```bash +jupyter notebook +# In the notebook interface, select Seq from the 'New' menu +jupyter qtconsole --kernel seq +jupyter console --kernel seq +``` + +## To Deploy + +1. Update the version number in `seq_kernel/__init__.py`. +1. Install flit if you don't already have it. + ```bash + python3 -m pip install flit + ``` +1. Test the package locally. + ```bash + flit install [--symlink] [--python path/to/python] + ``` +1. Run this command to upload to PyPi. + ```bash + flit publish + ``` diff --git a/jupyter/pyproject.toml b/jupyter/pyproject.toml new file mode 100644 index 000000000..327a16892 --- /dev/null +++ b/jupyter/pyproject.toml @@ -0,0 +1,15 @@ +[build-system] +requires = ["flit"] +build-backend = "flit.buildapi" + +[tool.flit.metadata] +module = "seq_kernel" +author = "" +author-email = "" +home-page = "https://github.com/seq-lang/seq" +description-file = "README.md" +classifiers = [ + "Framework :: IPython", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python :: 3", +] diff --git a/jupyter/seq_kernel/__init__.py b/jupyter/seq_kernel/__init__.py new file mode 100644 index 000000000..aa78d7ad5 --- /dev/null +++ b/jupyter/seq_kernel/__init__.py @@ -0,0 +1,3 @@ +"""A seq kernel for Jupyter""" + +from .kernel import __version__ diff --git a/jupyter/seq_kernel/__main__.py b/jupyter/seq_kernel/__main__.py new file mode 100644 index 000000000..1878b7d8e --- /dev/null +++ b/jupyter/seq_kernel/__main__.py @@ -0,0 +1,3 @@ +from ipykernel.kernelapp import IPKernelApp +from .kernel import SeqKernel +IPKernelApp.launch_instance(kernel_class=SeqKernel) \ No newline at end of file diff --git a/jupyter/seq_kernel/install.py b/jupyter/seq_kernel/install.py new file mode 100644 index 000000000..814339101 --- /dev/null +++ b/jupyter/seq_kernel/install.py @@ -0,0 +1,67 @@ +import json +import os +import sys +import argparse + +from jupyter_client.kernelspec import KernelSpecManager +from IPython.utils.tempdir import TemporaryDirectory + +kernel_json = { + "argv":[sys.executable,"-m","seq_kernel", "-f", "{connection_file}"], + "display_name":"Seq", + "language":"seq" +} + +def install_my_kernel_spec(user=True, prefix=None): + with TemporaryDirectory() as td: + os.chmod(td, 0o755) # Starts off as 700, not user readable + with open(os.path.join(td, 'kernel.json'), 'w') as f: + json.dump(kernel_json, f, sort_keys=True) + + print('Installing IPython kernel spec') + KernelSpecManager().install_kernel_spec(td, 'seq', user=user, prefix=prefix) + +def _is_root(): + try: + return os.geteuid() == 0 + except AttributeError: + return False # assume not an admin on non-Unix platforms + +def main(argv=None): + parser = argparse.ArgumentParser( + description='Install KernelSpec for Seq Kernel' + ) + prefix_locations = parser.add_mutually_exclusive_group() + + prefix_locations.add_argument( + '--user', + help='Install KernelSpec in user homedirectory', + action='store_true' + ) + prefix_locations.add_argument( + '--sys-prefix', + help='Install KernelSpec in sys.prefix. Useful in conda / virtualenv', + action='store_true', + dest='sys_prefix' + ) + prefix_locations.add_argument( + '--prefix', + help='Install KernelSpec in this prefix', + default=None + ) + + args = parser.parse_args(argv) + + user = False + prefix = None + if args.sys_prefix: + prefix = sys.prefix + elif args.prefix: + prefix = args.prefix + elif args.user or not _is_root(): + user = True + + install_my_kernel_spec(user=user, prefix=prefix) + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/jupyter/seq_kernel/kernel.json b/jupyter/seq_kernel/kernel.json new file mode 100644 index 000000000..76914b10e --- /dev/null +++ b/jupyter/seq_kernel/kernel.json @@ -0,0 +1,9 @@ +{ + "argv": [ + "python3", + "/home/jodiew/seq/jupyter/seqkernel/kernel.py", + "-f", + "{connection_file}" + ], + "display_name": "Seq" +} \ No newline at end of file diff --git a/jupyter/seq_kernel/kernel.py b/jupyter/seq_kernel/kernel.py new file mode 100644 index 000000000..ff060b0f8 --- /dev/null +++ b/jupyter/seq_kernel/kernel.py @@ -0,0 +1,65 @@ +from ipykernel.kernelbase import Kernel +from subprocess import check_output +from io import BytesIO +import re + +from .redirector import stdout_stderr_redirector +from .wrapper import SeqWrapper + +__version__ = '0.0.0' + +version_pat = re.compile(r'version (\d+(\.\d+)+)') + +class SeqKernel(Kernel): + implementation = 'seqkernel' + implementation_version = __version__ + + @property + def language_version(self): + m = version_pat.search(self.banner) + return m.group(1) + + _banner = None + + @property + def banner(self): + if self._banner is None: + self._banner = check_output(['seqc', '--version']).decode('utf-8') + return self._banner + + language_info = { + 'name': 'Seq', + 'mimetype': 'application/seq', + 'file_extension': '.seq', + } + + def __init__(self, **kwargs): + Kernel.__init__(self, **kwargs) + self.seqwrapper = SeqWrapper() + + def do_execute(self, code, silent, store_history=True, user_expressions=None, allow_stdin=False): + if not code.strip(): + return {'status': 'ok', 'execution_count': self.execution_count, + 'payload': [], 'user_expressions': {}} + + fout = BytesIO() + ferr = BytesIO() + + with stdout_stderr_redirector(fout, ferr): + self.seqwrapper.exec(code.rstrip()) + + fout_string = fout.getvalue().decode('utf-8').strip() + ferr_string = ferr.getvalue().decode('utf-8').strip() + + if ferr_string: + if not silent: + self.send_response(self.iopub_socket, 'stream', {'name': 'stderr', 'text': ferr_string}) + + return {'status': 'error', 'execution_count': self.execution_count} + + else: + if not silent: + self.send_response(self.iopub_socket, 'stream', {'name': 'stdout', 'text': fout_string}) + + return {'status': 'ok', 'execution_count': self.execution_count, + 'payload': [], 'user_expressions': {}} diff --git a/jupyter/seq_kernel/redirector.py b/jupyter/seq_kernel/redirector.py new file mode 100644 index 000000000..0b08f0e3c --- /dev/null +++ b/jupyter/seq_kernel/redirector.py @@ -0,0 +1,64 @@ +from contextlib import contextmanager +import ctypes +import io +import os +import sys +import tempfile + +libc = ctypes.CDLL(None) +c_stdout = ctypes.c_void_p.in_dll(libc, 'stdout') +c_stderr = ctypes.c_void_p.in_dll(libc, 'stderr') + +@contextmanager +def stdout_stderr_redirector(out_stream, err_stream): + # The original fd stdout points to. Usually 1 on POSIX systems. + original_stdout_fd = sys.__stdout__.fileno() + original_stderr_fd = sys.__stderr__.fileno() + + def _redirect_stdout(to_fd): + """Redirect stdout to the given file descriptor.""" + # Flush the C-level buffer stdout + libc.fflush(c_stdout) + # Flush and close sys.stdout - also closes the file descriptor (fd) + sys.__stdout__.close() + # Make original_stdout_fd point to the same file as to_fd + os.dup2(to_fd, original_stdout_fd) + # Create a new sys.stdout that points to the redirected fd + sys.__stdout__ = io.TextIOWrapper(os.fdopen(original_stdout_fd, 'wb')) + + def _redirect_stderr(to_fd): + """Redirect stderr to the given file descriptor.""" + # Flush the C-level buffer stderr + libc.fflush(c_stderr) + # Flush and close sys.stderr - also closes the file descriptor (fd) + sys.__stderr__.close() + # Make original_stderr_fd point to the same file as to_fd + os.dup2(to_fd, original_stderr_fd) + # Create a new sys.stderr that points to the redirected fd + sys.__stderr__ = io.TextIOWrapper(os.fdopen(original_stderr_fd, 'wb')) + + # Save a copy of the original stdout fd in saved_stdout_fd + saved_stdout_fd = os.dup(original_stdout_fd) + saved_stderr_fd = os.dup(original_stderr_fd) + try: + # Create a temporary file and redirect stdout to it + tfile_out = tempfile.TemporaryFile(mode='w+b') + tfile_err = tempfile.TemporaryFile(mode='w+b') + _redirect_stdout(tfile_out.fileno()) + _redirect_stderr(tfile_err.fileno()) + # Yield to caller, then redirect stdout back to the saved fd + yield + _redirect_stdout(saved_stdout_fd) + _redirect_stderr(saved_stderr_fd) + # Copy contents of temporary file to the given stream + tfile_out.flush() + tfile_err.flush() + tfile_out.seek(0, io.SEEK_SET) + tfile_err.seek(0, io.SEEK_SET) + out_stream.write(tfile_out.read()) + err_stream.write(tfile_err.read()) + finally: + tfile_out.close() + tfile_err.close() + os.close(saved_stdout_fd) + os.close(saved_stderr_fd) diff --git a/jupyter/wrapper.py b/jupyter/seq_kernel/wrapper.py similarity index 68% rename from jupyter/wrapper.py rename to jupyter/seq_kernel/wrapper.py index 40ec65a2b..6b48efd2c 100644 --- a/jupyter/wrapper.py +++ b/jupyter/seq_kernel/wrapper.py @@ -1,12 +1,10 @@ -# Exec - class SeqWrapper: def __init__(self): import ctypes import ctypes.util lib = ctypes.util.find_library("seqjit") - self._lib = ctypes.CDLL(lib) + self._lib = ctypes.CDLL(lib, ctypes.RTLD_GLOBAL) self._init_fn = self._lib.caml_jit_init self._init_fn.restype = ctypes.c_void_p self._exec_fn = self._lib.caml_jit_exec @@ -15,21 +13,3 @@ def __init__(self): def exec(self, code): self._exec_fn(self.handle, code.encode('utf-8')) - -s = SeqWrapper() - -s.exec(""" -print 'hello' -x = 1 -print x -def foo(bar): - print 'foo', bar -y = 2 -print x, y, x + y -""") - -s.exec(""" -x += 1 -print x -foo(x+100) -""") \ No newline at end of file From f8e9f2a6780d3a7d40e528ef4f7e1af46aadc7ea Mon Sep 17 00:00:00 2001 From: jodiew Date: Mon, 18 Nov 2019 15:16:27 -0800 Subject: [PATCH 04/30] Add url for redirect source --- jupyter/seq_kernel/redirector.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/jupyter/seq_kernel/redirector.py b/jupyter/seq_kernel/redirector.py index 0b08f0e3c..cf5aec248 100644 --- a/jupyter/seq_kernel/redirector.py +++ b/jupyter/seq_kernel/redirector.py @@ -1,3 +1,8 @@ +""" + Stdout and stderr redirector. + Based on https://eli.thegreenplace.net/2015/redirecting-all-kinds-of-stdout-in-python/ +""" + from contextlib import contextmanager import ctypes import io From 9b570a8189414fdcde1e3d69c6c9902ad67e9233 Mon Sep 17 00:00:00 2001 From: jodiew Date: Fri, 22 Nov 2019 11:47:06 -0800 Subject: [PATCH 05/30] Add syntax highlighting --- jupyter/.gitignore | 5 - jupyter/README.md | 18 +- jupyter/pyproject.toml | 8 +- jupyter/seq_kernel/install.py | 9 +- jupyter/seq_kernel/kernel.js | 384 +++++++++++++++++++++++++++++++++ jupyter/seq_kernel/kernel.json | 9 - jupyter/seq_kernel/kernel.py | 4 +- jupyter/seq_kernel/seqlex.py | 245 +++++++++++++++++++++ 8 files changed, 643 insertions(+), 39 deletions(-) delete mode 100644 jupyter/.gitignore create mode 100644 jupyter/seq_kernel/kernel.js delete mode 100644 jupyter/seq_kernel/kernel.json create mode 100644 jupyter/seq_kernel/seqlex.py diff --git a/jupyter/.gitignore b/jupyter/.gitignore deleted file mode 100644 index bd93d1d08..000000000 --- a/jupyter/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -__pycache__ -*.pyc -build/ -dist/ -MANIFEST \ No newline at end of file diff --git a/jupyter/README.md b/jupyter/README.md index e3911bcc2..7598ac4d7 100644 --- a/jupyter/README.md +++ b/jupyter/README.md @@ -5,7 +5,7 @@ A seq kernel for Jupyter. ## To Install ```bash -python3 -m pip install seq_kernel +python3 -m pip install /seq/jupyter python3 -m seq_kernel.install ``` @@ -17,19 +17,3 @@ jupyter notebook jupyter qtconsole --kernel seq jupyter console --kernel seq ``` - -## To Deploy - -1. Update the version number in `seq_kernel/__init__.py`. -1. Install flit if you don't already have it. - ```bash - python3 -m pip install flit - ``` -1. Test the package locally. - ```bash - flit install [--symlink] [--python path/to/python] - ``` -1. Run this command to upload to PyPi. - ```bash - flit publish - ``` diff --git a/jupyter/pyproject.toml b/jupyter/pyproject.toml index 327a16892..517621d99 100644 --- a/jupyter/pyproject.toml +++ b/jupyter/pyproject.toml @@ -8,8 +8,6 @@ author = "" author-email = "" home-page = "https://github.com/seq-lang/seq" description-file = "README.md" -classifiers = [ - "Framework :: IPython", - "License :: OSI Approved :: Apache Software License", - "Programming Language :: Python :: 3", -] + +[tool.flit.entrypoints."pygments.lexers"] +seq = "seq_kernel.seqlex:SeqLexer" \ No newline at end of file diff --git a/jupyter/seq_kernel/install.py b/jupyter/seq_kernel/install.py index 814339101..c6dab48e3 100644 --- a/jupyter/seq_kernel/install.py +++ b/jupyter/seq_kernel/install.py @@ -5,11 +5,12 @@ from jupyter_client.kernelspec import KernelSpecManager from IPython.utils.tempdir import TemporaryDirectory +from shutil import copyfile kernel_json = { "argv":[sys.executable,"-m","seq_kernel", "-f", "{connection_file}"], "display_name":"Seq", - "language":"seq" + "language":"seq", } def install_my_kernel_spec(user=True, prefix=None): @@ -21,6 +22,11 @@ def install_my_kernel_spec(user=True, prefix=None): print('Installing IPython kernel spec') KernelSpecManager().install_kernel_spec(td, 'seq', user=user, prefix=prefix) +def install_my_kernel_javascript(): + seq_js_file = os.path.join(os.environ['SEQ_PATH'][:-7], 'jupyter', 'seq_kernel', 'kernel.js') + kernel_js_file = os.path.join(KernelSpecManager().get_kernel_spec('seq').resource_dir, 'kernel.js') + os.system(f'cp {seq_js_file} {kernel_js_file}') + def _is_root(): try: return os.geteuid() == 0 @@ -62,6 +68,7 @@ def main(argv=None): user = True install_my_kernel_spec(user=user, prefix=prefix) + install_my_kernel_javascript() if __name__ == '__main__': main() \ No newline at end of file diff --git a/jupyter/seq_kernel/kernel.js b/jupyter/seq_kernel/kernel.js new file mode 100644 index 000000000..166183f1d --- /dev/null +++ b/jupyter/seq_kernel/kernel.js @@ -0,0 +1,384 @@ +define( + ["codemirror/lib/codemirror", "base/js/namespace"], + function(CodeMirror){ + "use strict"; + var onload = function() { + console.log("Loading kernel.js from Seq"); + enableSeqMode(CodeMirror); + }; + return{onload: onload}; +}); + +var enableSeqMode = function (CodeMirror) { + function wordRegexp(words) { + return new RegExp("^((" + words.join(")|(") + "))\\b"); + } + var wordOperators = wordRegexp(["and", "or", "not", "is"]); + var commonKeywords = ["as", "assert", "break", "class", "continue", + "def", "del", "elif", "else", "except", "finally", + "for", "from", "global", "if", "import", + "lambda", "pass", "raise", "return", + "try", "while", "with", "yield", "in", + // seq keywords + "match", "case", "pydef", "extern", "prefetch", "type", + "extend", "cimport", "pyimport" + ]; + var commonBuiltins = ["abs", "all", "any", "bin", "bool", "bytearray", "callable", "chr", + "classmethod", "compile", "complex", "delattr", "dict", "dir", "divmod", + "enumerate", "eval", "filter", "float", "format", "frozenset", + "getattr", "globals", "hasattr", "hash", "help", "hex", "id", + "input", "int", "isinstance", "issubclass", "iter", "len", + "list", "locals", "map", "max", "memoryview", "min", "next", + "object", "oct", "open", "ord", "pow", "property", "range", + "repr", "reversed", "round", "set", "setattr", "slice", + "sorted", "staticmethod", "str", "sum", "super", "tuple", + "type", "vars", "zip", "__import__", "NotImplemented", + "Ellipsis", "__debug__", + // seq buildins + "seq", "byte", "ptr", "array", "Kmer", "Int", "UInt", "optional" + ]; + CodeMirror.registerHelper("hintWords", "seq", commonKeywords.concat(commonBuiltins)); + + function top(state) { + return state.scopes[state.scopes.length - 1]; + } + CodeMirror.defineMode("seq", function(conf, parserConf) { + var ERRORCLASS = "error"; + + var delimiters = parserConf.delimiters || parserConf.singleDelimiters || /^[\(\)\[\]\{\}@,:`=;\.\\]/; + // (Backwards-compatiblity with old, cumbersome config system) + var operators = [parserConf.singleOperators, parserConf.doubleOperators, parserConf.doubleDelimiters, parserConf.tripleDelimiters, + parserConf.operators || /^([-+*/%\/&|^]=?|[<>=]+|\/\/=?|\*\*=?|!=|[~!@]|\.\.\.|\|\|>|\|>)/] + for (var i = 0; i < operators.length; i++) if (!operators[i]) operators.splice(i--, 1) + + var hangingIndent = parserConf.hangingIndent || conf.indentUnit; + + var myKeywords = commonKeywords, myBuiltins = commonBuiltins; + if (parserConf.extra_keywords != undefined) + myKeywords = myKeywords.concat(parserConf.extra_keywords); + + if (parserConf.extra_builtins != undefined) + myBuiltins = myBuiltins.concat(parserConf.extra_builtins); + + + // since http://legacy.python.org/dev/peps/pep-0465/ @ is also an operator + var identifiers = parserConf.identifiers|| /^[_A-Za-z\u00A1-\uFFFF][_A-Za-z0-9\u00A1-\uFFFF]*/; + myKeywords = myKeywords.concat(["nonlocal", "False", "True", "None", "async", "await"]); + myBuiltins = myBuiltins.concat(["ascii", "bytes", "exec", "print"]); + var stringPrefixes = new RegExp("^(([rbufs]|(br)|(fr))?('{3}|\"{3}|['\"]))", "i"); + + var keywords = wordRegexp(myKeywords); + var builtins = wordRegexp(myBuiltins); + + // tokenizers + function tokenBase(stream, state) { + var sol = stream.sol() && state.lastToken != "\\" + if (sol) state.indent = stream.indentation() + // Handle scope changes + if (sol && top(state).type == "py") { + var scopeOffset = top(state).offset; + if (stream.eatSpace()) { + var lineOffset = stream.indentation(); + if (lineOffset > scopeOffset) + pushPyScope(state); + else if (lineOffset < scopeOffset && dedent(stream, state) && stream.peek() != "#") + state.errorToken = true; + return null; + } else { + var style = tokenBaseInner(stream, state); + if (scopeOffset > 0 && dedent(stream, state)) + style += " " + ERRORCLASS; + return style; + } + } + return tokenBaseInner(stream, state); + } + + function tokenBaseInner(stream, state) { + if (stream.eatSpace()) return null; + + // Handle Comments + if (stream.match(/^#.*/)) return "comment"; + + // Handle Number Literals + if (stream.match(/^[0-9\.]/, false)) { + var floatLiteral = false; + // Floats + if (stream.match(/^[\d_]*\.\d+(e[\+\-]?\d+)?/i)) { floatLiteral = true; } + if (stream.match(/^[\d_]+\.\d*/)) { floatLiteral = true; } + if (stream.match(/^\.\d+/)) { floatLiteral = true; } + if (floatLiteral) { + // Float literals may be "imaginary" + stream.eat(/J/i); + return "number"; + } + // Integers + var intLiteral = false; + // Hex + if (stream.match(/^0x[0-9a-f_]+/i)) intLiteral = true; + // Binary + if (stream.match(/^0b[01_]+/i)) intLiteral = true; + // Octal + if (stream.match(/^0o[0-7_]+/i)) intLiteral = true; + // Decimal + if (stream.match(/^[1-9][\d_]*(e[\+\-]?[\d_]+)?/)) { + // Decimal literals may be "imaginary" + stream.eat(/J/i); + // TODO - Can you have imaginary longs? + intLiteral = true; + } + // Zero by itself with no other piece of number. + if (stream.match(/^0(?![\dx])/i)) intLiteral = true; + if (intLiteral) { + // Integer literals may be "long" + stream.eat(/L/i); + return "number"; + } + } + + // Handle Strings + if (stream.match(stringPrefixes)) { + var isFmtString = stream.current().toLowerCase().indexOf('f') !== -1; + if (!isFmtString) { + state.tokenize = tokenStringFactory(stream.current(), state.tokenize); + return state.tokenize(stream, state); + } else { + state.tokenize = formatStringFactory(stream.current(), state.tokenize); + return state.tokenize(stream, state); + } + } + + for (var i = 0; i < operators.length; i++) + if (stream.match(operators[i])) return "operator" + + if (stream.match(delimiters)) return "punctuation"; + + if (state.lastToken == "." && stream.match(identifiers)) + return "property"; + + if (stream.match(keywords) || stream.match(wordOperators)) + return "keyword"; + + if (stream.match(builtins)) + return "builtin"; + + if (stream.match(/^(self|cls)\b/)) + return "variable-2"; + + if (stream.match(identifiers)) { + if (state.lastToken == "def" || state.lastToken == "class") + return "def"; + return "variable"; + } + + // Handle non-detected items + stream.next(); + return ERRORCLASS; + } + + function formatStringFactory(delimiter, tokenOuter) { + while ("rubfs".indexOf(delimiter.charAt(0).toLowerCase()) >= 0) + delimiter = delimiter.substr(1); + + var singleline = delimiter.length == 1; + var OUTCLASS = "string"; + + function tokenNestedExpr(depth) { + return function(stream, state) { + var inner = tokenBaseInner(stream, state) + if (inner == "punctuation") { + if (stream.current() == "{") { + state.tokenize = tokenNestedExpr(depth + 1) + } else if (stream.current() == "}") { + if (depth > 1) state.tokenize = tokenNestedExpr(depth - 1) + else state.tokenize = tokenString + } + } + return inner + } + } + + function tokenString(stream, state) { + while (!stream.eol()) { + stream.eatWhile(/[^'"\{\}\\]/); + if (stream.eat("\\")) { + stream.next(); + if (singleline && stream.eol()) + return OUTCLASS; + } else if (stream.match(delimiter)) { + state.tokenize = tokenOuter; + return OUTCLASS; + } else if (stream.match('{{')) { + // ignore {{ in f-str + return OUTCLASS; + } else if (stream.match('{', false)) { + // switch to nested mode + state.tokenize = tokenNestedExpr(0) + if (stream.current()) return OUTCLASS; + else return state.tokenize(stream, state) + } else if (stream.match('}}')) { + return OUTCLASS; + } else if (stream.match('}')) { + // single } in f-string is an error + return ERRORCLASS; + } else { + stream.eat(/['"]/); + } + } + if (singleline) { + if (parserConf.singleLineStringErrors) + return ERRORCLASS; + else + state.tokenize = tokenOuter; + } + return OUTCLASS; + } + tokenString.isString = true; + return tokenString; + } + + function tokenStringFactory(delimiter, tokenOuter) { + while ("rubfs".indexOf(delimiter.charAt(0).toLowerCase()) >= 0) + delimiter = delimiter.substr(1); + + var singleline = delimiter.length == 1; + var OUTCLASS = "string"; + + function tokenString(stream, state) { + while (!stream.eol()) { + stream.eatWhile(/[^'"\\]/); + if (stream.eat("\\")) { + stream.next(); + if (singleline && stream.eol()) + return OUTCLASS; + } else if (stream.match(delimiter)) { + state.tokenize = tokenOuter; + return OUTCLASS; + } else { + stream.eat(/['"]/); + } + } + if (singleline) { + if (parserConf.singleLineStringErrors) + return ERRORCLASS; + else + state.tokenize = tokenOuter; + } + return OUTCLASS; + } + tokenString.isString = true; + return tokenString; + } + + function pushPyScope(state) { + while (top(state).type != "py") state.scopes.pop() + state.scopes.push({offset: top(state).offset + conf.indentUnit, + type: "py", + align: null}) + } + + function pushBracketScope(stream, state, type) { + var align = stream.match(/^([\s\[\{\(]|#.*)*$/, false) ? null : stream.column() + 1 + state.scopes.push({offset: state.indent + hangingIndent, + type: type, + align: align}) + } + + function dedent(stream, state) { + var indented = stream.indentation(); + while (state.scopes.length > 1 && top(state).offset > indented) { + if (top(state).type != "py") return true; + state.scopes.pop(); + } + return top(state).offset != indented; + } + + function tokenLexer(stream, state) { + if (stream.sol()) state.beginningOfLine = true; + + var style = state.tokenize(stream, state); + var current = stream.current(); + + // Handle decorators + if (state.beginningOfLine && current == "@") + return stream.match(identifiers, false) ? "meta" : "operator"; + + if (/\S/.test(current)) state.beginningOfLine = false; + + if ((style == "variable" || style == "builtin") + && state.lastToken == "meta") + style = "meta"; + + // Handle scope changes. + if (current == "pass" || current == "return") + state.dedent += 1; + + if (current == "lambda") state.lambda = true; + if (current == ":" && !state.lambda && top(state).type == "py") + pushPyScope(state); + + if (current.length == 1 && !/string|comment/.test(style)) { + var delimiter_index = "[({".indexOf(current); + if (delimiter_index != -1) + pushBracketScope(stream, state, "])}".slice(delimiter_index, delimiter_index+1)); + + delimiter_index = "])}".indexOf(current); + if (delimiter_index != -1) { + if (top(state).type == current) state.indent = state.scopes.pop().offset - hangingIndent + else return ERRORCLASS; + } + } + if (state.dedent > 0 && stream.eol() && top(state).type == "py") { + if (state.scopes.length > 1) state.scopes.pop(); + state.dedent -= 1; + } + + return style; + } + + var external = { + startState: function(basecolumn) { + return { + tokenize: tokenBase, + scopes: [{offset: basecolumn || 0, type: "py", align: null}], + indent: basecolumn || 0, + lastToken: null, + lambda: false, + dedent: 0 + }; + }, + + token: function(stream, state) { + var addErr = state.errorToken; + if (addErr) state.errorToken = false; + var style = tokenLexer(stream, state); + + if (style && style != "comment") + state.lastToken = (style == "keyword" || style == "punctuation") ? stream.current() : style; + if (style == "punctuation") style = null; + + if (stream.eol() && state.lambda) + state.lambda = false; + return addErr ? style + " " + ERRORCLASS : style; + }, + + indent: function(state, textAfter) { + if (state.tokenize != tokenBase) + return state.tokenize.isString ? CodeMirror.Pass : 0; + + var scope = top(state), closing = scope.type == textAfter.charAt(0) + if (scope.align != null) + return scope.align - (closing ? 1 : 0) + else + return scope.offset - (closing ? hangingIndent : 0) + }, + + electricInput: /^\s*[\}\]\)]$/, + closeBrackets: {triples: "'\""}, + lineComment: "#", + fold: "indent" + }; + return external; + }); + CodeMirror.defineMIME("text/seq", "seq"); +} \ No newline at end of file diff --git a/jupyter/seq_kernel/kernel.json b/jupyter/seq_kernel/kernel.json deleted file mode 100644 index 76914b10e..000000000 --- a/jupyter/seq_kernel/kernel.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "argv": [ - "python3", - "/home/jodiew/seq/jupyter/seqkernel/kernel.py", - "-f", - "{connection_file}" - ], - "display_name": "Seq" -} \ No newline at end of file diff --git a/jupyter/seq_kernel/kernel.py b/jupyter/seq_kernel/kernel.py index ff060b0f8..d368e8bf0 100644 --- a/jupyter/seq_kernel/kernel.py +++ b/jupyter/seq_kernel/kernel.py @@ -11,7 +11,7 @@ version_pat = re.compile(r'version (\d+(\.\d+)+)') class SeqKernel(Kernel): - implementation = 'seqkernel' + implementation = 'seq_kernel' implementation_version = __version__ @property @@ -28,7 +28,7 @@ def banner(self): return self._banner language_info = { - 'name': 'Seq', + 'name': 'seq', 'mimetype': 'application/seq', 'file_extension': '.seq', } diff --git a/jupyter/seq_kernel/seqlex.py b/jupyter/seq_kernel/seqlex.py new file mode 100644 index 000000000..0d245156f --- /dev/null +++ b/jupyter/seq_kernel/seqlex.py @@ -0,0 +1,245 @@ +# Seq Pygments lexer based on Python's + +import re + +from pygments.lexer import Lexer, RegexLexer, include, bygroups, using, \ + default, words, combined, do_insertions +from pygments.util import get_bool_opt, shebang_matches +from pygments.token import Text, Comment, Operator, Keyword, Name, String, \ + Number, Punctuation, Generic, Other, Error +from pygments import unistring as uni + +__all__ = ['SeqLexer'] + +line_re = re.compile('.*?\n') + +class SeqLexer(RegexLexer): + name = 'Seq' + aliases = ['seq'] + filenames = ['*.seq'] + mimetypes = [] + + flags = re.MULTILINE | re.UNICODE + + uni_name = "[%s][%s]*" % (uni.xid_start, uni.xid_continue) + + def innerstring_rules(ttype): + return [ + # the old style '%s' % (...) string formatting (still valid in Py3) + (r'%(\(\w+\))?[-#0 +]*([0-9]+|[*])?(\.([0-9]+|[*]))?' + '[hlL]?[E-GXc-giorsux%]', String.Interpol), + # the new style '{}'.format(...) string formatting + (r'\{' + '((\w+)((\.\w+)|(\[[^\]]+\]))*)?' # field name + '(\![sra])?' # conversion + '(\:(.?[<>=\^])?[-+ ]?#?0?(\d+)?,?(\.\d+)?[E-GXb-gnosx%]?)?' + '\}', String.Interpol), + + # backslashes, quotes and formatting signs must be parsed one at a time + (r'[^\\\'"%{\n]+', ttype), + (r'[\'"\\]', ttype), + # unhandled string formatting sign + (r'%|(\{{1,2})', ttype) + # newlines are an error (use "nl" state) + ] + + tokens = { + 'root': [ + (r'\n', Text), + (r'^(\s*)([rRuUbB]{,2})("""(?:.|\n)*?""")', + bygroups(Text, String.Affix, String.Doc)), + (r"^(\s*)([rRuUbB]{,2})('''(?:.|\n)*?''')", + bygroups(Text, String.Affix, String.Doc)), + (r'[^\S\n]+', Text), + (r'\A#!.+$', Comment.Hashbang), + (r'#.*$', Comment.Single), + (r'[]{}:(),;[]', Punctuation), + (r'\\\n', Text), + (r'\\', Text), + (r'(in|is|and|or|not)\b', Operator.Word), + (r'!=|==|<<|>>|[-~+/*%=<>&^|.]', Operator), + include('keywords'), + (r'(def)((?:\s|\\\s)+)', bygroups(Keyword, Text), 'funcname'), + (r'(class)((?:\s|\\\s)+)', bygroups(Keyword, Text), 'classname'), + (r'(from)((?:\s|\\\s)+)', bygroups(Keyword.Namespace, Text), + 'fromimport'), + (r'(import)((?:\s|\\\s)+)', bygroups(Keyword.Namespace, Text), + 'import'), + include('builtins'), + include('magicfuncs'), + include('magicvars'), + include('backtick'), + ('([skrR]|[uUbB][rR]|[rR][uUbB])(""")', + bygroups(String.Affix, String.Double), 'tdqs'), + ("([skrR]|[uUbB][rR]|[rR][uUbB])(''')", + bygroups(String.Affix, String.Single), 'tsqs'), + ('([skrR]|[uUbB][rR]|[rR][uUbB])(")', + bygroups(String.Affix, String.Double), 'dqs'), + ("([skrR]|[uUbB][rR]|[rR][uUbB])(')", + bygroups(String.Affix, String.Single), 'sqs'), + ('([uUbB]?)(""")', bygroups(String.Affix, String.Double), + combined('stringescape', 'tdqs')), + ("([uUbB]?)(''')", bygroups(String.Affix, String.Single), + combined('stringescape', 'tsqs')), + ('([uUbB]?)(")', bygroups(String.Affix, String.Double), + combined('stringescape', 'dqs')), + ("([uUbB]?)(')", bygroups(String.Affix, String.Single), + combined('stringescape', 'sqs')), + include('name'), + include('numbers'), + ], + 'funcname': [ + include('magicfuncs'), + ('[a-zA-Z_]\w*', Name.Function, '#pop'), + default('#pop'), + ], + 'stringescape': [ + (r'\\([\\abfnrtv"\']|\n|N\{.*?\}|u[a-fA-F0-9]{4}|' + r'U[a-fA-F0-9]{8}|x[a-fA-F0-9]{2}|[0-7]{1,3})', String.Escape) + ], + 'strings-single': innerstring_rules(String.Single), + 'strings-double': innerstring_rules(String.Double), + 'dqs': [ + (r'"', String.Double, '#pop'), + (r'\\\\|\\"|\\\n', String.Escape), # included here for raw strings + include('strings-double') + ], + 'sqs': [ + (r"'", String.Single, '#pop'), + (r"\\\\|\\'|\\\n", String.Escape), # included here for raw strings + include('strings-single') + ], + 'tdqs': [ + (r'"""', String.Double, '#pop'), + include('strings-double'), + (r'\n', String.Double) + ], + 'tsqs': [ + (r"'''", String.Single, '#pop'), + include('strings-single'), + (r'\n', String.Single) + ], + } + + tokens['keywords'] = [ + (words(( + 'assert', 'async', 'await', 'break', 'continue', 'del', 'elif', + 'else', 'except', 'finally', 'for', 'global', 'if', 'lambda', 'pass', + 'raise', 'nonlocal', 'return', 'try', 'while', 'yield', 'yield from', + 'as', 'with', 'match', 'case', 'default', 'cdef', 'extern', 'prefetch', + 'type', 'extend', 'print'), suffix=r'\b'), + Keyword), + (words(( + 'True', 'False', 'None'), suffix=r'\b'), + Keyword.Constant), + ] + + seqwords = ['__import__', 'abs', 'all', 'any', 'bin', 'bool', 'bytearray', 'bytes', + 'chr', 'classmethod', 'cmp', 'compile', 'complex', 'delattr', 'dict', + 'dir', 'divmod', 'enumerate', 'eval', 'filter', 'float', 'format', + 'frozenset', 'getattr', 'globals', 'hasattr', 'hash', 'hex', 'id', + 'input', 'int', 'isinstance', 'issubclass', 'iter', 'len', 'list', + 'locals', 'map', 'max', 'memoryview', 'min', 'object', 'oct', + 'open', 'ord', 'pow', 'property', 'range', 'repr', 'reversed', + 'round', 'set', 'setattr', 'slice', 'sorted', 'staticmethod', 'str', + 'sum', 'super', 'tuple', 'vars', 'zip', 'seq', 'byte', 'ptr', 'array', + 'Kmer', 'Int', 'UInt'] + + tokens['builtins'] = [ + (words(seqwords, prefix=r'(? Date: Sat, 23 Nov 2019 13:54:07 -0800 Subject: [PATCH 06/30] Add kernel inspect hook --- jupyter/seq_kernel/kernel.py | 15 +++++++++++++++ jupyter/seq_kernel/wrapper.py | 6 ++++++ 2 files changed, 21 insertions(+) diff --git a/jupyter/seq_kernel/kernel.py b/jupyter/seq_kernel/kernel.py index d368e8bf0..47ab9f008 100644 --- a/jupyter/seq_kernel/kernel.py +++ b/jupyter/seq_kernel/kernel.py @@ -63,3 +63,18 @@ def do_execute(self, code, silent, store_history=True, user_expressions=None, al return {'status': 'ok', 'execution_count': self.execution_count, 'payload': [], 'user_expressions': {}} + + def _get_object(self, code, cursor_pos): + # TODO: Get the correct section of code to be inspected + left = code[:cursor_pos].rsplit(' ', 1)[-1] + right = code[cursor_pos:].split(' ', 1)[0] + return left + right + + def do_inspect(self, code, cursor_pos, detail_level=0): + data = self.seqwrapper.inspect(self._get_object(code, cursor_pos), detail_level) + return { + 'status': 'ok', + 'found': True, + 'data': data, + 'metadata': {} + } diff --git a/jupyter/seq_kernel/wrapper.py b/jupyter/seq_kernel/wrapper.py index 6b48efd2c..831308860 100644 --- a/jupyter/seq_kernel/wrapper.py +++ b/jupyter/seq_kernel/wrapper.py @@ -13,3 +13,9 @@ def __init__(self): def exec(self, code): self._exec_fn(self.handle, code.encode('utf-8')) + + def inspect(self, request_object, detail_level): + # TODO: Inspect the requested object and return the doc and/or source + return { + 'text/plain': f'Hello, World! {request_object} {detail_level}' + } From f8590cd6dcf9c0592c24b5d6be81ca2d31b2d8e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ibrahim=20Numanagi=C4=87?= Date: Fri, 29 Nov 2019 09:44:37 -0800 Subject: [PATCH 07/30] Fix pyrpoject.toml dependencies; Fix macOS support --- jupyter/pyproject.toml | 2 +- jupyter/seq_kernel/redirector.py | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/jupyter/pyproject.toml b/jupyter/pyproject.toml index 517621d99..b6a23b242 100644 --- a/jupyter/pyproject.toml +++ b/jupyter/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["flit"] +requires = ["flit", "ipykernel"] build-backend = "flit.buildapi" [tool.flit.metadata] diff --git a/jupyter/seq_kernel/redirector.py b/jupyter/seq_kernel/redirector.py index cf5aec248..1386f8103 100644 --- a/jupyter/seq_kernel/redirector.py +++ b/jupyter/seq_kernel/redirector.py @@ -11,8 +11,12 @@ import tempfile libc = ctypes.CDLL(None) -c_stdout = ctypes.c_void_p.in_dll(libc, 'stdout') -c_stderr = ctypes.c_void_p.in_dll(libc, 'stderr') +if sys.platform == "darwin": + c_stdout = ctypes.c_void_p.in_dll(libc, '__stdoutp') + c_stderr = ctypes.c_void_p.in_dll(libc, '__stderrp') +else: + c_stdout = ctypes.c_void_p.in_dll(libc, 'stdout') + c_stderr = ctypes.c_void_p.in_dll(libc, 'stderr') @contextmanager def stdout_stderr_redirector(out_stream, err_stream): From 23b203b26e7dfc5351160f754effcd47f2c93a43 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ibrahim=20Numanagi=C4=87?= Date: Fri, 29 Nov 2019 14:28:38 -0800 Subject: [PATCH 08/30] Initial support for JIT autocompletion and introspection --- compiler/util/ocaml.cpp | 10 ++++++++ parser/jit.ml | 43 ++++++++++++++++++++++++++++------- parser/seqaml/codegen_expr.ml | 15 +++++++++--- parser/seqaml/codegen_stmt.ml | 4 ++++ parser/seqaml/ctx.ml | 32 ++++++++++++++++++++++++++ parser/seqaml/llvm.ml | 11 +++++++++ runtime/main.cpp | 1 + 7 files changed, 105 insertions(+), 11 deletions(-) diff --git a/compiler/util/ocaml.cpp b/compiler/util/ocaml.cpp index caebbba2d..4213bd7f6 100644 --- a/compiler/util/ocaml.cpp +++ b/compiler/util/ocaml.cpp @@ -664,6 +664,16 @@ FOREIGN types::Type *realize_type(types::RefType *t, types::Type **types, return types::GenericType::get(t, vector(types, types + sz)); } +FOREIGN char* get_pos_str (SrcObject *v) +{ + if (!v) return strdup(""); + auto info = v->getSrcInfo(); + char *er = 0; + asprintf(&er, "%s\b%d\b%d\b%d", info.file.c_str(), info.line, + info.col, info.len); + return er; +} + /// Anything below throws exceptions // Yes, separator character here is \b diff --git a/parser/jit.ml b/parser/jit.ml index 4062a903e..0e6390503 100644 --- a/parser/jit.ml +++ b/parser/jit.ml @@ -62,6 +62,22 @@ let exec (jit : t) code = with | Err.SeqCError (msg, pos) -> raise @@ Err.CompilerError (Compiler msg, [ pos ]) +let locate cmd f l c = + let open Option.Monad_infix in + Hashtbl.find Ctx.inspect_lookup f + >>= (fun t -> Hashtbl.find t l) + >>= (fun s -> Stack.find_map s ~f:(fun Ctx.{ pos; el; name } -> + if pos.col < c || pos.col + pos.len >= c + then None + else match Llvm.JIT.get_pos el with + | None -> None + | Some orig_pos -> + eprintf "%%JIT%%: found %s, orig pos %s" + name (Ast.Ann.to_string orig_pos); + Some 1 + )) + + (** JIT entry point. *) let repl () = let banner = String.make 78 '=' in @@ -83,15 +99,26 @@ let repl () = code := !code ^ s ^ "\n" with | End_of_file -> - eprintf " \r%!"; - (try exec jit !code with - | Err.CompilerError (typ, pos_lst) -> - eprintf "%s\n%!" @@ Err.to_string ~pos_lst ~file:!code typ); - if !code = "" - then raise Exit - else ( + eprintf "------------------\n%!"; + match String.is_prefix ~prefix:"%%" !code with + | true -> + let ll = Array.of_list @@ String.split ~on:' ' !code in + let cmd = ll.(0) in + let file = ll.(1) in + let line = Int.of_string @@ ll.(2) in + let pos = Int.of_string @@ ll.(3) in + ignore @@ locate cmd file line pos; code := ""; - start := true) + start := true + | false -> + (try exec jit !code with + | Err.CompilerError (typ, pos_lst) -> + eprintf "%s\n%!" @@ Err.to_string ~pos_lst ~file:!code typ); + if !code = "" + then raise Exit + else ( + code := ""; + start := true) done with | Exit -> eprintf "\n\027[31mbye (%d) \027[0m\n%!" jit.cnt diff --git a/parser/seqaml/codegen_expr.ml b/parser/seqaml/codegen_expr.ml index 61d2bdea1..ba3abf114 100644 --- a/parser/seqaml/codegen_expr.ml +++ b/parser/seqaml/codegen_expr.ml @@ -135,15 +135,20 @@ module Codegen (S : Codegen_intf.Stmt) : Codegen_intf.Expr = struct match is_type, Hashtbl.find map var with (* Make sure that a variable is either accessible within the same base (function) or that it is global variable *) - | _, Some ((Ctx_namespace.Type t, _) :: _) -> Llvm.Expr.typ t + | _, Some ((Ctx_namespace.Type t, _) :: _) -> + Ctx.add_inspect pos var t; + Llvm.Expr.typ t | true, _ -> serr ~pos "type '%s' not found or realized" var | false, Some ((Ctx_namespace.Var v, { base; global; _ }) :: _) when ctx.base = base || global -> + Ctx.add_inspect pos var v; let e = Llvm.Expr.var v in if global && ctx.base = base && Stack.exists ctx.flags ~f:(( = ) "atomic") then Llvm.Var.set_atomic e; e - | false, Some ((Ctx_namespace.Func (t, _), _) :: _) -> Llvm.Expr.func t + | false, Some ((Ctx_namespace.Func (t, _), _) :: _) -> + Ctx.add_inspect pos var t; + Llvm.Expr.func t | _ -> serr ~pos "identifier '%s' not found" var and parse_tuple ctx _ args = @@ -418,17 +423,21 @@ module Codegen (S : Codegen_intf.Stmt) : Codegen_intf.Expr = struct | _ -> None) | _, _ -> None in + let npos = { pos with col = (fst lh_expr).col + (fst lh_expr).len } in match imports ctx.map lh_expr with | Some ictx -> (* Util.dbg "import helper..."; *) parse_id ctx ~map:ictx pos rhs | None -> let lh_expr = parse ~ctx lh_expr in - if Llvm.Expr.is_type lh_expr + let el = if Llvm.Expr.is_type lh_expr then ( let typ = Llvm.Type.expr_type lh_expr in Llvm.Expr.static typ rhs) else Llvm.Expr.element lh_expr rhs + in + Ctx.add_inspect npos (sprintf ".%s" rhs) el; + el and parse_slice ctx pos (a, b, c) = let prefix, args = match a, b, c with diff --git a/parser/seqaml/codegen_stmt.ml b/parser/seqaml/codegen_stmt.ml index 0e78c0bc9..b89284ffc 100644 --- a/parser/seqaml/codegen_stmt.ml +++ b/parser/seqaml/codegen_stmt.ml @@ -549,6 +549,10 @@ module Codegen (E : Codegen_intf.Expr) : Codegen_intf.Stmt = struct typ ~f:(fun typ -> Llvm.Func.set_type fn (E.parse_type ~ctx:new_ctx typ)) ~default:(); + ( match List.hd fn_stmts with + | Some (_, Expr (_, String doc)) -> + Ctx.add_origin pos doc + | _ -> () ); add_block new_ctx fn_stmts ~preprocess:(fun ctx -> List.iter names ~f:(fun name -> let var = Ctx_namespace.Var (Llvm.Func.get_arg fn name) in diff --git a/parser/seqaml/ctx.ml b/parser/seqaml/ctx.ml index 2b9d847fe..bd21396a0 100644 --- a/parser/seqaml/ctx.ml +++ b/parser/seqaml/ctx.ml @@ -32,6 +32,37 @@ type t = (** A context that holds the internal Seq objects and libraries. *) } +(* + lookup: + file -> line -> [ pos ; pointer ] + origin: + file -> line -> [ doc; type_name ] +*) + +type tinspect = + { pos: Ast_ann.t + ; el: Llvm.Types.typ_t + ; name: string } +let inspect_lookup : + (string, (int, tinspect Stack.t) Hashtbl.t) Hashtbl.t + = String.Table.create () +type torigin = + { doc: string + } +let origin_lookup : + (string, (int, torigin Stack.t) Hashtbl.t) Hashtbl.t + = String.Table.create () + +let add_inspect (pos: Ast_ann.t) name el = + let s = Hashtbl.find_or_add inspect_lookup pos.file ~default:Int.Table.create in + let s = Hashtbl.find_or_add s pos.line ~default:Stack.create in + Stack.push s { pos; el; name } + +let add_origin (pos: Ast_ann.t) doc = + let s = Hashtbl.find_or_add origin_lookup pos.file ~default:Int.Table.create in + let s = Hashtbl.find_or_add s pos.line ~default:Stack.create in + Stack.push s { doc } + (** [add_block context] adds a new block to the context stack. *) let add_block ctx = Stack.push ctx.stack (String.Hash_set.create ()) @@ -54,6 +85,7 @@ let add ~(ctx : t) ?(toplevel = false) ?(global = false) ?(internal = false) ?(a (match Hashtbl.find ctx.map key with | None -> Hashtbl.set ctx.map ~key ~data:[ var ] | Some lst -> Hashtbl.set ctx.map ~key ~data:(var :: lst)); + Hash_set.add (Stack.top_exn ctx.stack) key (** [remove ~ctx name] removes the most recent variable [name] from the namespace. *) diff --git a/parser/seqaml/llvm.ml b/parser/seqaml/llvm.ml index cd8af77a8..74018be29 100644 --- a/parser/seqaml/llvm.ml +++ b/parser/seqaml/llvm.ml @@ -565,6 +565,17 @@ module JIT = struct let init = foreign "jit_init" (void @-> returning t) let var = foreign "jit_var" (t @-> Types.expr @-> returning Types.var) + let get_pos expr = + let str = foreign "get_pos_str" Ctypes.(t @-> returning string) expr in + if str = "" then None else + let l = Array.of_list @@ String.split ~on:'\b' str in + assert (Array.length l = 5); + let file = l.(0) in + let line = Int.of_string l.(1) in + let col = Int.of_string l.(2) in + let len = Int.of_string l.(3) in + Some Ast_ann.{ file; line; col; len } + let func jit fn = let err_addr = Ctypes.allocate (ptr char) (from_voidp char null) in foreign diff --git a/runtime/main.cpp b/runtime/main.cpp index 486409716..270a429b6 100644 --- a/runtime/main.cpp +++ b/runtime/main.cpp @@ -20,6 +20,7 @@ static void versMsg(raw_ostream &out) { } int main(int argc, char **argv) { + repl(""); exit(0); opt input(Positional, desc(""), init("-")); opt debug("d", desc("Compile in debug mode (disable optimizations; " "print LLVM IR to stderr)")); From 72c05164bd755fb830f2f4c2e2902105362a2129 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ibrahim=20Numanagic=CC=81?= Date: Sun, 1 Dec 2019 14:38:47 -0800 Subject: [PATCH 09/30] Fix .gitignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 591381780..f64e1ac53 100644 --- a/.gitignore +++ b/.gitignore @@ -64,3 +64,5 @@ dox # Testing playground # ###################### scratch.seq + +.venv \ No newline at end of file From a6e577023578265fa17c9afa1d7b6dba61e2f953 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ibrahim=20Numanagic=CC=81?= Date: Sun, 1 Dec 2019 21:07:40 -0800 Subject: [PATCH 10/30] Fix JIT issues; Add complete/introspect/doc hooks --- compiler/util/jit.cpp | 60 +++++++++++++++--------- compiler/util/ocaml.cpp | 3 ++ jupyter/seq_kernel/kernel.py | 38 ++++++++++----- jupyter/seq_kernel/wrapper.py | 37 +++++++++++---- jupyter/wrapper.py | 38 +++++++++++++-- parser/jit.ml | 85 +++++++++++++++++++++++++++------- parser/main.ml | 3 ++ parser/runner.ml | 6 +-- parser/seqaml/codegen_intf.mli | 2 +- parser/seqaml/codegen_stmt.ml | 38 ++++++--------- parser/seqaml/llvm.ml | 6 +-- runtime/main.cpp | 7 ++- 12 files changed, 227 insertions(+), 96 deletions(-) diff --git a/compiler/util/jit.cpp b/compiler/util/jit.cpp index 7e2104028..bc803d678 100644 --- a/compiler/util/jit.cpp +++ b/compiler/util/jit.cpp @@ -5,35 +5,49 @@ #define FOREIGN extern "C" -value *caml_jit_init_f() { - static value *closure_f = nullptr; - if (!closure_f) { - closure_f = caml_named_value("jit_init_c"); +#define CAMLDEF(x) \ + value *_jit_##x () { \ + static value *closure_f = nullptr; \ + if (!closure_f) { \ + closure_f = caml_named_value("jit_" #x "_c"); \ + } \ + return closure_f; \ } - return closure_f; -} -value *caml_jit_exec_f() { - static value *closure_f = nullptr; - if (!closure_f) { - closure_f = caml_named_value("jit_exec_c"); - } - return closure_f; -} +CAMLDEF(init) +CAMLDEF(exec) +CAMLDEF(inspect) +CAMLDEF(document) +CAMLDEF(complete) -FOREIGN void *caml_jit_init() { +FOREIGN void *seq_jit_init() { static char *caml_argv[] = {(char *)"main.so", (char *)"--parse", nullptr}; caml_startup(caml_argv); + return (void *)Nativeint_val(caml_callback(*(_jit_init()), Val_unit)); +} + +FOREIGN void seq_jit_exec(void *handle, const char *code) { + caml_callback2(*(_jit_exec()), + caml_copy_nativeint((unsigned long)handle), + caml_copy_string(code)); +} + +FOREIGN char *seq_jit_inspect(void *handle, const char *cell, int line, int col) { + value args[4] = { + caml_copy_nativeint((unsigned long)handle), + caml_copy_string(cell), Val_int(line), Val_int(col) + }; + return String_val(caml_callbackN(*(_jit_inspect()), 4, args)); +} - value *closure_f = caml_jit_init_f(); - void *res = (void *)Nativeint_val(caml_callback(*closure_f, Val_unit)); - // fprintf(stderr, "[link] got %llx\n", res); - return res; +FOREIGN char *seq_jit_document(void *handle, const char *id) { + return String_val(caml_callback2(*(_jit_document()), + caml_copy_nativeint((unsigned long)handle), + caml_copy_string(id))); } -FOREIGN void caml_jit_exec(void *handle, const char *code) { - value *closure_f = caml_jit_exec_f(); - // fprintf(stderr, "[link] ex.got %llx\n", handle); - caml_callback2(*closure_f, caml_copy_nativeint((unsigned long)handle), - caml_copy_string(code)); +FOREIGN char *seq_jit_complete(void *handle, const char *prefix) { + return String_val(caml_callback2(*(_jit_complete()), + caml_copy_nativeint((unsigned long)handle), + caml_copy_string(prefix))); } diff --git a/compiler/util/ocaml.cpp b/compiler/util/ocaml.cpp index 4213bd7f6..425c13a25 100644 --- a/compiler/util/ocaml.cpp +++ b/compiler/util/ocaml.cpp @@ -668,6 +668,9 @@ FOREIGN char* get_pos_str (SrcObject *v) { if (!v) return strdup(""); auto info = v->getSrcInfo(); + fprintf(stderr, "--? %s %d %d %d\n", + info.file.c_str(), info.line, info.col, info.len); + char *er = 0; asprintf(&er, "%s\b%d\b%d\b%d", info.file.c_str(), info.line, info.col, info.len); diff --git a/jupyter/seq_kernel/kernel.py b/jupyter/seq_kernel/kernel.py index 47ab9f008..d36716bf9 100644 --- a/jupyter/seq_kernel/kernel.py +++ b/jupyter/seq_kernel/kernel.py @@ -18,7 +18,7 @@ class SeqKernel(Kernel): def language_version(self): m = version_pat.search(self.banner) return m.group(1) - + _banner = None @property @@ -36,6 +36,7 @@ def banner(self): def __init__(self, **kwargs): Kernel.__init__(self, **kwargs) self.seqwrapper = SeqWrapper() + self.cells = set() def do_execute(self, code, silent, store_history=True, user_expressions=None, allow_stdin=False): if not code.strip(): @@ -44,19 +45,20 @@ def do_execute(self, code, silent, store_history=True, user_expressions=None, al fout = BytesIO() ferr = BytesIO() - + with stdout_stderr_redirector(fout, ferr): - self.seqwrapper.exec(code.rstrip()) - + code = code.rstrip() + self.seqwrapper.exec(code) + self.cells.add(code) + fout_string = fout.getvalue().decode('utf-8').strip() ferr_string = ferr.getvalue().decode('utf-8').strip() - + if ferr_string: if not silent: self.send_response(self.iopub_socket, 'stream', {'name': 'stderr', 'text': ferr_string}) - return {'status': 'error', 'execution_count': self.execution_count} - + else: if not silent: self.send_response(self.iopub_socket, 'stream', {'name': 'stdout', 'text': fout_string}) @@ -64,14 +66,26 @@ def do_execute(self, code, silent, store_history=True, user_expressions=None, al return {'status': 'ok', 'execution_count': self.execution_count, 'payload': [], 'user_expressions': {}} - def _get_object(self, code, cursor_pos): + def _get_object(self, code, cursor_pos, detail_level): # TODO: Get the correct section of code to be inspected - left = code[:cursor_pos].rsplit(' ', 1)[-1] - right = code[cursor_pos:].split(' ', 1)[0] - return left + right + print(code, cursor_pos) + if code not in self.cells: + return None + + cell = self.cells[code] + lines = code.split('\n') + line = code[:cursor_pos].count('\n') + col = cursor_pos - code[:cursor_pos].rsearch('\n') + + fout = BytesIO() + ferr = BytesIO() + with stdout_stderr_redirector(fout, ferr): + self.seqwrapper.inspect(cell, line, col) + ferr_string = ferr.getvalue().decode('utf-8').strip() + return {'text/plain': f'Hello, World! {cell} {line} {col} -> {ferr_string}'} def do_inspect(self, code, cursor_pos, detail_level=0): - data = self.seqwrapper.inspect(self._get_object(code, cursor_pos), detail_level) + data = self._get_object(code, cursor_pos, detail_level) return { 'status': 'ok', 'found': True, diff --git a/jupyter/seq_kernel/wrapper.py b/jupyter/seq_kernel/wrapper.py index 831308860..fcf3f3dda 100644 --- a/jupyter/seq_kernel/wrapper.py +++ b/jupyter/seq_kernel/wrapper.py @@ -5,17 +5,38 @@ def __init__(self): lib = ctypes.util.find_library("seqjit") self._lib = ctypes.CDLL(lib, ctypes.RTLD_GLOBAL) - self._init_fn = self._lib.caml_jit_init + + self._init_fn = self._lib.seq_jit_init self._init_fn.restype = ctypes.c_void_p - self._exec_fn = self._lib.caml_jit_exec + + self._exec_fn = self._lib.seq_jit_exec self._exec_fn.argtypes = [ctypes.c_void_p, ctypes.c_char_p] + + self._inspect_fn = self._lib.seq_jit_inspect + self._inspect_fn.argtypes = [ + ctypes.c_void_p, ctypes.c_char_p, ctypes.c_int, ctypes.c_int + ] + self._inspect_fn.restype = ctypes.c_char_p + + self._document_fn = self._lib.seq_jit_document + self._document_fn.argtypes = [ctypes.c_void_p, ctypes.c_char_p] + self._document_fn.restype = ctypes.c_char_p + + self._complete_fn = self._lib.seq_jit_complete + self._complete_fn.argtypes = [ctypes.c_void_p, ctypes.c_char_p] + self._complete_fn.restype = ctypes.c_char_p + self.handle = self._init_fn() def exec(self, code): self._exec_fn(self.handle, code.encode('utf-8')) - - def inspect(self, request_object, detail_level): - # TODO: Inspect the requested object and return the doc and/or source - return { - 'text/plain': f'Hello, World! {request_object} {detail_level}' - } + + def inspect(self, cell, line, column): + return self._inspect_fn(self.handle, str(cell).encode('utf-8'), line, column).decode('ascii') + + def document(self, idn): + return self._document_fn(self.handle, idn.encode('utf-8')).decode('ascii') + + def complete(self, prefix): + l = self._complete_fn(self.handle, prefix.encode('utf-8')).decode('ascii') + return l.split('\b') \ No newline at end of file diff --git a/jupyter/wrapper.py b/jupyter/wrapper.py index 40ec65a2b..092a40756 100644 --- a/jupyter/wrapper.py +++ b/jupyter/wrapper.py @@ -7,15 +7,42 @@ def __init__(self): lib = ctypes.util.find_library("seqjit") self._lib = ctypes.CDLL(lib) - self._init_fn = self._lib.caml_jit_init + + self._init_fn = self._lib.seq_jit_init self._init_fn.restype = ctypes.c_void_p - self._exec_fn = self._lib.caml_jit_exec + + self._exec_fn = self._lib.seq_jit_exec self._exec_fn.argtypes = [ctypes.c_void_p, ctypes.c_char_p] + + self._inspect_fn = self._lib.seq_jit_inspect + self._inspect_fn.argtypes = [ + ctypes.c_void_p, ctypes.c_char_p, ctypes.c_int, ctypes.c_int + ] + self._inspect_fn.restype = ctypes.c_char_p + + self._document_fn = self._lib.seq_jit_document + self._document_fn.argtypes = [ctypes.c_void_p, ctypes.c_char_p] + self._document_fn.restype = ctypes.c_char_p + + self._complete_fn = self._lib.seq_jit_complete + self._complete_fn.argtypes = [ctypes.c_void_p, ctypes.c_char_p] + self._complete_fn.restype = ctypes.c_char_p + self.handle = self._init_fn() def exec(self, code): self._exec_fn(self.handle, code.encode('utf-8')) + def inspect(self, cell, line, column): + return self._inspect_fn(self.handle, str(cell).encode('utf-8'), line, column).decode('ascii') + + def document(self, idn): + return self._document_fn(self.handle, idn.encode('utf-8')).decode('ascii') + + def complete(self, prefix): + l = self._complete_fn(self.handle, prefix.encode('utf-8')).decode('ascii') + return l.split('\b') + s = SeqWrapper() s.exec(""" @@ -32,4 +59,9 @@ def foo(bar): x += 1 print x foo(x+100) -""") \ No newline at end of file +""") + +print('i: ', s.inspect(1, 2, 7)) +print('d: ', s.document("x")) +print('c: ', s.complete("m")) + diff --git a/parser/jit.ml b/parser/jit.ml index 72d33e0a3..b6509da7a 100644 --- a/parser/jit.ml +++ b/parser/jit.ml @@ -26,7 +26,7 @@ let init () : t = ~base:anon_fn ~block:(Llvm.Block.func anon_fn) ~jit:true - (Runner.exec_string ~debug:false ~jit:true) + (Runner.exec_string ~debug:false ~jit:true ~cell:false) in let jit = { cnt = 1; ctx } in (* load stdlib *) @@ -52,7 +52,7 @@ let exec (jit : t) code = in Ctx.add_block anon_ctx; jit.cnt <- jit.cnt + 1; - Runner.exec_string ~file:"" ~jit:true anon_ctx code; + Runner.exec_string ~file:"" ~jit:true ~cell:true anon_ctx code; try Llvm.JIT.func jit.ctx.mdl anon_fn; Hash_set.iter (Stack.pop_exn anon_ctx.stack) ~f:(fun key -> @@ -65,21 +65,38 @@ let exec (jit : t) code = with | Err.SeqCError (msg, pos) -> raise @@ Err.CompilerError (Compiler msg, [ pos ]) -let locate cmd f l c = +let locate f l c = let open Option.Monad_infix in + let f = sprintf "" f in + Some (sprintf "Cell %s, line %d, col %d" f l c) + (* Hashtbl.iteri Ctx.inspect_lookup ~f:(fun ~key ~data -> + eprintf "File %s:\n" key; + Hashtbl.iteri data ~f:(fun ~key ~data -> + eprintf " Line %d:\n" key; + Stack.iter data ~f:(fun {name;pos;el} -> + eprintf " %s: %s << " name (Ast.Ann.to_string pos); + ( match Llvm.JIT.get_pos el with + | None -> eprintf "-"; + | Some orig_pos -> + eprintf "%s" @@ Ast.Ann.to_string orig_pos ); + eprintf "\n"; + ); + ) + ); + eprintf "\n%!"; + Hashtbl.find Ctx.inspect_lookup f >>= (fun t -> Hashtbl.find t l) >>= (fun s -> Stack.find_map s ~f:(fun Ctx.{ pos; el; name } -> - if pos.col < c || pos.col + pos.len >= c + if pos.col < c || pos.col + pos.len >= c then None else match Llvm.JIT.get_pos el with | None -> None | Some orig_pos -> - eprintf "%%JIT%%: found %s, orig pos %s" - name (Ast.Ann.to_string orig_pos); - Some 1 - )) - + Some (sprintf "%%JIT%%: found %s, orig pos %s" + name (Ast.Ann.to_string orig_pos)) + )) *) + (** JIT entry point. *) let repl () = @@ -105,15 +122,15 @@ let repl () = eprintf "------------------\n%!"; match String.is_prefix ~prefix:"%%" !code with | true -> - let ll = Array.of_list @@ String.split ~on:' ' !code in - let cmd = ll.(0) in + let ll = Array.of_list @@ String.split ~on:' ' @@ String.strip !code in + (* let cmd = ll.(0) in *) let file = ll.(1) in let line = Int.of_string @@ ll.(2) in let pos = Int.of_string @@ ll.(3) in - ignore @@ locate cmd file line pos; + ignore @@ locate file line pos; code := ""; start := true - | false -> + | false -> (try exec jit !code with | Err.CompilerError (typ, pos_lst) -> eprintf "%s\n%!" @@ Err.to_string ~pos_lst ~file:!code typ); @@ -141,13 +158,45 @@ let c_exec hnd code = let jit = Hashtbl.find_exn jits hnd in (* Hashtbl.iter_keys jit.ctx.map ~f:(fun k -> eprintf "[lib] keys: %s ... \n%!" k); *) - try - exec jit code - with - | Err.CompilerError (typ, pos_lst) -> - eprintf "%s\n%!" @@ Err.to_string ~pos_lst ~file:code typ + match String.is_prefix ~prefix:"%%" code with + | true -> + let ll = Array.of_list @@ String.split ~on:' ' code in + (* let cmd = ll.(0) in *) + let file = ll.(1) in + let line = Int.of_string @@ ll.(2) in + let pos = Int.of_string @@ ll.(3) in + ignore @@ locate file line pos; + | false -> + try + exec jit code + with + | Err.CompilerError (typ, pos_lst) -> + eprintf "%s\n%!" @@ Err.to_string ~pos_lst ~file:code typ let c_close hnd = let jit = Hashtbl.find_exn jits hnd in eprintf "Closing JIT handle, %d commands executed\n%!" jit.cnt; () + +let c_inspect hnd file line col = + let jit = Hashtbl.find_exn jits hnd in + (* match locate file line col with + | Some s -> + sprintf "JIT %s\n" s; + | None -> *) + sprintf "JIT %s %d %d" file line col + +let c_document hnd identifier = + sprintf "Some docs for %s --- to be done!" identifier + (* match locate file line col with + | Some s -> + eprintf "JIT %s\n" s; + | None -> + eprintf "JIT\n" *) + +let c_complete hnd prefix = + let jit = Hashtbl.find_exn jits hnd in + Hashtbl.filter_mapi jit.ctx.map ~f:(fun ~key ~data -> + if String.is_prefix ~prefix key then Some data else None) + |> Hashtbl.keys + |> String.concat ~sep:"\b" \ No newline at end of file diff --git a/parser/main.ml b/parser/main.ml index 1a2ec21c9..6e1d1d3e2 100644 --- a/parser/main.ml +++ b/parser/main.ml @@ -12,6 +12,9 @@ let () = Callback.register "parse_c" Runner.parse_c; Callback.register "jit_init_c" Jit.c_init; Callback.register "jit_exec_c" Jit.c_exec; + Callback.register "jit_inspect_c" Jit.c_inspect; + Callback.register "jit_document_c" Jit.c_document; + Callback.register "jit_complete_c" Jit.c_complete; match List.nth (Array.to_list Sys.argv) 1 with | None -> Jit.repl () | Some "--parse" -> () diff --git a/parser/runner.ml b/parser/runner.ml index dd31d1847..ad8f12302 100644 --- a/parser/runner.ml +++ b/parser/runner.ml @@ -11,8 +11,8 @@ open Seqaml (** [exec_string ~file ~debug context code] parses a code within string [code] as a module and returns parsed module AST. [file] is code filename used for error reporting. *) -let exec_string ctx ?(file = "") ?(debug = false) ?(jit = false) code = - ignore @@ Codegen.parse ~file code ~f:(Codegen.Stmt.parse_module ~jit ~ctx) +let exec_string ctx ?(file = "") ?(debug = false) ?(jit = false) ?(cell=false) code = + ignore @@ Codegen.parse ~file code ~f:(Codegen.Stmt.parse_module ~cell ~jit ~ctx) (** [init file error_handler] initializes Seq session with file [file]. [error_handler typ position] is a callback called upon encountering @@ -20,7 +20,7 @@ let exec_string ctx ?(file = "") ?(debug = false) ?(jit = false) code let init ~filename code error_handler = let mdl = Llvm.Module.init () in try - let exec = exec_string ~debug:false ~jit:false in + let exec = exec_string ~debug:false ~jit:false ~cell:false in let ctx = Ctx.init_module ~filename ~mdl ~base:mdl ~block:(Llvm.Module.block mdl) exec in (* parse the file *) Ctx.parse_code ~ctx ~file:filename code; diff --git a/parser/seqaml/codegen_intf.mli b/parser/seqaml/codegen_intf.mli index acf7eb05f..f56d66ca6 100644 --- a/parser/seqaml/codegen_intf.mli +++ b/parser/seqaml/codegen_intf.mli @@ -28,7 +28,7 @@ module type Stmt = sig -> Llvm.Types.stmt_t (** Parses a module ([Ast.t]) AST *) - val parse_module : ?jit:bool -> ctx:Ctx.t -> Stmt.t Ann.ann list -> unit + val parse_module : ?cell:bool -> ?jit:bool -> ctx:Ctx.t -> Stmt.t Ann.ann list -> unit (** Parses a [For] statement AST. Public in order to allow access to it from [ExprIntf]. *) val parse_for diff --git a/parser/seqaml/codegen_stmt.ml b/parser/seqaml/codegen_stmt.ml index a2dcb0a74..efd5c0db7 100644 --- a/parser/seqaml/codegen_stmt.ml +++ b/parser/seqaml/codegen_stmt.ml @@ -55,34 +55,18 @@ module Codegen (E : Codegen_intf.Expr) : Codegen_intf.Stmt = struct (** [parse_module ~ctx stmts] parses a module. A module is just a simple list [stmts] of statements. *) - and parse_module ?(jit = false) ~(ctx : Ctx.t) stmts = + and parse_module ?(cell=false) ?(jit = false) ~(ctx : Ctx.t) stmts = (* Util.dbg "%s\n" @@ Util.ppl ~sep:"\n" stmts ~f:Ast.Stmt.to_string; *) let stmts = - if jit + if jit && cell then List.rev @@ match List.rev stmts with | (pos, Expr e) :: tl -> (pos, Print ([ e ], "\n")) :: tl | l -> l - else - List.map stmts ~f:(fun s -> - (match snd s with - | Generic (Function p) -> - let name = p.fn_name.name in - let fn = Llvm.Func.func name in - let names = List.map p.fn_args ~f:(fun (_, x) -> x.name) in - Ctx.add ~ctx ~toplevel:true ~global:true ~attrs:["preload"] name (Ctx_namespace.Func (fn, names)) - | Generic (Class p) -> - let name = p.class_name in - let t = Llvm.Type.cls name in - Ctx.add ~ctx ~toplevel:true ~global:true ~attrs:["preload"] name (Ctx_namespace.Type t) - | Generic (Type p) -> - let name = p.class_name in - let t = Llvm.Type.record [] [] name in - Ctx.add ~ctx ~toplevel:true ~global:true ~attrs:["preload"] name (Ctx_namespace.Type t) - | _ -> ()); - s) + else + stmts in ignore @@ List.map stmts ~f:(parse ~ctx ~toplevel:true ~jit) @@ -142,6 +126,7 @@ module Codegen (E : Codegen_intf.Expr) : Codegen_intf.Stmt = struct let var_expr = Llvm.Expr.var v in (* eprintf ">> jit_var %s := %s\n%!" (Ast.Expr.to_string lhs) (Ast.Expr.to_string rhs); *) (* finalize ~ctx stmt pos; *) + (* Llvm.Stmt.set_pos v (fst lhs); *) Ctx.add ~ctx ~toplevel ~global:true var (Ctx_namespace.Var v); (* Llvm.Stmt.expr var_expr *) (* Llvm.Stmt.assign v rh_expr *) @@ -153,6 +138,7 @@ module Codegen (E : Codegen_intf.Expr) : Codegen_intf.Stmt = struct let var_stmt = Llvm.Stmt.var ~typ rh_expr in let v = Llvm.Var.stmt var_stmt in if toplevel then Llvm.Var.set_global v; + (* Llvm.Stmt.set_pos v (fst lhs); *) Ctx.add ~ctx ~toplevel ~global:toplevel var (Ctx_namespace.Var v); var_stmt) | pos, Dot (lh_lhs, lh_rhs) -> @@ -185,6 +171,7 @@ module Codegen (E : Codegen_intf.Expr) : Codegen_intf.Stmt = struct let var_stmt = Llvm.Stmt.var ~typ Ctypes.null in let v = Llvm.Var.stmt var_stmt in if toplevel then Llvm.Var.set_global v; + (* Llvm.Stmt.set_pos v pos; *) Ctx.add ~ctx ~toplevel ~global:toplevel name (Ctx_namespace.Var v); var_stmt @@ -235,6 +222,7 @@ module Codegen (E : Codegen_intf.Expr) : Codegen_intf.Stmt = struct and parse_type_alias ctx pos ~toplevel (name, expr) = let typ = E.parse_type ~ctx expr in + (* Llvm.Stmt.set_pos typ pos; *) Ctx.add ~ctx ~toplevel ~global:toplevel name (Ctx_namespace.Type typ); Llvm.Stmt.pass () @@ -382,6 +370,7 @@ module Codegen (E : Codegen_intf.Expr) : Codegen_intf.Stmt = struct let typ = E.parse_type ~ctx (Option.value_exn ext.e_name.typ) in Llvm.Func.set_type fn typ; let names = List.map ext.e_args ~f:(fun (_, x) -> x.name) in + (* Llvm.Stmt.set_pos fn pos; *) Ctx.add ~ctx ~toplevel ~global:toplevel ctx_name (Ctx_namespace.Func (fn, names)); if toplevel && jit then ( ignore @@ finalize ~ctx ~add:false (Llvm.Stmt.func fn) pos; @@ -406,6 +395,7 @@ module Codegen (E : Codegen_intf.Expr) : Codegen_intf.Stmt = struct match snd g with | Id n -> (* Util.dbg ">> extend %s :: add %s" (Ast.Expr.to_string name) n; *) + (* Llvm.Stmt.set_pos t (fst g); *) Ctx.add ~ctx:new_ctx n (Ctx_namespace.Type t) | _ -> serr ~pos:(fst g) "not a valid generic specifier") in @@ -507,13 +497,14 @@ module Codegen (E : Codegen_intf.Expr) : Codegen_intf.Stmt = struct Llvm.Type.add_cls_method cls name fn; fn | None -> - match toplevel, Ctx.in_block ~ctx name with + match toplevel, Ctx.in_block ~ctx name with | true, Some ((Ctx_namespace.Func (fn, names)), annt) when Hash_set.mem annt.attrs "preload" -> ( if ctx.base <> ctx.mdl then Llvm.Func.set_enclosing fn ctx.base ); fn - | _ -> + | _ -> let fn = Llvm.Func.func name in let names = List.map fn_args ~f:(fun (_, x) -> x.name) in + (* Llvm.Stmt.set_pos fn pos; *) Ctx.add ~ctx ~toplevel ~global:toplevel name (Ctx_namespace.Func (fn, names)); ( if ctx.base <> ctx.mdl then Llvm.Func.set_enclosing fn ctx.base ); fn @@ -568,7 +559,7 @@ module Codegen (E : Codegen_intf.Expr) : Codegen_intf.Stmt = struct and parse_class ctx pos ~toplevel ?(is_type = false) cls = let typ = - match toplevel, Ctx.in_block ~ctx cls.class_name with + match toplevel, Ctx.in_block ~ctx cls.class_name with | true, Some ((Ctx_namespace.Type typ), annt) when Hash_set.mem annt.attrs "preload" -> typ | _ -> @@ -577,6 +568,7 @@ module Codegen (E : Codegen_intf.Expr) : Codegen_intf.Stmt = struct then Llvm.Type.record [] [] cls.class_name else Llvm.Type.cls cls.class_name in + (* Llvm.Stmt.set_pos typ pos; *) Ctx.add ~ctx ~toplevel ~global:toplevel cls.class_name (Ctx_namespace.Type typ); typ in diff --git a/parser/seqaml/llvm.ml b/parser/seqaml/llvm.ml index 74018be29..90ce66aaf 100644 --- a/parser/seqaml/llvm.ml +++ b/parser/seqaml/llvm.ml @@ -567,15 +567,15 @@ module JIT = struct let get_pos expr = let str = foreign "get_pos_str" Ctypes.(t @-> returning string) expr in - if str = "" then None else + if str = "" then None else let l = Array.of_list @@ String.split ~on:'\b' str in - assert (Array.length l = 5); + assert (Array.length l = 4); let file = l.(0) in let line = Int.of_string l.(1) in let col = Int.of_string l.(2) in let len = Int.of_string l.(3) in Some Ast_ann.{ file; line; col; len } - + let func jit fn = let err_addr = Ctypes.allocate (ptr char) (from_voidp char null) in foreign diff --git a/runtime/main.cpp b/runtime/main.cpp index f5c862fc4..d61c690e8 100644 --- a/runtime/main.cpp +++ b/runtime/main.cpp @@ -20,8 +20,11 @@ static void versMsg(raw_ostream &out) { } int main(int argc, char **argv) { - repl(argv[0]); - exit(0); + if (argc==2 && string(argv[1])=="--repl") + { + repl(argv[0]); + exit(0); + } opt input(Positional, desc(""), init("-")); opt debug("d", desc("Compile in debug mode (disable optimizations; " From 7030a27ada28cc47493ac1ab6d7ec9fa0091bc85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ibrahim=20Numanagi=C4=87?= Date: Mon, 2 Dec 2019 12:06:34 -0800 Subject: [PATCH 11/30] Full support for JIT autocompletion and introspection --- compiler/util/ocaml.cpp | 5 +- jupyter/wrapper.py | 10 +- parser/jit.ml | 205 ++++++++++++++++++------------- parser/runner.ml | 6 +- parser/seqaml/ast.ml | 2 + parser/seqaml/ast_stmt.ml | 1 + parser/seqaml/codegen_expr.ml | 8 +- parser/seqaml/codegen_intf.mli | 3 +- parser/seqaml/codegen_stmt.ml | 73 +++++++---- parser/seqaml/ctx.ml | 72 ++++++----- parser/seqaml/grammar.mly | 3 +- parser/seqaml/llvm.ml | 6 + stdlib/core/collections/list.seq | 2 + 13 files changed, 235 insertions(+), 161 deletions(-) diff --git a/compiler/util/ocaml.cpp b/compiler/util/ocaml.cpp index 425c13a25..1d21cff27 100644 --- a/compiler/util/ocaml.cpp +++ b/compiler/util/ocaml.cpp @@ -628,6 +628,8 @@ FOREIGN char is_ref_type(types::Type *t) { FOREIGN BaseFunc *get_func(FuncExpr *e) { return e->getFunc(); } +FOREIGN types::Type *get_func_type(Func *e) { return e->getFuncType(); } + FOREIGN types::Type *get_var_type(Var *e) { return e->getType(); } FOREIGN void set_base(Stmt *st, BaseFunc *base) { @@ -668,9 +670,6 @@ FOREIGN char* get_pos_str (SrcObject *v) { if (!v) return strdup(""); auto info = v->getSrcInfo(); - fprintf(stderr, "--? %s %d %d %d\n", - info.file.c_str(), info.line, info.col, info.len); - char *er = 0; asprintf(&er, "%s\b%d\b%d\b%d", info.file.c_str(), info.line, info.col, info.len); diff --git a/jupyter/wrapper.py b/jupyter/wrapper.py index 092a40756..47c04263c 100644 --- a/jupyter/wrapper.py +++ b/jupyter/wrapper.py @@ -34,7 +34,8 @@ def exec(self, code): self._exec_fn(self.handle, code.encode('utf-8')) def inspect(self, cell, line, column): - return self._inspect_fn(self.handle, str(cell).encode('utf-8'), line, column).decode('ascii') + file = f'' + return self._inspect_fn(self.handle, file.encode('utf-8'), line, column).decode('ascii') def document(self, idn): return self._document_fn(self.handle, idn.encode('utf-8')).decode('ascii') @@ -50,6 +51,7 @@ def complete(self, prefix): x = 1 print x def foo(bar): + ''' HAI ''' print 'foo', bar y = 2 print x, y, x + y @@ -61,7 +63,9 @@ def foo(bar): foo(x+100) """) -print('i: ', s.inspect(1, 2, 7)) -print('d: ', s.document("x")) +print('i: ', s.inspect(2, 3, 6)) +print('i: ', s.inspect(2, 4, 1)) +print('d: ', s.document("foo")) +print('d: ', s.document("list")) print('c: ', s.complete("m")) diff --git a/parser/jit.ml b/parser/jit.ml index b6509da7a..c700cf9a5 100644 --- a/parser/jit.ml +++ b/parser/jit.ml @@ -26,7 +26,7 @@ let init () : t = ~base:anon_fn ~block:(Llvm.Block.func anon_fn) ~jit:true - (Runner.exec_string ~debug:false ~jit:true ~cell:false) + (Runner.exec_string ~debug:false ~cell:false) in let jit = { cnt = 1; ctx } in (* load stdlib *) @@ -42,6 +42,7 @@ let init () : t = (** Execute [code] within a JIT context [jit]. *) let exec (jit : t) code = + let file = sprintf "" jit.cnt in let anon_fn = Llvm.Func.func (sprintf "" jit.cnt) in let anon_ctx = { jit.ctx with @@ -52,7 +53,7 @@ let exec (jit : t) code = in Ctx.add_block anon_ctx; jit.cnt <- jit.cnt + 1; - Runner.exec_string ~file:"" ~jit:true ~cell:true anon_ctx code; + Runner.exec_string ~file ~cell:true anon_ctx code; try Llvm.JIT.func jit.ctx.mdl anon_fn; Hash_set.iter (Stack.pop_exn anon_ctx.stack) ~f:(fun key -> @@ -65,84 +66,58 @@ let exec (jit : t) code = with | Err.SeqCError (msg, pos) -> raise @@ Err.CompilerError (Compiler msg, [ pos ]) -let locate f l c = +let locate ~(ctx : Ctx.t) ?(full = true) f l c = let open Option.Monad_infix in - let f = sprintf "" f in - Some (sprintf "Cell %s, line %d, col %d" f l c) - (* Hashtbl.iteri Ctx.inspect_lookup ~f:(fun ~key ~data -> - eprintf "File %s:\n" key; - Hashtbl.iteri data ~f:(fun ~key ~data -> - eprintf " Line %d:\n" key; - Stack.iter data ~f:(fun {name;pos;el} -> - eprintf " %s: %s << " name (Ast.Ann.to_string pos); - ( match Llvm.JIT.get_pos el with - | None -> eprintf "-"; - | Some orig_pos -> - eprintf "%s" @@ Ast.Ann.to_string orig_pos ); - eprintf "\n"; - ); - ) - ); - eprintf "\n%!"; - - Hashtbl.find Ctx.inspect_lookup f + (* let f = sprintf "" f in *) + (* Some (sprintf "Cell %s, line %d, col %d" f l c) *) + Hashtbl.find ctx.inspect_lookup f >>= (fun t -> Hashtbl.find t l) >>= (fun s -> Stack.find_map s ~f:(fun Ctx.{ pos; el; name } -> - if pos.col < c || pos.col + pos.len >= c + if c < pos.col || c >= pos.col + pos.len then None - else match Llvm.JIT.get_pos el with - | None -> None - | Some orig_pos -> - Some (sprintf "%%JIT%%: found %s, orig pos %s" - name (Ast.Ann.to_string orig_pos)) - )) *) - - -(** JIT entry point. *) -let repl () = - let banner = String.make 78 '=' in - eprintf "\027[102m%s\027[0m \n" banner; - eprintf "\027[102m=%s%s%s=\027[0m \n" - (String.make 34 ' ') "Seq REPL" (String.make 34 ' '); - eprintf "\027[102m%s\027[0m \n%!" banner; - let jit = init () in - let start = ref true in - let code = ref "" in - try - while true do - try - if !start - then ( - eprintf "\027[92min[%d]>\027[0m \n%!" jit.cnt; - start := false); - let s = In_channel.(input_line_exn stdin) in - code := !code ^ s ^ "\n" - with - | End_of_file -> - eprintf "------------------\n%!"; - match String.is_prefix ~prefix:"%%" !code with - | true -> - let ll = Array.of_list @@ String.split ~on:' ' @@ String.strip !code in - (* let cmd = ll.(0) in *) - let file = ll.(1) in - let line = Int.of_string @@ ll.(2) in - let pos = Int.of_string @@ ll.(3) in - ignore @@ locate file line pos; - code := ""; - start := true - | false -> - (try exec jit !code with - | Err.CompilerError (typ, pos_lst) -> - eprintf "%s\n%!" @@ Err.to_string ~pos_lst ~file:!code typ); - if !code = "" - then raise Exit - else ( - code := ""; - start := true) - done - with - | Exit -> eprintf "\n\027[31mbye (%d) \027[0m\n%!" jit.cnt - + else ( + (* eprintf "> querying %s: %nx\n%!" name (Ctypes.raw_address_of_ptr el); *) + let gt, el = match el with + | IVar v -> Llvm.Var.get_type, v + | IFunc v -> Llvm.Func.get_type, v + | IType v -> (fun f -> f), v + | IExpr v -> Llvm.Expr.get_type, v + | IImport v -> (fun f -> f), Ctypes.null + in + match el, Hashtbl.find ctx.inspect_var_lookup el with + | t, Some orig when t <> Ctypes.null -> + let typ = Llvm.Type.get_name (gt el) in + Some ( + sprintf "%s |- %s => defined at %s%s" + orig.name + typ + (Ast.Ann.to_string orig.pos) + (if full && orig.doc <> "" then sprintf "\nDocs: %s" orig.doc else "")) + | _ -> None + ))) + +let locate_dump ~(ctx : Ctx.t) = + Hashtbl.iteri ctx.inspect_lookup ~f:(fun ~key ~data -> + if String.is_prefix key ~prefix:" + eprintf " Line %d:\n" key; + Stack.iter data ~f:(fun {name;pos;el} -> + eprintf " %s\n" + (Option.value (locate ~ctx file key pos.col) ~default:"?"))))); + eprintf "%!" + +let document ctx identifier = + let el = match Ctx.in_block ~ctx identifier with + | Some (Ctx_namespace.Type v, _) -> v + | Some (Ctx_namespace.Func (v, _), _) -> v + | _ -> Ctypes.null + in + if el = Ctypes.null then None + else match Hashtbl.find ctx.inspect_var_lookup el with + | Some { doc; _ } -> Some doc + | None -> None let jits: (nativeint, t) Hashtbl.t = Hashtbl.Poly.create () @@ -180,23 +155,77 @@ let c_close hnd = let c_inspect hnd file line col = let jit = Hashtbl.find_exn jits hnd in - (* match locate file line col with + (* locate_dump ~ctx:jit.ctx; *) + match locate ~ctx:jit.ctx file line col with | Some s -> - sprintf "JIT %s\n" s; - | None -> *) - sprintf "JIT %s %d %d" file line col - -let c_document hnd identifier = - sprintf "Some docs for %s --- to be done!" identifier - (* match locate file line col with - | Some s -> - eprintf "JIT %s\n" s; + sprintf "Found: %s" s; | None -> - eprintf "JIT\n" *) + sprintf "Not found: %s %d %d" file line col +let c_document hnd identifier = + let jit = Hashtbl.find_exn jits hnd in + match document jit.ctx identifier with + | Some d -> d + | None -> "" + let c_complete hnd prefix = let jit = Hashtbl.find_exn jits hnd in Hashtbl.filter_mapi jit.ctx.map ~f:(fun ~key ~data -> if String.is_prefix ~prefix key then Some data else None) |> Hashtbl.keys - |> String.concat ~sep:"\b" \ No newline at end of file + |> String.concat ~sep:"\b" + + +(** JIT entry point. *) +let repl () = + let banner = String.make 78 '=' in + eprintf "\027[102m%s\027[0m \n" banner; + eprintf "\027[102m=%s%s%s=\027[0m \n" + (String.make 34 ' ') "Seq REPL" (String.make 34 ' '); + eprintf "\027[102m%s\027[0m \n%!" banner; + let jit = init () in + let start = ref true in + let code = ref "" in + try + while true do + try + if !start + then ( + eprintf "\027[92min[%d]>\027[0m \n%!" jit.cnt; + start := false); + let s = In_channel.(input_line_exn stdin) in + code := !code ^ s ^ "\n" + with + | End_of_file -> + eprintf "------------------\n%!"; + match String.is_prefix ~prefix:"%%" !code with + | true -> + let ll = Array.of_list @@ String.split ~on:' ' @@ String.strip !code in + ( match ll.(0) with + | "%%i" -> + let file = ll.(1) in + let line = Int.of_string @@ ll.(2) in + let pos = Int.of_string @@ ll.(3) in + let str = match locate ~ctx:jit.ctx file line pos with + | Some s -> sprintf "" s + | None -> "" + in + eprintf "%s\n%!" str + + | _ -> + eprintf "Unknown JIT command\n%!" + ); + code := ""; + start := true + | false -> + (try exec jit !code with + | Err.CompilerError (typ, pos_lst) -> + eprintf "%s\n%!" @@ Err.to_string ~pos_lst ~file:!code typ); + if !code = "" + then raise Exit + else ( + code := ""; + start := true) + done + with + | Exit -> eprintf "\n\027[31mbye (%d) \027[0m\n%!" jit.cnt diff --git a/parser/runner.ml b/parser/runner.ml index ad8f12302..494260e2a 100644 --- a/parser/runner.ml +++ b/parser/runner.ml @@ -11,8 +11,8 @@ open Seqaml (** [exec_string ~file ~debug context code] parses a code within string [code] as a module and returns parsed module AST. [file] is code filename used for error reporting. *) -let exec_string ctx ?(file = "") ?(debug = false) ?(jit = false) ?(cell=false) code = - ignore @@ Codegen.parse ~file code ~f:(Codegen.Stmt.parse_module ~cell ~jit ~ctx) +let exec_string ctx ?(file = "") ?(debug = false) ?(cell=false) code = + ignore @@ Codegen.parse ~file code ~f:(Codegen.Stmt.parse_module ~cell ~ctx) (** [init file error_handler] initializes Seq session with file [file]. [error_handler typ position] is a callback called upon encountering @@ -20,7 +20,7 @@ let exec_string ctx ?(file = "") ?(debug = false) ?(jit = false) ?(cel let init ~filename code error_handler = let mdl = Llvm.Module.init () in try - let exec = exec_string ~debug:false ~jit:false ~cell:false in + let exec = exec_string ~debug:false ~cell:false in let ctx = Ctx.init_module ~filename ~mdl ~base:mdl ~block:(Llvm.Module.block mdl) exec in (* parse the file *) Ctx.parse_code ~ctx ~file:filename code; diff --git a/parser/seqaml/ast.ml b/parser/seqaml/ast.ml index a7cfa203c..87784cf3d 100644 --- a/parser/seqaml/ast.ml +++ b/parser/seqaml/ast.ml @@ -252,6 +252,8 @@ module Stmt = struct (ppl f.fn_args ~f:(param_to_string ~pythonic)) (Option.value_map f.fn_name.typ ~default:"" ~f:(fun x -> sprintf " -> %s" (e_to_string x))) (ppl f.fn_stmts ~sep:"\n" ~f:(to_string ~indent:(indent + 1))) + | Generic (Docstring x) -> + sprintf "\"%s\"" (String.escaped x) | Generic (Class f | Type f) -> sprintf "%s %s%s%s%s" ( match pythonic, snd snode with diff --git a/parser/seqaml/ast_stmt.ml b/parser/seqaml/ast_stmt.ml index 6d026635c..036e1de9f 100644 --- a/parser/seqaml/ast_stmt.ml +++ b/parser/seqaml/ast_stmt.ml @@ -64,6 +64,7 @@ and generic = | Class of class_t | Type of class_t | Declare of param + | Docstring of string and if_case = { cond : texpr ann option diff --git a/parser/seqaml/codegen_expr.ml b/parser/seqaml/codegen_expr.ml index ba3abf114..5646839be 100644 --- a/parser/seqaml/codegen_expr.ml +++ b/parser/seqaml/codegen_expr.ml @@ -136,18 +136,18 @@ module Codegen (S : Codegen_intf.Stmt) : Codegen_intf.Expr = struct (* Make sure that a variable is either accessible within the same base (function) or that it is global variable *) | _, Some ((Ctx_namespace.Type t, _) :: _) -> - Ctx.add_inspect pos var t; + Ctx.add_inspect ~ctx pos var (Ctx.IType t); Llvm.Expr.typ t | true, _ -> serr ~pos "type '%s' not found or realized" var | false, Some ((Ctx_namespace.Var v, { base; global; _ }) :: _) when ctx.base = base || global -> - Ctx.add_inspect pos var v; + Ctx.add_inspect ~ctx pos var (Ctx.IVar v); let e = Llvm.Expr.var v in if global && ctx.base = base && Stack.exists ctx.flags ~f:(( = ) "atomic") then Llvm.Var.set_atomic e; e | false, Some ((Ctx_namespace.Func (t, _), _) :: _) -> - Ctx.add_inspect pos var t; + Ctx.add_inspect ~ctx pos var (Ctx.IFunc t); Llvm.Expr.func t | _ -> serr ~pos "identifier '%s' not found" var @@ -436,7 +436,7 @@ module Codegen (S : Codegen_intf.Stmt) : Codegen_intf.Expr = struct Llvm.Expr.static typ rhs) else Llvm.Expr.element lh_expr rhs in - Ctx.add_inspect npos (sprintf ".%s" rhs) el; + Ctx.add_inspect ~ctx npos (sprintf ".%s" rhs) (Ctx.IExpr el); el and parse_slice ctx pos (a, b, c) = diff --git a/parser/seqaml/codegen_intf.mli b/parser/seqaml/codegen_intf.mli index f56d66ca6..d0ec180d2 100644 --- a/parser/seqaml/codegen_intf.mli +++ b/parser/seqaml/codegen_intf.mli @@ -22,13 +22,12 @@ module type Stmt = sig (** Parses an [Ast_stmt.t] within a context [Ctx.t] and returns a LLVM handle. *) val parse : ?toplevel:bool - -> ?jit:bool -> ctx:Ctx.t -> Stmt.t Ann.ann -> Llvm.Types.stmt_t (** Parses a module ([Ast.t]) AST *) - val parse_module : ?cell:bool -> ?jit:bool -> ctx:Ctx.t -> Stmt.t Ann.ann list -> unit + val parse_module : ?cell:bool -> ctx:Ctx.t -> Stmt.t Ann.ann list -> unit (** Parses a [For] statement AST. Public in order to allow access to it from [ExprIntf]. *) val parse_for diff --git a/parser/seqaml/codegen_stmt.ml b/parser/seqaml/codegen_stmt.ml index 7cb31c577..1118d4ac4 100644 --- a/parser/seqaml/codegen_stmt.ml +++ b/parser/seqaml/codegen_stmt.ml @@ -19,15 +19,15 @@ module Codegen (E : Codegen_intf.Expr) : Codegen_intf.Stmt = struct *************************************************************** *) (** [parse ~ctx expr] dispatches a statement AST node [expr] to the proper code generation function. *) - let rec parse ?(toplevel = false) ?(jit = false) ~(ctx : Ctx.t) (pos, node) = + let rec parse ?(toplevel = false) ~(ctx : Ctx.t) (pos, node) = let stmt = match node with | Break p -> parse_break ctx pos p | Continue p -> parse_continue ctx pos p | Expr p -> parse_expr ctx pos p - | Assign p -> parse_assign ctx pos p ~toplevel ~jit + | Assign p -> parse_assign ctx pos p ~toplevel | Del p -> parse_del ctx pos p - | Print p -> parse_print ctx pos p ~jit + | Print p -> parse_print ctx pos p | Return p -> parse_return ctx pos p | Yield p -> parse_yield ctx pos p | Assert p -> parse_assert ctx pos p @@ -39,15 +39,16 @@ module Codegen (E : Codegen_intf.Expr) : Codegen_intf.Stmt = struct | Extend p -> parse_extend ctx pos p ~toplevel | Import p -> parse_import ctx pos p ~toplevel | ImportPaste p -> parse_impaste ctx pos p - | ImportExtern p -> parse_extern ctx pos p ~toplevel ~jit + | ImportExtern p -> parse_extern ctx pos p ~toplevel | Pass p -> parse_pass ctx pos p | Try p -> parse_try ctx pos p | Throw p -> parse_throw ctx pos p | Global p -> parse_global ctx pos p - | Generic (Function p) -> parse_function ctx pos p ~toplevel ~jit + | Generic (Function p) -> parse_function ctx pos p ~toplevel | Generic (Class p) -> parse_class ctx pos p ~toplevel - | Generic (Declare p) -> parse_declare ctx pos p ~toplevel ~jit + | Generic (Declare p) -> parse_declare ctx pos p ~toplevel | Generic (Type p) -> parse_class ctx pos p ~toplevel ~is_type:true + | Generic (Docstring _) -> Llvm.Stmt.pass () | Special p -> parse_special ctx pos p | Prefetch p -> parse_prefetch ctx pos p in @@ -55,10 +56,10 @@ module Codegen (E : Codegen_intf.Expr) : Codegen_intf.Stmt = struct (** [parse_module ~ctx stmts] parses a module. A module is just a simple list [stmts] of statements. *) - and parse_module ?(cell=false) ?(jit = false) ~(ctx : Ctx.t) stmts = + and parse_module ?(cell=false) ~(ctx : Ctx.t) stmts = (* Util.dbg "%s\n" @@ Util.ppl ~sep:"\n" stmts ~f:Ast.Stmt.to_string; *) let stmts = - if jit && cell + if ctx.is_jit && cell then List.rev @@ @@ -67,7 +68,7 @@ module Codegen (E : Codegen_intf.Expr) : Codegen_intf.Stmt = struct | l -> l else stmts in - ignore @@ List.map stmts ~f:(parse ~ctx ~toplevel:true ~jit) + ignore @@ List.map stmts ~f:(parse ~ctx ~toplevel:true) (* *************************************************************** Node code generators @@ -88,10 +89,10 @@ module Codegen (E : Codegen_intf.Expr) : Codegen_intf.Stmt = struct let expr = E.parse ~ctx expr in Llvm.Stmt.expr expr - and parse_assign ctx pos ~toplevel ~jit (lhs, rhs, shadow, typ) = + and parse_assign ctx pos ~toplevel (lhs, rhs, shadow, typ) = match lhs with | pos, Id var -> - (match jit && toplevel, Hashtbl.find ctx.map var, shadow with + (match ctx.is_jit && toplevel, Hashtbl.find ctx.map var, shadow with | _, None, Update -> serr ~pos "identifier '%s' not found" var | false, Some ((Ctx_namespace.Var _, { base; global; _ }) :: _), Update when ctx.base <> base && not global -> @@ -126,6 +127,8 @@ module Codegen (E : Codegen_intf.Expr) : Codegen_intf.Stmt = struct (* eprintf ">> jit_var %s := %s\n%!" (Ast.Expr.to_string lhs) (Ast.Expr.to_string rhs); *) (* finalize ~ctx stmt pos; *) (* Llvm.Stmt.set_pos v (fst lhs); *) + Ctx.add_inspect_var ~ctx v (fst lhs) var ""; + Ctx.add_inspect ~ctx (fst lhs) var (Ctx.IVar v); Ctx.add ~ctx ~toplevel ~global:true var (Ctx_namespace.Var v); (* Llvm.Stmt.expr var_expr *) (* Llvm.Stmt.assign v rh_expr *) @@ -138,6 +141,8 @@ module Codegen (E : Codegen_intf.Expr) : Codegen_intf.Stmt = struct let v = Llvm.Var.stmt var_stmt in if toplevel then Llvm.Var.set_global v; (* Llvm.Stmt.set_pos v (fst lhs); *) + Ctx.add_inspect_var ~ctx v (fst lhs) var ""; + Ctx.add_inspect ~ctx (fst lhs) var (Ctx.IVar v); Ctx.add ~ctx ~toplevel ~global:toplevel var (Ctx_namespace.Var v); var_stmt) | pos, Dot (lh_lhs, lh_rhs) -> @@ -159,8 +164,8 @@ module Codegen (E : Codegen_intf.Expr) : Codegen_intf.Stmt = struct Llvm.Stmt.assign_index var_expr index_expr rh_expr | _ -> serr ~pos "invalid assignment statement" - and parse_declare ctx pos ~toplevel ~jit { name; typ; _ } = - if jit then serr ~pos "declarations not yet supported in JIT mode"; + and parse_declare ctx pos ~toplevel { name; typ; _ } = + if ctx.is_jit then serr ~pos "declarations not yet supported in JIT mode"; match Hashtbl.find ctx.map name with | Some ((Ctx_namespace.Var _, { base; global; toplevel; _ }) :: _) when ctx.base = base -> @@ -171,6 +176,8 @@ module Codegen (E : Codegen_intf.Expr) : Codegen_intf.Stmt = struct let v = Llvm.Var.stmt var_stmt in if toplevel then Llvm.Var.set_global v; (* Llvm.Stmt.set_pos v pos; *) + Ctx.add_inspect_var ~ctx v pos name ""; + Ctx.add_inspect ~ctx pos name (Ctx.IVar v); Ctx.add ~ctx ~toplevel ~global:toplevel name (Ctx_namespace.Var v); var_stmt @@ -188,8 +195,8 @@ module Codegen (E : Codegen_intf.Expr) : Codegen_intf.Stmt = struct | _ -> serr ~pos "cannot find %s" var) | _ -> serr ~pos "cannot delete a non-index expression" - and parse_print ctx pos ~jit (exprs, ed) = - let ll = if jit then Llvm.Stmt.print_jit else Llvm.Stmt.print in + and parse_print ctx pos (exprs, ed) = + let ll = if ctx.is_jit then Llvm.Stmt.print_jit else Llvm.Stmt.print in List.iteri exprs ~f:(fun i expr -> let expr = E.parse ~ctx expr in ignore @@ finalize ~ctx (ll expr) pos; @@ -222,6 +229,8 @@ module Codegen (E : Codegen_intf.Expr) : Codegen_intf.Stmt = struct and parse_type_alias ctx pos ~toplevel (name, expr) = let typ = E.parse_type ~ctx expr in (* Llvm.Stmt.set_pos typ pos; *) + Ctx.add_inspect_var ~ctx typ pos name ""; + Ctx.add_inspect ~ctx pos name (Ctx.IType typ); Ctx.add ~ctx ~toplevel ~global:toplevel name (Ctx_namespace.Type typ); Llvm.Stmt.pass () @@ -255,7 +264,7 @@ module Codegen (E : Codegen_intf.Expr) : Codegen_intf.Stmt = struct match next with | Some next -> next ctx for_ctx for_stmt | None -> - ignore @@ List.map stmts ~f:(parse ~ctx:for_ctx ~toplevel:false ~jit:false) + ignore @@ List.map stmts ~f:(parse ~ctx:for_ctx ~toplevel:false) in Ctx.clear_block for_ctx; for_stmt @@ -298,13 +307,13 @@ module Codegen (E : Codegen_intf.Expr) : Codegen_intf.Stmt = struct | None -> ())); match_stmt - and parse_extern ctx pos ~toplevel ?(jit = false) ext = + and parse_extern ctx pos ~toplevel ext = List.iter ext ~f:(fun s -> - let s' = parse_extern_single ctx pos ~toplevel ~jit s in + let s' = parse_extern_single ctx pos ~toplevel s in ignore @@ finalize ~ctx s' pos); Llvm.Stmt.pass () - and parse_extern_single ctx pos ~toplevel ~jit ext = + and parse_extern_single ctx pos ~toplevel ext = let ctx_name = Option.value ext.e_as ~default:ext.e_name.name in let open Ast in match ext.lang, ext.e_from with @@ -327,7 +336,7 @@ module Codegen (E : Codegen_intf.Expr) : Codegen_intf.Stmt = struct e_call (e_dot (Option.value_exn ext.e_name.typ) "__from_py__") [rhs] in (* @pyhandle def y ( x : [tuple] ) -> typ: return rhs *) - parse_function ctx pos ~toplevel ~jit + parse_function ctx pos ~toplevel { fn_name = { ext.e_name with name = ctx_name } ; fn_generics = [] ; fn_args = [ pos, { name = "x"; typ = None; default = None } ] @@ -370,8 +379,10 @@ module Codegen (E : Codegen_intf.Expr) : Codegen_intf.Stmt = struct Llvm.Func.set_type fn typ; let names = List.map ext.e_args ~f:(fun (_, x) -> x.name) in (* Llvm.Stmt.set_pos fn pos; *) + Ctx.add_inspect_var ~ctx fn pos ctx_name ""; + Ctx.add_inspect ~ctx pos ctx_name (Ctx.IFunc fn); Ctx.add ~ctx ~toplevel ~global:toplevel ctx_name (Ctx_namespace.Func (fn, names)); - if toplevel && jit then ( + if toplevel && ctx.is_jit then ( ignore @@ finalize ~ctx ~add:false (Llvm.Stmt.func fn) pos; Llvm.Stmt.pass ()) else Llvm.Stmt.func fn @@ -485,7 +496,6 @@ module Codegen (E : Codegen_intf.Expr) : Codegen_intf.Stmt = struct pos ?cls ?(toplevel = false) - ?(jit = false) { fn_name = { name; typ; _ }; fn_generics; fn_args; fn_stmts; fn_attrs } = let fn = @@ -498,6 +508,12 @@ module Codegen (E : Codegen_intf.Expr) : Codegen_intf.Stmt = struct let fn = Llvm.Func.func name in let names = List.map fn_args ~f:(fun (_, x) -> x.name) in (* Llvm.Stmt.set_pos fn pos; *) + let doc = match List.hd fn_stmts with + | Some (_, Expr (_, String doc)) -> doc + | _ -> "" + in + Ctx.add_inspect_var ~ctx fn pos name doc; + Ctx.add_inspect ~ctx pos name (Ctx.IFunc fn); Ctx.add ~ctx ~toplevel ~global:toplevel name (Ctx_namespace.Func (fn, names)); ( if ctx.base <> ctx.mdl then Llvm.Func.set_enclosing fn ctx.base ); fn @@ -537,15 +553,11 @@ module Codegen (E : Codegen_intf.Expr) : Codegen_intf.Stmt = struct typ ~f:(fun typ -> Llvm.Func.set_type fn (E.parse_type ~ctx:new_ctx typ)) ~default:(); - ( match List.hd fn_stmts with - | Some (_, Expr (_, String doc)) -> - Ctx.add_origin pos doc - | _ -> () ); add_block new_ctx fn_stmts ~preprocess:(fun ctx -> List.iter names ~f:(fun name -> let var = Ctx_namespace.Var (Llvm.Func.get_arg fn name) in Ctx.add ~ctx name var)); - if toplevel && jit then ( + if toplevel && ctx.is_jit then ( ignore @@ finalize ~ctx ~add:false (Llvm.Stmt.func fn) pos; Llvm.Stmt.pass ()) else Llvm.Stmt.func fn @@ -558,6 +570,12 @@ module Codegen (E : Codegen_intf.Expr) : Codegen_intf.Stmt = struct else Llvm.Type.cls cls.class_name in (* Llvm.Stmt.set_pos typ pos; *) + let doc = match List.hd cls.members with + | Some (_, Docstring s) -> s + | _ -> "" + in + Ctx.add_inspect_var ~ctx typ pos cls.class_name doc; + Ctx.add_inspect ~ctx pos cls.class_name (Ctx.IType typ); Ctx.add ~ctx ~toplevel ~global:toplevel cls.class_name (Ctx_namespace.Type typ); typ in @@ -594,6 +612,7 @@ module Codegen (E : Codegen_intf.Expr) : Codegen_intf.Stmt = struct ignore @@ List.map cls.members ~f:(function | pos, Function f -> parse_function new_ctx pos f ~cls:typ + | pos, Docstring _ -> Ctypes.null | _ -> serr ~pos "only function can be defined within a class definition"); if not is_type then Llvm.Type.set_cls_done typ; Llvm.Stmt.pass () diff --git a/parser/seqaml/ctx.ml b/parser/seqaml/ctx.ml index 8982a106f..32ef48a44 100644 --- a/parser/seqaml/ctx.ml +++ b/parser/seqaml/ctx.ml @@ -7,6 +7,28 @@ open Core +(* + lookup: + file -> line -> [ pos ; pointer ] + origin: + file -> line -> [ doc; type_name ] +*) +type tinstecttype = + | IFunc of Llvm.Types.func_t + | IType of Llvm.Types.typ_t + | IVar of Llvm.Types.var_t + | IExpr of Llvm.Types.expr_t + | IImport of string +type tinspect = + { pos: Ast_ann.t + ; el: tinstecttype + ; name: string } +type tinspectvar = + { doc: string + ; pos: Ast_ann.t + ; name: string + } + (** Main parsing context. *) type t = { filename : string (** The file that is being parsed. *) @@ -30,39 +52,26 @@ type t = to populate [Ast.Ann] annotations. *) ; stdlib : Ctx_namespace.t (** A context that holds the internal Seq objects and libraries. *) - } -(* - lookup: - file -> line -> [ pos ; pointer ] - origin: - file -> line -> [ doc; type_name ] -*) - -type tinspect = - { pos: Ast_ann.t - ; el: Llvm.Types.typ_t - ; name: string } -let inspect_lookup : - (string, (int, tinspect Stack.t) Hashtbl.t) Hashtbl.t - = String.Table.create () -type torigin = - { doc: string + ; is_jit: bool + ; inspect_lookup : + (string, (int, tinspect Stack.t) Hashtbl.t) Hashtbl.t + ; inspect_var_lookup: + (unit Ctypes_static.ptr, tinspectvar) Hashtbl.t } -let origin_lookup : - (string, (int, torigin Stack.t) Hashtbl.t) Hashtbl.t - = String.Table.create () - -let add_inspect (pos: Ast_ann.t) name el = - let s = Hashtbl.find_or_add inspect_lookup pos.file ~default:Int.Table.create in - let s = Hashtbl.find_or_add s pos.line ~default:Stack.create in - Stack.push s { pos; el; name } - -let add_origin (pos: Ast_ann.t) doc = - let s = Hashtbl.find_or_add origin_lookup pos.file ~default:Int.Table.create in - let s = Hashtbl.find_or_add s pos.line ~default:Stack.create in - Stack.push s { doc } +let add_inspect ~ctx (pos: Ast_ann.t) name el = + if ctx.is_jit then + let s = Hashtbl.find_or_add ctx.inspect_lookup pos.file ~default:Int.Table.create in + let s = Hashtbl.find_or_add s pos.line ~default:Stack.create in + Stack.push s { pos; el; name } + +let add_inspect_var ~ctx key (pos: Ast_ann.t) name doc = + if ctx.is_jit then + ( (* eprintf "< adding %s: %nx\n%!" name (Ctypes.raw_address_of_ptr key); *) + ignore @@ Hashtbl.add ctx.inspect_var_lookup ~key ~data:{ name; doc; pos }; + ) + (** [add_block context] adds a new block to the context stack. *) let add_block ctx = Stack.push ctx.stack (String.Hash_set.create ()) @@ -130,6 +139,9 @@ let init_module ?(argv = true) ?(jit = false) ~filename ~mdl ~base ~block parser ; imported = String.Table.create () ; trycatch = Ctypes.null ; stdlib = String.Table.create () + ; is_jit = jit + ; inspect_lookup = String.Table.create () + ; inspect_var_lookup = Hashtbl.Poly.create () } in add_block ctx; diff --git a/parser/seqaml/grammar.mly b/parser/seqaml/grammar.mly index 373dadaca..6eab81eb9 100644 --- a/parser/seqaml/grammar.mly +++ b/parser/seqaml/grammar.mly @@ -1021,7 +1021,8 @@ extend: // Class suite members class_member: // Empty statements - | PASS NL | STRING NL { None } + | PASS NL { None } + | STRING NL { Some (fst $1, Docstring (snd $1)) } // TODO later: | class_statement // Functions | func_statement diff --git a/parser/seqaml/llvm.ml b/parser/seqaml/llvm.ml index 90ce66aaf..7929122ea 100644 --- a/parser/seqaml/llvm.ml +++ b/parser/seqaml/llvm.ml @@ -249,6 +249,8 @@ module Expr = struct let get_name = foreign "get_expr_name" (t @-> returning string) + let get_type = foreign "get_type" (t @-> returning Types.typ) + (* Utilities *) let is_type expr = @@ -389,6 +391,8 @@ module Var = struct let set_global = foreign "set_global" (t @-> returning void) let set_atomic = foreign "var_expr_set_atomic" (Types.expr @-> returning Ctypes.void) + + let get_type = foreign "get_var_type" (t @-> returning Types.typ) end (** [Func] wraps Seq BaseFunc methods and helpers. *) @@ -437,6 +441,8 @@ module Func = struct let set_type = foreign "set_func_out" (t @-> Types.typ @-> returning void) let set_extern = foreign "set_func_extern" (t @-> returning void) let set_enclosing = foreign "set_func_enclosing" (t @-> t @-> returning void) + + let get_type = foreign "get_func_type" (t @-> returning Types.typ) end (** [Generics] wraps Seq generic methods and helpers. *) diff --git a/stdlib/core/collections/list.seq b/stdlib/core/collections/list.seq index 2821430c3..fb2ea5b14 100644 --- a/stdlib/core/collections/list.seq +++ b/stdlib/core/collections/list.seq @@ -1,4 +1,6 @@ class list[T]: + """ List implementation """ + arr: array[T] len: int From 8e5c475fda597b065b3af1ba042e2281f2086649 Mon Sep 17 00:00:00 2001 From: jodiew Date: Tue, 3 Dec 2019 12:18:02 -0800 Subject: [PATCH 12/30] Switch to setuptools --- jupyter/pyproject.toml | 13 ------------- jupyter/setup.py | 20 ++++++++++++++++++++ 2 files changed, 20 insertions(+), 13 deletions(-) delete mode 100644 jupyter/pyproject.toml create mode 100644 jupyter/setup.py diff --git a/jupyter/pyproject.toml b/jupyter/pyproject.toml deleted file mode 100644 index b6a23b242..000000000 --- a/jupyter/pyproject.toml +++ /dev/null @@ -1,13 +0,0 @@ -[build-system] -requires = ["flit", "ipykernel"] -build-backend = "flit.buildapi" - -[tool.flit.metadata] -module = "seq_kernel" -author = "" -author-email = "" -home-page = "https://github.com/seq-lang/seq" -description-file = "README.md" - -[tool.flit.entrypoints."pygments.lexers"] -seq = "seq_kernel.seqlex:SeqLexer" \ No newline at end of file diff --git a/jupyter/setup.py b/jupyter/setup.py new file mode 100644 index 000000000..12c421ab2 --- /dev/null +++ b/jupyter/setup.py @@ -0,0 +1,20 @@ +import setuptools +from seq_kernel.kernel import __version__ + +with open('README.md', 'r') as fh: + long_description = fh.read() + +setuptools.setup( + name='seq_kernel', + version=__version__, + description='Seq Kernel', + long_description=long_description, + long_description_content_type='text/markdown', + url='https://github.com/seq-lang/seq', + packages=setuptools.find_packages(), + classifiers=[ + 'Framework :: Jupyter', + 'Intended Audience :: Science/Research', + 'Programming Language :: Seq', + ] +) \ No newline at end of file From db9e5ed87c7a969be83cc67cea94ee6b9777f28f Mon Sep 17 00:00:00 2001 From: jodiew Date: Tue, 3 Dec 2019 20:48:01 -0800 Subject: [PATCH 13/30] Add Jupyter completion and inspection --- jupyter/seq_kernel/install.py | 2 + jupyter/seq_kernel/kernel.py | 182 ++++++++++++++++++++++++++++------ jupyter/seq_kernel/wrapper.py | 7 +- jupyter/wrapper.py | 2 +- parser/jit.ml | 4 +- 5 files changed, 161 insertions(+), 36 deletions(-) diff --git a/jupyter/seq_kernel/install.py b/jupyter/seq_kernel/install.py index c6dab48e3..40aefaeaa 100644 --- a/jupyter/seq_kernel/install.py +++ b/jupyter/seq_kernel/install.py @@ -1,3 +1,5 @@ +#!/.linuxbrew/bin python3 + import json import os import sys diff --git a/jupyter/seq_kernel/kernel.py b/jupyter/seq_kernel/kernel.py index d36716bf9..4d4126026 100644 --- a/jupyter/seq_kernel/kernel.py +++ b/jupyter/seq_kernel/kernel.py @@ -6,7 +6,7 @@ from .redirector import stdout_stderr_redirector from .wrapper import SeqWrapper -__version__ = '0.0.0' +__version__ = '0.0.1' version_pat = re.compile(r'version (\d+(\.\d+)+)') @@ -29,19 +29,34 @@ def banner(self): language_info = { 'name': 'seq', - 'mimetype': 'application/seq', + 'mimetype': 'text/seq', 'file_extension': '.seq', } def __init__(self, **kwargs): Kernel.__init__(self, **kwargs) self.seqwrapper = SeqWrapper() - self.cells = set() + self.cells = [] def do_execute(self, code, silent, store_history=True, user_expressions=None, allow_stdin=False): + """ + Execute user code. + Parameters: + code (str) -- The code to be executed. + silent (bool) -- Whether to display output. + store_history (bool) -- Whether to record this code in history and increase the execution count. + If silent is True, this is implicitly False. + user_expressions (dict) -- Mapping of names to expressions to evaluate after the code has run. + You can ignore this if you need to. + allow_stdin (bool) -- Whether the frontend can provide input on request (e.g. for Python's raw_input()). + """ if not code.strip(): - return {'status': 'ok', 'execution_count': self.execution_count, - 'payload': [], 'user_expressions': {}} + return { + 'status': 'ok', + 'execution_count': self.execution_count, + 'payload': [], + 'user_expressions': {} + } fout = BytesIO() ferr = BytesIO() @@ -49,46 +64,151 @@ def do_execute(self, code, silent, store_history=True, user_expressions=None, al with stdout_stderr_redirector(fout, ferr): code = code.rstrip() self.seqwrapper.exec(code) - self.cells.add(code) + self.cells.append(code) fout_string = fout.getvalue().decode('utf-8').strip() ferr_string = ferr.getvalue().decode('utf-8').strip() if ferr_string: if not silent: - self.send_response(self.iopub_socket, 'stream', {'name': 'stderr', 'text': ferr_string}) - return {'status': 'error', 'execution_count': self.execution_count} + self.send_response( + self.iopub_socket, + 'stream', + {'name': 'stderr', 'text': ferr_string} + ) + return { + 'status': 'error', + 'ename': 'ExecutonError', + 'evalue': fout_string, + 'traceback': [], + 'execution_count': self.execution_count + } + + if not silent: + if fout_string.startswith('\0x1'): + mime_type, data = fout_string.strip('\0x1').split('\b', 1) + self.send_response( + self.iopub_socket, + 'display_data', + {'data': {mime_type: data}, 'metadata': {}} + ) + else: + self.send_response( + self.iopub_socket, + 'stream', + {'name': 'stdout', 'text': fout_string} + ) - else: - if not silent: - self.send_response(self.iopub_socket, 'stream', {'name': 'stdout', 'text': fout_string}) - - return {'status': 'ok', 'execution_count': self.execution_count, - 'payload': [], 'user_expressions': {}} + return { + 'status': 'ok', + 'execution_count': self.execution_count, + 'payload': [], + 'user_expressions': {} + } + + def do_complete(self, code, cursor_pos): + """ + Code completion. + Parameters: + code (str) -- The code already present. + cursor_pos (int) -- The position in the code where completion is requested. + """ + + matches = [] + ferr = BytesIO() + with stdout_stderr_redirector(BytesIO(), ferr): + matches = self.seqwrapper.complete(code) - def _get_object(self, code, cursor_pos, detail_level): - # TODO: Get the correct section of code to be inspected - print(code, cursor_pos) - if code not in self.cells: - return None + ferr_string = ferr.getvalue().decode('utf-8').strip() - cell = self.cells[code] - lines = code.split('\n') - line = code[:cursor_pos].count('\n') - col = cursor_pos - code[:cursor_pos].rsearch('\n') + if ferr_string: + return { + 'status': 'error', + 'ename': 'CompletionError', + 'evalue': ferr_string, + 'traceback': [] + } + + return { + 'status': 'ok', + 'matches': [ match for match in matches], + 'cursor_start': 0, + 'cursor_end': cursor_pos, + 'metadata': {} + } - fout = BytesIO() + def do_inspect(self, code, cursor_pos, detail_level=0): + """ + Object introspection. + Parameters: + code (str) -- The code. + cursor_pos(int) -- The position in the code where introspection is requested. + detail_level (int) -- 0 or 1 for more or less detail. + In IPython, 1 gets the source code. + """ + + inspect_text = '' + doc_text = '' ferr = BytesIO() - with stdout_stderr_redirector(fout, ferr): - self.seqwrapper.inspect(cell, line, col) + cell = 0 + line = 0 + col = 0 + + if code in self.cells: + cell = self.cells.index(code) + 1 + line = code[:cursor_pos].count('\n') + 1 + col = cursor_pos - code[:cursor_pos].rfind('\n') - 1 + if col < 0: + col = 0 + + + with stdout_stderr_redirector(BytesIO(), ferr): + if cell > 0: + inspect_text = self.seqwrapper.inspect(cell, line, col) + if cell <= 0 or inspect_text.startswith('Not found:'): + doc_text = self.seqwrapper.document(code) + inspect_text = '' + ferr_string = ferr.getvalue().decode('utf-8').strip() - return {'text/plain': f'Hello, World! {cell} {line} {col} -> {ferr_string}'} - def do_inspect(self, code, cursor_pos, detail_level=0): - data = self._get_object(code, cursor_pos, detail_level) + if ferr_string: + return { + 'status': 'error', + 'ename': 'InspectionError', + 'evalue': ferr_string, + 'traceback': [] + } + + if inspect_text: + # TODO: This piece of code will break on any format changes from the jit side. + obj_name, obj_type, obj_loc, obj_doc = inspect_text[7:].split('\b') + obj_cell, obj_line, obj_col = obj_loc.split(':') + obj_cell = obj_cell.split('_')[1][:-1] + inspect_obj = [ + f'Name: {obj_name}', + f'Type: {obj_type}', + f'Defined at: In [{obj_cell}] line: {obj_line} col: {obj_col}', + f'Documentation: {obj_doc}' + ] + + return { + 'status': 'ok', + 'found': True, + 'data': {'text/plain': "\n".join(inspect_obj)}, + 'metadata': {} + } + + if doc_text: + return { + 'status': 'ok', + 'found': True, + 'data': {'text/plain': f'Documentation: {doc_text}'}, + 'metadata': {} + } + return { 'status': 'ok', - 'found': True, - 'data': data, + 'found': False, + 'data': {}, 'metadata': {} } diff --git a/jupyter/seq_kernel/wrapper.py b/jupyter/seq_kernel/wrapper.py index fcf3f3dda..bc9ed2394 100644 --- a/jupyter/seq_kernel/wrapper.py +++ b/jupyter/seq_kernel/wrapper.py @@ -1,3 +1,5 @@ +# Exec + class SeqWrapper: def __init__(self): import ctypes @@ -32,11 +34,12 @@ def exec(self, code): self._exec_fn(self.handle, code.encode('utf-8')) def inspect(self, cell, line, column): - return self._inspect_fn(self.handle, str(cell).encode('utf-8'), line, column).decode('ascii') + file = f'' + return self._inspect_fn(self.handle, file.encode('utf-8'), line, column).decode('ascii') def document(self, idn): return self._document_fn(self.handle, idn.encode('utf-8')).decode('ascii') def complete(self, prefix): l = self._complete_fn(self.handle, prefix.encode('utf-8')).decode('ascii') - return l.split('\b') \ No newline at end of file + return l.split('\b') diff --git a/jupyter/wrapper.py b/jupyter/wrapper.py index 47c04263c..27cb7c86e 100644 --- a/jupyter/wrapper.py +++ b/jupyter/wrapper.py @@ -6,7 +6,7 @@ def __init__(self): import ctypes.util lib = ctypes.util.find_library("seqjit") - self._lib = ctypes.CDLL(lib) + self._lib = ctypes.CDLL(lib, ctypes.RTLD_GLOBAL) self._init_fn = self._lib.seq_jit_init self._init_fn.restype = ctypes.c_void_p diff --git a/parser/jit.ml b/parser/jit.ml index c700cf9a5..b0705573a 100644 --- a/parser/jit.ml +++ b/parser/jit.ml @@ -88,11 +88,11 @@ let locate ~(ctx : Ctx.t) ?(full = true) f l c = | t, Some orig when t <> Ctypes.null -> let typ = Llvm.Type.get_name (gt el) in Some ( - sprintf "%s |- %s => defined at %s%s" + sprintf "%s\b%s\b%s\b%s" orig.name typ (Ast.Ann.to_string orig.pos) - (if full && orig.doc <> "" then sprintf "\nDocs: %s" orig.doc else "")) + (if full && orig.doc <> "" then sprintf "%s" orig.doc else "")) | _ -> None ))) From ed0eea7f69e8a85021360331ad80895ff9dad2ce Mon Sep 17 00:00:00 2001 From: jodiew Date: Tue, 3 Dec 2019 21:02:17 -0800 Subject: [PATCH 14/30] Add pygments lexer to setup --- jupyter/README.md | 2 +- jupyter/seq_kernel/kernel.py | 12 ++++++------ jupyter/setup.py | 2 ++ 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/jupyter/README.md b/jupyter/README.md index 7598ac4d7..f0aad8caa 100644 --- a/jupyter/README.md +++ b/jupyter/README.md @@ -5,7 +5,7 @@ A seq kernel for Jupyter. ## To Install ```bash -python3 -m pip install /seq/jupyter +python3 -m pip install /seq/jupyter/ python3 -m seq_kernel.install ``` diff --git a/jupyter/seq_kernel/kernel.py b/jupyter/seq_kernel/kernel.py index 4d4126026..c30ed122d 100644 --- a/jupyter/seq_kernel/kernel.py +++ b/jupyter/seq_kernel/kernel.py @@ -6,7 +6,7 @@ from .redirector import stdout_stderr_redirector from .wrapper import SeqWrapper -__version__ = '0.0.1' +__version__ = '0.0.0' version_pat = re.compile(r'version (\d+(\.\d+)+)') @@ -105,7 +105,7 @@ def do_execute(self, code, silent, store_history=True, user_expressions=None, al 'payload': [], 'user_expressions': {} } - + def do_complete(self, code, cursor_pos): """ Code completion. @@ -128,7 +128,7 @@ def do_complete(self, code, cursor_pos): 'evalue': ferr_string, 'traceback': [] } - + return { 'status': 'ok', 'matches': [ match for match in matches], @@ -160,7 +160,7 @@ def do_inspect(self, code, cursor_pos, detail_level=0): col = cursor_pos - code[:cursor_pos].rfind('\n') - 1 if col < 0: col = 0 - + with stdout_stderr_redirector(BytesIO(), ferr): if cell > 0: @@ -178,7 +178,7 @@ def do_inspect(self, code, cursor_pos, detail_level=0): 'evalue': ferr_string, 'traceback': [] } - + if inspect_text: # TODO: This piece of code will break on any format changes from the jit side. obj_name, obj_type, obj_loc, obj_doc = inspect_text[7:].split('\b') @@ -197,7 +197,7 @@ def do_inspect(self, code, cursor_pos, detail_level=0): 'data': {'text/plain': "\n".join(inspect_obj)}, 'metadata': {} } - + if doc_text: return { 'status': 'ok', diff --git a/jupyter/setup.py b/jupyter/setup.py index 12c421ab2..1f1e068f1 100644 --- a/jupyter/setup.py +++ b/jupyter/setup.py @@ -12,9 +12,11 @@ long_description_content_type='text/markdown', url='https://github.com/seq-lang/seq', packages=setuptools.find_packages(), + entry_points={'pygments.lexers': 'seq = seq_kernel.seqlex:SeqLexer'}, classifiers=[ 'Framework :: Jupyter', 'Intended Audience :: Science/Research', + 'Programming Language :: Python' 'Programming Language :: Seq', ] ) \ No newline at end of file From 81aad2cb664e05810ec2c6b58eb4f0f6c31263ce Mon Sep 17 00:00:00 2001 From: jodiew Date: Tue, 3 Dec 2019 21:05:03 -0800 Subject: [PATCH 15/30] Small changes --- jupyter/README.md | 2 +- jupyter/seq_kernel/install.py | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/jupyter/README.md b/jupyter/README.md index f0aad8caa..7598ac4d7 100644 --- a/jupyter/README.md +++ b/jupyter/README.md @@ -5,7 +5,7 @@ A seq kernel for Jupyter. ## To Install ```bash -python3 -m pip install /seq/jupyter/ +python3 -m pip install /seq/jupyter python3 -m seq_kernel.install ``` diff --git a/jupyter/seq_kernel/install.py b/jupyter/seq_kernel/install.py index 40aefaeaa..c6dab48e3 100644 --- a/jupyter/seq_kernel/install.py +++ b/jupyter/seq_kernel/install.py @@ -1,5 +1,3 @@ -#!/.linuxbrew/bin python3 - import json import os import sys From 962b3b3ddb91442863f168ffe9d76723a1735bf3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ibrahim=20Numanagic=CC=81?= Date: Wed, 4 Dec 2019 21:55:57 -0800 Subject: [PATCH 16/30] Add IGV support --- jupyter/seq_kernel/kernel.js | 8 ++++- jupyter/seq_kernel/kernel.py | 1 + parser/jit.ml | 67 ++++++++++++++++++----------------- parser/seqaml/codegen_stmt.ml | 9 +++-- stdlib/core/__init__.seq | 2 +- stdlib/core/python.seq | 3 ++ 6 files changed, 53 insertions(+), 37 deletions(-) diff --git a/jupyter/seq_kernel/kernel.js b/jupyter/seq_kernel/kernel.js index 166183f1d..03774650d 100644 --- a/jupyter/seq_kernel/kernel.js +++ b/jupyter/seq_kernel/kernel.js @@ -1,5 +1,5 @@ define( - ["codemirror/lib/codemirror", "base/js/namespace"], + ["codemirror/lib/codemirror", "base/js/namespace", "https://cdn.jsdelivr.net/npm/igv@2.3.5/dist/igv.min.js"], function(CodeMirror){ "use strict"; var onload = function() { @@ -9,6 +9,12 @@ define( return{onload: onload}; }); +require.config({ + paths: { + igv: 'https://cdn.jsdelivr.net/npm/igv@2.3.5/dist/igv.min' + } +}); + var enableSeqMode = function (CodeMirror) { function wordRegexp(words) { return new RegExp("^((" + words.join(")|(") + "))\\b"); diff --git a/jupyter/seq_kernel/kernel.py b/jupyter/seq_kernel/kernel.py index c30ed122d..a6bc43c29 100644 --- a/jupyter/seq_kernel/kernel.py +++ b/jupyter/seq_kernel/kernel.py @@ -36,6 +36,7 @@ def banner(self): def __init__(self, **kwargs): Kernel.__init__(self, **kwargs) self.seqwrapper = SeqWrapper() + self.seqwrapper.exec('from jupyter import *') self.cells = [] def do_execute(self, code, silent, store_history=True, user_expressions=None, allow_stdin=False): diff --git a/parser/jit.ml b/parser/jit.ml index b0705573a..ce7946b59 100644 --- a/parser/jit.ml +++ b/parser/jit.ml @@ -14,32 +14,6 @@ type t = ; ctx : Ctx.t (** Execution context. *) } -(** Initialize a JIT context. *) -let init () : t = - try - let anon_fn = Llvm.Func.func "" in - let ctx = - Ctx.init_module - ~argv:false - ~filename:"" - ~mdl:(Llvm.JIT.init ()) - ~base:anon_fn - ~block:(Llvm.Block.func anon_fn) - ~jit:true - (Runner.exec_string ~debug:false ~cell:false) - in - let jit = { cnt = 1; ctx } in - (* load stdlib *) - Util.dbg "===== launching exec..."; - let _t = Unix.gettimeofday () in - Llvm.JIT.func ctx.mdl anon_fn; - Util.dbg "... took %f for the whole OCaml part" (Unix.gettimeofday() -. _t); - jit - with - | Err.CompilerError (typ, pos_lst) -> - eprintf "%s\n%!" @@ Err.to_string ~pos_lst typ; - exit 1 - (** Execute [code] within a JIT context [jit]. *) let exec (jit : t) code = let file = sprintf "" jit.cnt in @@ -66,6 +40,33 @@ let exec (jit : t) code = with | Err.SeqCError (msg, pos) -> raise @@ Err.CompilerError (Compiler msg, [ pos ]) +(** Initialize a JIT context. *) +let init () : t = + try + let anon_fn = Llvm.Func.func "" in + let ctx = + Ctx.init_module + ~argv:false + ~filename:"" + ~mdl:(Llvm.JIT.init ()) + ~base:anon_fn + ~block:(Llvm.Block.func anon_fn) + ~jit:true + (Runner.exec_string ~debug:false ~cell:false) + in + let jit = { cnt = 1; ctx } in + (* load stdlib *) + Util.dbg "===== launching exec..."; + let _t = Unix.gettimeofday () in + Llvm.JIT.func ctx.mdl anon_fn; + Util.dbg "... took %f for the whole OCaml part" (Unix.gettimeofday() -. _t); + exec jit "from jupyter import *"; + jit + with + | Err.CompilerError (typ, pos_lst) -> + eprintf "%s\n%!" @@ Err.to_string ~pos_lst typ; + exit 1 + let locate ~(ctx : Ctx.t) ?(full = true) f l c = let open Option.Monad_infix in (* let f = sprintf "" f in *) @@ -89,14 +90,14 @@ let locate ~(ctx : Ctx.t) ?(full = true) f l c = let typ = Llvm.Type.get_name (gt el) in Some ( sprintf "%s\b%s\b%s\b%s" - orig.name + orig.name typ - (Ast.Ann.to_string orig.pos) + (Ast.Ann.to_string orig.pos) (if full && orig.doc <> "" then sprintf "%s" orig.doc else "")) | _ -> None ))) -let locate_dump ~(ctx : Ctx.t) = +let locate_dump ~(ctx : Ctx.t) = Hashtbl.iteri ctx.inspect_lookup ~f:(fun ~key ~data -> if String.is_prefix key ~prefix:" eprintf " Line %d:\n" key; Stack.iter data ~f:(fun {name;pos;el} -> - eprintf " %s\n" + eprintf " %s\n" (Option.value (locate ~ctx file key pos.col) ~default:"?"))))); eprintf "%!" @@ -167,7 +168,7 @@ let c_document hnd identifier = match document jit.ctx identifier with | Some d -> d | None -> "" - + let c_complete hnd prefix = let jit = Hashtbl.find_exn jits hnd in Hashtbl.filter_mapi jit.ctx.map ~f:(fun ~key ~data -> @@ -201,7 +202,7 @@ let repl () = match String.is_prefix ~prefix:"%%" !code with | true -> let ll = Array.of_list @@ String.split ~on:' ' @@ String.strip !code in - ( match ll.(0) with + ( match ll.(0) with | "%%i" -> let file = ll.(1) in let line = Int.of_string @@ ll.(2) in @@ -212,7 +213,7 @@ let repl () = in eprintf "%s\n%!" str - | _ -> + | _ -> eprintf "Unknown JIT command\n%!" ); code := ""; diff --git a/parser/seqaml/codegen_stmt.ml b/parser/seqaml/codegen_stmt.ml index 1118d4ac4..f7c586ec0 100644 --- a/parser/seqaml/codegen_stmt.ml +++ b/parser/seqaml/codegen_stmt.ml @@ -64,7 +64,12 @@ module Codegen (E : Codegen_intf.Expr) : Codegen_intf.Stmt = struct List.rev @@ match List.rev stmts with - | (pos, Expr e) :: tl -> (pos, Print ([ e ], "\n")) :: tl + (* x -> JupyterView(s).s *) + | (pos, Expr e) :: tl -> + (pos, Print ([ + pos, Dot ((pos, Call ((pos, Id "JupyterView"), + [pos, { name = None; value = e }])) , "s") + ], "\n")) :: tl | l -> l else stmts in @@ -510,7 +515,7 @@ module Codegen (E : Codegen_intf.Expr) : Codegen_intf.Stmt = struct (* Llvm.Stmt.set_pos fn pos; *) let doc = match List.hd fn_stmts with | Some (_, Expr (_, String doc)) -> doc - | _ -> "" + | _ -> "" in Ctx.add_inspect_var ~ctx fn pos name doc; Ctx.add_inspect ~ctx pos name (Ctx.IFunc fn); diff --git a/stdlib/core/__init__.seq b/stdlib/core/__init__.seq index 06ace5ce8..2b5979798 100644 --- a/stdlib/core/__init__.seq +++ b/stdlib/core/__init__.seq @@ -24,7 +24,7 @@ from core.file import File, gzFile, open, gzopen from pickle import pickle, unpickle from core.dlopen import dlsym as _dlsym -from core.python import init as _py_init, imp as _py_import, exec as _py_exec +from core.python import init as _py_init, imp as _py_import, exec as _py_exec, pyobj try: _py_init() except CError: diff --git a/stdlib/core/python.seq b/stdlib/core/python.seq index 56593ae07..e593df0b6 100644 --- a/stdlib/core/python.seq +++ b/stdlib/core/python.seq @@ -46,6 +46,9 @@ type pyobj(p: cobj): def __to_py__(self: pyobj): return self + def __from_py__(self: pyobj): + return self + def to_str(self: pyobj, errors: str, empty: str = "") -> str: obj = PyUnicode_AsEncodedString(self.p, "utf-8".c_str(), errors.c_str()) if obj == cobj(): From daa3057328e0aa766eee7b0a2a96a1a9abc219a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ibrahim=20Numanagic=CC=81?= Date: Wed, 4 Dec 2019 21:56:10 -0800 Subject: [PATCH 17/30] Add IGV support --- jupyter/seq_kernel/kernel.py | 1 - 1 file changed, 1 deletion(-) diff --git a/jupyter/seq_kernel/kernel.py b/jupyter/seq_kernel/kernel.py index a6bc43c29..c30ed122d 100644 --- a/jupyter/seq_kernel/kernel.py +++ b/jupyter/seq_kernel/kernel.py @@ -36,7 +36,6 @@ def banner(self): def __init__(self, **kwargs): Kernel.__init__(self, **kwargs) self.seqwrapper = SeqWrapper() - self.seqwrapper.exec('from jupyter import *') self.cells = [] def do_execute(self, code, silent, store_history=True, user_expressions=None, allow_stdin=False): From 0e0e999620123bed5a5f63ae83c844d49d90e833 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ibrahim=20Numanagic=CC=81?= Date: Wed, 4 Dec 2019 22:06:24 -0800 Subject: [PATCH 18/30] Add jupyter.seq --- stdlib/jupyter.seq | 57 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 stdlib/jupyter.seq diff --git a/stdlib/jupyter.seq b/stdlib/jupyter.seq new file mode 100644 index 000000000..cf863e998 --- /dev/null +++ b/stdlib/jupyter.seq @@ -0,0 +1,57 @@ + +COUNTER = 1 + +class SequencingBrowser: + url: str + genome: str + chr: str + start: int + end: int + + def __jupyter__(self: SequencingBrowser) -> str: + global COUNTER + cnt = COUNTER + COUNTER += 1 + + format = self.url.split('.')[-1] + fidx = 'bai' + if format == 'cram': fidx = 'crai' + js = str.cat([ + "igv.createBrowser(document.getElementById('igv-div-" + str(cnt) + "'), {", + "genome: '" + self.genome + "', locus: '" + self.chr + ":" + str(self.start) + "-" + str(self.end) + "',", + "tracks: [ {", + '"name": "Sample",', + '"url": "' + self.url + '",', + '"indexURL": "' + self.url + '.' + fidx + '",', + '"format": "' + format + '"', + "}]}).then(function (browser) { })" + ]) + return '\0x1text/html\b
' + +class JupyterView[T]: + s: str + + def __init__(self: JupyterView[T], x: T): + self.s = str(x) + + def __init__(self: JupyterView[T], x: SequencingBrowser): + self.s = x.__jupyter__() + + +# pydef seq_plot(g: pyobj) -> str: +# from io import BytesIO +# import base64 +# figfile = BytesIO() +# g.save(figfile, format='png') +# figfile.seek(0) +# figdata_png = base64.b64encode(figfile.getvalue()).decode('ascii') +# return '\1text/html\b' + +# pydef plot(y: list[float]) -> pyobj: +# from plotnine import ggplot, aes, geom_line +# import pandas as pd +# df = pd.DataFrame.from_dict({'x': range(len(y)), 'y': y}) +# return ggplot(df) + aes(x='x', y='y') + geom_line() + +# p = plot([1.0, 2.0, -3.4, 8.3, 4.4]) +# print seq_plot(p) From 2686ff6c427506fbf9c9e2108ab38b3a21233e90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ibrahim=20Numanagic=CC=81?= Date: Fri, 20 Dec 2019 18:56:49 -0800 Subject: [PATCH 19/30] os.path addition; Fix parameters in stdlib; Fix f-string error reporting --- parser/seqaml/codegen_expr.ml | 3 +- parser/seqaml/lexer.mll | 9 ++++-- parser/seqaml/parser.ml | 4 +-- stdlib/bio/bam.seq | 9 ++++++ stdlib/core/c_stubs.seq | 1 + stdlib/core/collections/dict.seq | 5 +++ stdlib/core/file.seq | 13 ++++++++ stdlib/itertools.seq | 7 ++--- stdlib/os.seq | 52 -------------------------------- stdlib/random.seq | 7 +++-- 10 files changed, 46 insertions(+), 64 deletions(-) delete mode 100644 stdlib/os.seq diff --git a/parser/seqaml/codegen_expr.ml b/parser/seqaml/codegen_expr.ml index faeb90d2c..7e1708c93 100644 --- a/parser/seqaml/codegen_expr.ml +++ b/parser/seqaml/codegen_expr.ml @@ -128,11 +128,12 @@ module Codegen (S : Codegen_intf.Stmt) : Codegen_intf.Expr = struct then true, String.prefix code ((String.length code) - 1) else false, code in - (* eprintf "??? %s\n" @@ code; *) + Lexer.global_offset.line <- pos.line - 1; let expr = match Parser.parse ~file:ctx.filename code with | [ _, Expr e ] -> Ast.(e_call ~pos (e_id ~pos "str") [e_setpos pos e]) | _ -> failwith "invalid f-parse" in + Lexer.global_offset.line <- 0; if extra then (expr :: ((pos, String ((String.strip code) ^ "=")) :: acc)), 0, i + 1 else diff --git a/parser/seqaml/lexer.mll b/parser/seqaml/lexer.mll index d30359448..983cce4da 100644 --- a/parser/seqaml/lexer.mll +++ b/parser/seqaml/lexer.mll @@ -14,6 +14,11 @@ (* open Core *) + type offset = + { mutable line: int + ; mutable col: int } + let global_offset = { line = 0; col = 0 } + (* Used for tracking indentation levels *) type stack = { stack: int Core.Stack.t; @@ -36,8 +41,8 @@ let cur_pos state ?(len=1) (lexbuf: Lexing.lexbuf) = Ast.Ann. { file = state.fname; - line = lexbuf.lex_start_p.pos_lnum; - col = lexbuf.lex_start_p.pos_cnum - lexbuf.lex_start_p.pos_bol; + line = lexbuf.lex_start_p.pos_lnum + global_offset.line; + col = lexbuf.lex_start_p.pos_cnum - lexbuf.lex_start_p.pos_bol + global_offset.col; len } let count_lines s = diff --git a/parser/seqaml/parser.ml b/parser/seqaml/parser.ml index 70d894eae..af91214c1 100644 --- a/parser/seqaml/parser.ml +++ b/parser/seqaml/parser.ml @@ -23,8 +23,8 @@ let parse ?f ?(file = "") code = let pos = Ast.Ann. { file - ; line = lexbuf.lex_start_p.pos_lnum - ; col = lexbuf.lex_start_p.pos_cnum - lexbuf.lex_start_p.pos_bol + ; line = lexbuf.lex_start_p.pos_lnum + Lexer.global_offset.line + ; col = lexbuf.lex_start_p.pos_cnum - lexbuf.lex_start_p.pos_bol + Lexer.global_offset.col ; len = 1 } in diff --git a/stdlib/bio/bam.seq b/stdlib/bio/bam.seq index 43b7852fb..20b345990 100644 --- a/stdlib/bio/bam.seq +++ b/stdlib/bio/bam.seq @@ -99,6 +99,10 @@ class SAMRecord: def name(self: SAMRecord): return self._name + @property + def query_name(self: SAMRecord): + return self._name + @property def read(self: SAMRecord): return self._read @@ -296,6 +300,11 @@ class BAM: def __exit__(self: BAM): self.close() + def reference_name(self: BAM, tid: int) -> str: + if 0 <= tid < len(self.targets): + return str(self.targets[tid]) + return "*" + class SAM: file: cobj hdr: cobj diff --git a/stdlib/core/c_stubs.seq b/stdlib/core/c_stubs.seq index d16f1cdb4..85248bd7d 100644 --- a/stdlib/core/c_stubs.seq +++ b/stdlib/core/c_stubs.seq @@ -83,6 +83,7 @@ cimport ferror(cobj) -> i32 cimport fgetc(cobj) -> i32 cimport fopen(cobj, cobj) -> cobj cimport fclose(cobj) -> int +cimport fread(cobj, int, int, cobj) -> int cimport fwrite(cobj, int, int, cobj) -> int cimport ftell(cobj) -> int cimport fseek(cobj, int, i32) -> i32 diff --git a/stdlib/core/collections/dict.seq b/stdlib/core/collections/dict.seq index f8938a558..97ddbbc8a 100644 --- a/stdlib/core/collections/dict.seq +++ b/stdlib/core/collections/dict.seq @@ -35,6 +35,11 @@ class dict[K,V]: for k,v in g: self[k] = v + def __init__(self: dict[K,V], g: tuple[K,V]): + self._init() + k, v = g + self[k] = v + def __getitem__(self: dict[K,V], key: K): x = self._kh_get(key) if x != self._kh_end(): diff --git a/stdlib/core/file.seq b/stdlib/core/file.seq index ed99603e1..7469bb3ff 100644 --- a/stdlib/core/file.seq +++ b/stdlib/core/file.seq @@ -80,6 +80,13 @@ class File: for s in g: self.write(str(s)) + def read(self: File, sz: int): + self._ensure_open() + buf = cobj(sz) + ret = _C.fread(buf, 1, sz, self.fp) + _f_errcheck(self.fp, "error in read") + return str(buf, ret) + def tell(self: File): ret = _C.ftell(self.fp) _f_errcheck(self.fp, "error in tell") @@ -196,3 +203,9 @@ def open(path: str, mode: str = "r"): def gzopen(path: str, mode: str = "r"): return gzFile(path, mode) + +def is_binary(path: str): + textchars = {7, 8, 9, 10, 12, 13, 27} | set(range(0x20, 0x100)) - {0x7f} + with open(path, "rb") as f: + header = f.read(1024) + return any(ord(c) not in textchars for c in header) diff --git a/stdlib/itertools.seq b/stdlib/itertools.seq index 196bd4904..0db077412 100644 --- a/stdlib/itertools.seq +++ b/stdlib/itertools.seq @@ -185,15 +185,14 @@ def filterfalse(predicate, iterable): if not predicate(x): yield x -def permutations(pool, r: int): +def permutations[T](pool: list[T], r_: optional[int] = None) -> list[T]: """ permutations(iterable, r) Return successive r-length permutations of elements in the iterable. - - Unlike python, seq does not use None for now so default for r will be len(pool) """ - n = len(pool) + r = ~r_ if r_ else len(pool) + n = len(pool) if r > n: return diff --git a/stdlib/os.seq b/stdlib/os.seq deleted file mode 100644 index 572f31fb8..000000000 --- a/stdlib/os.seq +++ /dev/null @@ -1,52 +0,0 @@ -def system(cmd: str) -> int: - return _C.system(cmd.c_str()) - -SEEK_SET = 0 -SEEK_CUR = 1 -SEEK_END = 2 - -type EnvMap(_map: dict[str,str]): - def __init__(self: EnvMap) -> EnvMap: - return (dict[str,str](),) - - def _init_if_needed(self: EnvMap): - if len(self._map) == 0: - env = _C.seq_env() - p = env[0] - i = 0 - while p: - s = str.from_ptr(p) - if s: - j = 0 - found = False - while j < len(s): - if s[j] == "=": - found = True - break - j += 1 - k = s[0:j] if found else s - v = s[j+1:] if found else "" - self._map[k] = v - i += 1 - p = env[i] - - def __getitem__(self: EnvMap, key: str): - self._init_if_needed() - return self._map[key] - - def __str__(self: EnvMap): - self._init_if_needed() - return str(self._map) - - def __contains__(self: EnvMap, key: str): - self._init_if_needed() - return key in self._map - - def __iter__(self: EnvMap): - self._init_if_needed() - return self._map.items() - -environ = EnvMap() - -def getenv(key: str, default: str = ""): - return environ[key] if key in environ else default diff --git a/stdlib/random.seq b/stdlib/random.seq index 8f11b2722..8690e270e 100644 --- a/stdlib/random.seq +++ b/stdlib/random.seq @@ -296,16 +296,17 @@ class Random: """ return self.gen.genrand_res53() - def choice(self: Random, seq): + def choice[T](self: Random, seq: generator[T]) -> T: """ Choose a random element from a non-empty sequence. """ i = 0 + l = list(seq) try: - i = self._randbelow_with_getrandbits(len(seq)) + i = self._randbelow_with_getrandbits(len(l)) except ValueError: raise IndexError("Cannot choose from an empty sequence") - return seq[i] + return l[i] def shuffle(self: Random, x): """Shuffle list x in place, and return None. From 76913c525db667d5a94072aecbed6585c3bd800a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ibrahim=20Numanagic=CC=81?= Date: Fri, 20 Dec 2019 19:52:56 -0800 Subject: [PATCH 20/30] os.path addition --- stdlib/os/__init__.seq | 52 ++++++++++++++++++++++++++++++++++++++++++ stdlib/os/path.seq | 17 ++++++++++++++ 2 files changed, 69 insertions(+) create mode 100644 stdlib/os/__init__.seq create mode 100644 stdlib/os/path.seq diff --git a/stdlib/os/__init__.seq b/stdlib/os/__init__.seq new file mode 100644 index 000000000..572f31fb8 --- /dev/null +++ b/stdlib/os/__init__.seq @@ -0,0 +1,52 @@ +def system(cmd: str) -> int: + return _C.system(cmd.c_str()) + +SEEK_SET = 0 +SEEK_CUR = 1 +SEEK_END = 2 + +type EnvMap(_map: dict[str,str]): + def __init__(self: EnvMap) -> EnvMap: + return (dict[str,str](),) + + def _init_if_needed(self: EnvMap): + if len(self._map) == 0: + env = _C.seq_env() + p = env[0] + i = 0 + while p: + s = str.from_ptr(p) + if s: + j = 0 + found = False + while j < len(s): + if s[j] == "=": + found = True + break + j += 1 + k = s[0:j] if found else s + v = s[j+1:] if found else "" + self._map[k] = v + i += 1 + p = env[i] + + def __getitem__(self: EnvMap, key: str): + self._init_if_needed() + return self._map[key] + + def __str__(self: EnvMap): + self._init_if_needed() + return str(self._map) + + def __contains__(self: EnvMap, key: str): + self._init_if_needed() + return key in self._map + + def __iter__(self: EnvMap): + self._init_if_needed() + return self._map.items() + +environ = EnvMap() + +def getenv(key: str, default: str = ""): + return environ[key] if key in environ else default diff --git a/stdlib/os/path.seq b/stdlib/os/path.seq new file mode 100644 index 000000000..e4f9c7696 --- /dev/null +++ b/stdlib/os/path.seq @@ -0,0 +1,17 @@ +def splitext(p: str) -> tuple[str, str]: + """Split the extension from a pathname. + Extension is everything from the last dot to the end, ignoring + leading dots. Returns "(root, ext)"; ext may be empty.""" + sep = '/' + extsep = '.' + + sepIndex = p.rfind(sep) + dotIndex = p.rfind(extsep) + if dotIndex > sepIndex: + # skip all leading dots + filenameIndex = sepIndex + 1 + while filenameIndex < dotIndex: + if p[filenameIndex:filenameIndex+1] != extsep: + return p[:dotIndex], p[dotIndex:] + filenameIndex += 1 + return p, p[:0] From a1f0268fac5087bc0fedcfcedd85613a186f2b01 Mon Sep 17 00:00:00 2001 From: "A. R. Shajii" Date: Sat, 21 Dec 2019 23:07:17 -0500 Subject: [PATCH 21/30] Trim whitespace --- parser/seqaml/codegen_expr.ml | 4 ++-- parser/seqaml/codegen_stmt.ml | 4 ++-- parser/seqaml/grammar.mly | 8 ++++---- parser/seqaml/lexer.mll | 2 +- stdlib/getopt.seq | 10 +++++----- 5 files changed, 14 insertions(+), 14 deletions(-) diff --git a/parser/seqaml/codegen_expr.ml b/parser/seqaml/codegen_expr.ml index 7e1708c93..d77f87dca 100644 --- a/parser/seqaml/codegen_expr.ml +++ b/parser/seqaml/codegen_expr.ml @@ -123,7 +123,7 @@ module Codegen (S : Codegen_intf.Stmt) : Codegen_intf.Expr = struct | '{' -> acc, depth + 1, last | '}' when depth = 1 -> let code = String.sub s ~pos:last ~len:(i - last) in - let extra, code = + let extra, code = if (String.suffix code 1) = "=" then true, String.prefix code ((String.length code) - 1) else false, code @@ -134,7 +134,7 @@ module Codegen (S : Codegen_intf.Stmt) : Codegen_intf.Expr = struct | _ -> failwith "invalid f-parse" in Lexer.global_offset.line <- 0; - if extra then + if extra then (expr :: ((pos, String ((String.strip code) ^ "=")) :: acc)), 0, i + 1 else (expr :: acc), 0, i + 1 diff --git a/parser/seqaml/codegen_stmt.ml b/parser/seqaml/codegen_stmt.ml index dabbf51c9..392cf5a33 100644 --- a/parser/seqaml/codegen_stmt.ml +++ b/parser/seqaml/codegen_stmt.ml @@ -667,7 +667,7 @@ module Codegen (E : Codegen_intf.Expr) : Codegen_intf.Stmt = struct Ctx.clear_block ctx (** Helper for parsing match patterns *) - and parse_pattern ctx pos p = + and parse_pattern ctx pos p = let p = match p with | StarPattern -> Llvm.Pattern.star () | BoundPattern _ -> serr ~pos "invalid bound pattern" @@ -698,7 +698,7 @@ module Codegen (E : Codegen_intf.Expr) : Codegen_intf.Stmt = struct let pat = parse_pattern ctx pos pat in let expr = E.parse ~ctx expr in Llvm.Pattern.guarded pat expr - in + in Llvm.Stmt.set_pos p pos; p diff --git a/parser/seqaml/grammar.mly b/parser/seqaml/grammar.mly index d915e893e..1f24705ff 100644 --- a/parser/seqaml/grammar.mly +++ b/parser/seqaml/grammar.mly @@ -144,7 +144,7 @@ reverse_separated_nonempty_llist(separator, X): %inline flexible_nonempty_list_flag(delim, X): | x = X { [x], false } - | x = X; delim; xs = separated_llist(delim, X); f= delim? + | x = X; delim; xs = separated_llist(delim, X); f= delim? { x :: xs, Option.is_some f } @@ -379,9 +379,9 @@ arith_term: Call ($1, [pos $2 $5, { name = None; value = (pos $2 $5, Generator ($3, $4)) }]) } // Index (foo[bar]) - | arith_term LS index_term RS + | arith_term LS index_term RS { pos (fst $1) $4, Index ($1, $3) } - | arith_term LS index_term COMMA RS + | arith_term LS index_term COMMA RS { pos (fst $1) $4, Index ($1, (fst $3, Tuple [$3])) } | arith_term LS index_term COMMA flexible_nonempty_list(COMMA, index_term) RS { pos (fst $1) $6, @@ -531,7 +531,7 @@ small_statement: in [ pos, Yield expr ]} - | YIELD FROM expr + | YIELD FROM expr { (* for i in expr: yield i *) let p = pos $1 (fst $3) in diff --git a/parser/seqaml/lexer.mll b/parser/seqaml/lexer.mll index 983cce4da..e6a1b9eff 100644 --- a/parser/seqaml/lexer.mll +++ b/parser/seqaml/lexer.mll @@ -14,7 +14,7 @@ (* open Core *) - type offset = + type offset = { mutable line: int ; mutable col: int } let global_offset = { line = 0; col = 0 } diff --git a/stdlib/getopt.seq b/stdlib/getopt.seq index fc2b66cea..456ae1d7b 100644 --- a/stdlib/getopt.seq +++ b/stdlib/getopt.seq @@ -64,7 +64,7 @@ def do_longs(opts: list[tuple[str, str]], opt: str, longopts: list[str], args: l i = opt.index('=') opt, optarg = opt[:i], opt[i+1:] except ValueError: - pass + pass has_arg, opt = long_has_args(opt, longopts) if has_arg: @@ -100,8 +100,8 @@ def do_shorts(opts: list[tuple[str, str]], optstring: str, shortopts: str, args: def getopt( - args: list[str], - shortopts: str, + args: list[str], + shortopts: str, longopts: list[str] = list[str]() ): """getopt(args, options[, long_options]) -> opts, args @@ -141,7 +141,7 @@ def getopt( opts = list[tuple[str, str]]() prog_args = list[str]() - + # Allow options after non-option arguments? all_options_first = False if shortopts.startswith('+'): @@ -149,7 +149,7 @@ def getopt( all_options_first = True elif "POSIXLY_CORRECT" in os.environ: all_options_first = True - + while args: if args[0] == '--': prog_args += args[1:] From 114be8821caf09f41902dddbf5c90d0f6d7ca747 Mon Sep 17 00:00:00 2001 From: "A. R. Shajii" Date: Sat, 21 Dec 2019 23:07:37 -0500 Subject: [PATCH 22/30] Remove unused htslib option --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 19488a91d..108dfb2b5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -94,7 +94,7 @@ install: # install htslib: - wget -c https://github.com/samtools/htslib/releases/download/1.9/htslib-1.9.tar.bz2 -O - | tar -xj - cd htslib-1.9 - - ./configure CFLAGS="-fPIC" --disable-libcurl --without-curses + - ./configure CFLAGS="-fPIC" --disable-libcurl - make - sudo make install - cd - From 3632caa302e90f6228717021cace57cecba1ea96 Mon Sep 17 00:00:00 2001 From: "A. R. Shajii" Date: Sat, 21 Dec 2019 23:08:02 -0500 Subject: [PATCH 23/30] Fix itertools --- stdlib/itertools.seq | 36 +++++++++++++++++++----------------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/stdlib/itertools.seq b/stdlib/itertools.seq index 0db077412..f7378f946 100644 --- a/stdlib/itertools.seq +++ b/stdlib/itertools.seq @@ -1,7 +1,7 @@ import math import collections -def combinations(pool, r: int): +def combinations[T](pool: generator[T], r: int): """ combinations(iterable, r) @@ -9,11 +9,12 @@ def combinations(pool, r: int): combinations(range(4), 3) --> (0,1,2), (0,1,3), (0,2,3), (1,2,3) """ - n = len(pool) + pool_list = [a for a in pool] + n = len(pool_list) if r > n: return indices = list(range(r).__iter__()) - yield [pool[i] for i in indices] + yield [pool_list[i] for i in indices] while True: b = -1 for i in reversed(range(r)): @@ -25,9 +26,9 @@ def combinations(pool, r: int): indices[b] += 1 for j in range(b+1, r): indices[j] = indices[j-1] + 1 - yield [pool[i] for i in indices] + yield [pool_list[i] for i in indices] -def combinations_with_replacement(pool, r: int): +def combinations_with_replacement[T](pool: generator[T], r: int): """ combinations_with_replacement(iterable, r) @@ -36,11 +37,12 @@ def combinations_with_replacement(pool, r: int): """ if r < 0: raise ValueError("r must be a positive integer.") - n = len(pool) + pool_list = [a for a in pool] + n = len(pool_list) if not n and r: return indices = [0] * r - yield [pool[i] for i in indices] + yield [pool_list[i] for i in indices] while True: b = -1 for i in reversed(range(r)): @@ -51,8 +53,7 @@ def combinations_with_replacement(pool, r: int): return indices = [indices[i] for i in range(b)] + [indices[b]+1 for i in range(r-b)] - yield [pool[indices[i]] for i in range(len(indices))] - + yield [pool_list[indices[i]] for i in range(len(indices))] def islice[T](iterable: generator[T], start: int, stop: int, step: int): """ @@ -185,23 +186,24 @@ def filterfalse(predicate, iterable): if not predicate(x): yield x -def permutations[T](pool: list[T], r_: optional[int] = None) -> list[T]: +def permutations[T](pool: generator[T], r: optional[int] = None) -> list[T]: """ permutations(iterable, r) Return successive r-length permutations of elements in the iterable. """ - r = ~r_ if r_ else len(pool) - n = len(pool) - if r > n: + pool_list = [a for a in pool] + n = len(pool_list) + rx = ~r if r else n + if rx > n: return indices = list(iter(range(n))) - cycles = list(iter(range(n, n-r, -1))) - yield [pool[i] for i in indices[:r]] + cycles = list(iter(range(n, n-rx, -1))) + yield [pool_list[i] for i in indices[:rx]] while n: b = -1 - for i in reversed(range(r)): + for i in reversed(range(rx)): cycles[i] -= 1 if cycles[i] == 0: indices = indices[:i] + indices[i+1:] + indices[i:i+1] @@ -210,7 +212,7 @@ def permutations[T](pool: list[T], r_: optional[int] = None) -> list[T]: b = i j = cycles[i] indices[i], indices[-j] = indices[-j], indices[i] - yield [pool[i] for i in indices[:r]] + yield [pool_list[i] for i in indices[:rx]] break if b == -1: return From a3aa5ec2e44ce654bb9c6298a3ffd16afe4b36ab Mon Sep 17 00:00:00 2001 From: "A. R. Shajii" Date: Mon, 23 Dec 2019 18:57:10 -0500 Subject: [PATCH 24/30] Add yield expression --- compiler/include/seq/expr.h | 12 +++++++++++ compiler/include/seq/func.h | 1 + compiler/include/seq/funct.h | 4 +++- compiler/lang/expr.cpp | 20 +++++++++++++++++ compiler/lang/func.cpp | 21 ++++++++++++++++++ compiler/types/funct.cpp | 42 ++++++++++++++++++++++++++++++++++-- compiler/util/ocaml.cpp | 2 ++ 7 files changed, 99 insertions(+), 3 deletions(-) diff --git a/compiler/include/seq/expr.h b/compiler/include/seq/expr.h index fe612f6c1..894b26fd1 100644 --- a/compiler/include/seq/expr.h +++ b/compiler/include/seq/expr.h @@ -578,6 +578,18 @@ class OptExpr : public Expr { OptExpr *clone(Generic *ref) override; }; +class YieldExpr : public Expr { +private: + Func *base; + +public: + YieldExpr(Func *base); + void resolveTypes() override; + llvm::Value *codegen0(BaseFunc *base, llvm::BasicBlock *&block) override; + types::Type *getType0() const override; + YieldExpr *clone(Generic *ref) override; +}; + class DefaultExpr : public Expr { public: explicit DefaultExpr(types::Type *type); diff --git a/compiler/include/seq/func.h b/compiler/include/seq/func.h index 08df40431..c1f6aa116 100644 --- a/compiler/include/seq/func.h +++ b/compiler/include/seq/func.h @@ -162,6 +162,7 @@ class Func : public BaseFunc, public Generic, public SrcObject { void codegenYield(llvm::Value *val, types::Type *type, llvm::BasicBlock *&block, bool empty = false, bool dryrun = false); + llvm::Value *codegenYieldExpr(llvm::BasicBlock *&block); bool isGen() override; Var *getArgVar(std::string name); diff --git a/compiler/include/seq/funct.h b/compiler/include/seq/funct.h index fc54e2337..0f5f767d7 100644 --- a/compiler/include/seq/funct.h +++ b/compiler/include/seq/funct.h @@ -54,7 +54,9 @@ class GenType : public Type { llvm::Value *done(llvm::Value *self, llvm::BasicBlock *block); void resume(llvm::Value *self, llvm::BasicBlock *block, llvm::BasicBlock *normal, llvm::BasicBlock *unwind); - llvm::Value *promise(llvm::Value *self, llvm::BasicBlock *block); + llvm::Value *promise(llvm::Value *self, llvm::BasicBlock *block, + bool returnPtr = false); + void send(llvm::Value *self, llvm::Value *val, llvm::BasicBlock *block); void destroy(llvm::Value *self, llvm::BasicBlock *block); bool fromPrefetch(); diff --git a/compiler/lang/expr.cpp b/compiler/lang/expr.cpp index 494710531..69ddb8e84 100644 --- a/compiler/lang/expr.cpp +++ b/compiler/lang/expr.cpp @@ -2089,6 +2089,26 @@ OptExpr *OptExpr::clone(Generic *ref) { SEQ_RETURN_CLONE(new OptExpr(val->clone(ref))); } +YieldExpr::YieldExpr(Func *base) : Expr(), base(base) {} + +void YieldExpr::resolveTypes() { base->resolveTypes(); } + +Value *YieldExpr::codegen0(BaseFunc *base, BasicBlock *&block) { + assert(dynamic_cast(base) == this->base); + return this->base->codegenYieldExpr(block); +} + +types::Type *YieldExpr::getType0() const { + types::GenType *gen = base->getFuncType()->getBaseType(0)->asGen(); + if (!gen) + throw exc::SeqException("yield expression in non-generator"); + return gen->getBaseType(0); +} + +YieldExpr *YieldExpr::clone(Generic *ref) { + SEQ_RETURN_CLONE(new YieldExpr(base->clone(ref))); +} + DefaultExpr::DefaultExpr(types::Type *type) : Expr(type) {} Value *DefaultExpr::codegen0(BaseFunc *base, BasicBlock *&block) { diff --git a/compiler/lang/func.cpp b/compiler/lang/func.cpp index 1ef43edb5..c3adfef7c 100644 --- a/compiler/lang/func.cpp +++ b/compiler/lang/func.cpp @@ -596,6 +596,27 @@ void Func::codegenYield(Value *val, types::Type *type, BasicBlock *&block, } } +Value *Func::codegenYieldExpr(BasicBlock *&block) { + if (!gen) + throw exc::SeqException("yield expression in non-generator"); + + LLVMContext &context = block->getContext(); + Function *suspFn = Intrinsic::getDeclaration(module, Intrinsic::coro_suspend); + Value *tok = ConstantTokenNone::get(context); + Value *final = ConstantInt::get(IntegerType::getInt1Ty(context), 0); + IRBuilder<> builder(block); + Value *susp = builder.CreateCall(suspFn, {tok, final}); + + block = BasicBlock::Create(block->getContext(), "", block->getParent()); + + SwitchInst *inst = builder.CreateSwitch(susp, suspend, 2); + inst->addCase(ConstantInt::get(IntegerType::getInt8Ty(context), 0), block); + inst->addCase(ConstantInt::get(IntegerType::getInt8Ty(context), 1), cleanup); + + builder.SetInsertPoint(block); + return builder.CreateLoad(promise); +} + bool Func::isGen() { return yield != nullptr; } std::vector Func::getArgNames() { return argNames; } diff --git a/compiler/types/funct.cpp b/compiler/types/funct.cpp index 08317f757..4a59a3c71 100644 --- a/compiler/types/funct.cpp +++ b/compiler/types/funct.cpp @@ -168,7 +168,7 @@ void types::GenType::resume(Value *self, BasicBlock *block, BasicBlock *normal, builder.CreateCall(resFn, self); } -Value *types::GenType::promise(Value *self, BasicBlock *block) { +Value *types::GenType::promise(Value *self, BasicBlock *block, bool returnPtr) { if (outType->is(types::Void)) return nullptr; @@ -187,7 +187,15 @@ Value *types::GenType::promise(Value *self, BasicBlock *block) { Value *ptr = builder.CreateCall(promFn, {self, aln, from}); ptr = builder.CreateBitCast( ptr, PointerType::get(outType->getLLVMType(context), 0)); - return builder.CreateLoad(ptr); + return returnPtr ? ptr : builder.CreateLoad(ptr); +} + +void types::GenType::send(Value *self, Value *val, BasicBlock *block) { + Value *promisePtr = promise(self, block, /*returnPtr=*/true); + if (!promisePtr) + throw exc::SeqException("cannot send value to void generator"); + IRBuilder<> builder(block); + builder.CreateStore(val, promisePtr); } void types::GenType::destroy(Value *self, BasicBlock *block) { @@ -269,6 +277,36 @@ void types::GenType::initOps() { }), true); + addMethod("send", + new BaseFuncLite( + {this, outType}, Void, + [this](Module *module) { + const std::string name = "seq." + getName() + ".send"; + Function *func = module->getFunction(name); + + if (!func) { + LLVMContext &context = module->getContext(); + func = cast(module->getOrInsertFunction( + name, Void->getLLVMType(context), getLLVMType(context), + outType->getLLVMType(context))); + func->setLinkage(GlobalValue::PrivateLinkage); + func->setDoesNotThrow(); + func->addFnAttr(Attribute::AlwaysInline); + + auto iter = func->arg_begin(); + Value *self = iter++; + Value *val = iter; + BasicBlock *entry = + BasicBlock::Create(context, "entry", func); + send(self, val, entry); + IRBuilder<> builder(entry); + builder.CreateRetVoid(); + } + + return func; + }), + true); + addMethod( "destroy", new BaseFuncLite( diff --git a/compiler/util/ocaml.cpp b/compiler/util/ocaml.cpp index ccf9ab39c..97c68d89a 100644 --- a/compiler/util/ocaml.cpp +++ b/compiler/util/ocaml.cpp @@ -287,6 +287,8 @@ FOREIGN Expr *method_expr(Expr *self, Func *func) { return new MethodExpr(self, func); } +FOREIGN Expr *yield_expr(Func *base) { return new YieldExpr(base); } + FOREIGN Expr *typeof_expr(Expr *val) { return new TypeOfExpr(val); } FOREIGN Expr *ptr_expr(Var *val) { return new VarPtrExpr(val); } From 0161cd5a483f5c874868bb98d483213798fdfb23 Mon Sep 17 00:00:00 2001 From: "A. R. Shajii" Date: Mon, 23 Dec 2019 20:29:34 -0500 Subject: [PATCH 25/30] Fix revcomp'd sequence matching --- compiler/include/seq/num.h | 4 ++++ compiler/lang/patterns.cpp | 37 +++++++++++++++++++++++++++++++------ compiler/types/num.cpp | 8 +++----- test/core/match.seq | 2 ++ 4 files changed, 40 insertions(+), 11 deletions(-) diff --git a/compiler/include/seq/num.h b/compiler/include/seq/num.h index ec99eb751..bcbac7237 100644 --- a/compiler/include/seq/num.h +++ b/compiler/include/seq/num.h @@ -98,6 +98,10 @@ class ByteType : public Type { llvm::Type *getLLVMType(llvm::LLVMContext &context) const override; size_t size(llvm::Module *module) const override; static ByteType *get() noexcept; + /// returns ASCII complement table + static llvm::GlobalVariable * + getByteCompTable(llvm::Module *module, + const std::string &name = "seq.byte_comp_table"); }; } // namespace types diff --git a/compiler/lang/patterns.cpp b/compiler/lang/patterns.cpp index 8cb643af3..aa82a54ea 100644 --- a/compiler/lang/patterns.cpp +++ b/compiler/lang/patterns.cpp @@ -312,6 +312,29 @@ void SeqPattern::resolveTypes(types::Type *type) { getSrcInfo()); } +// Returns the appropriate character for the given logical index, respecting +// reverse complementation. Given `lenActual` should be non-negative. +static Value *indexIntoSeq(Value *ptr, Value *lenActual, Value *rc, Value *idx, + BasicBlock *block) { + LLVMContext &context = block->getContext(); + IRBuilder<> builder(block); + Value *backIdx = builder.CreateSub(lenActual, idx); + backIdx = builder.CreateSub(backIdx, oneLLVM(context)); + + Value *charFwdPtr = builder.CreateGEP(ptr, idx); + Value *charRevPtr = builder.CreateGEP(ptr, backIdx); + + Value *charFwd = builder.CreateLoad(charFwdPtr); + Value *charRev = builder.CreateLoad(charRevPtr); + + GlobalVariable *table = types::ByteType::getByteCompTable(block->getModule()); + charRev = builder.CreateZExt(charRev, builder.getInt64Ty()); + charRev = builder.CreateInBoundsGEP(table, {builder.getInt64(0), charRev}); + charRev = builder.CreateLoad(charRev); + + return builder.CreateSelect(rc, charRev, charFwd); +} + static Value *codegenSeqMatchForSeq(const std::vector &patterns, BaseFunc *base, types::Type *type, Value *val, BasicBlock *&block) { @@ -330,14 +353,16 @@ static Value *codegenSeqMatchForSeq(const std::vector &patterns, Value *lenMatch = nullptr; BasicBlock *startBlock = block; IRBuilder<> builder(block); + Value *rc = builder.CreateICmpSLT(len, zeroLLVM(context)); + Value *lenActual = builder.CreateSelect(rc, builder.CreateNeg(len), len); // check lengths: if (hasStar) { Value *minLen = ConstantInt::get(seqIntLLVM(context), patterns.size() - 1); - lenMatch = builder.CreateICmpSGE(len, minLen); + lenMatch = builder.CreateICmpSGE(lenActual, minLen); } else { Value *expectedLen = ConstantInt::get(seqIntLLVM(context), patterns.size()); - lenMatch = builder.CreateICmpEQ(len, expectedLen); + lenMatch = builder.CreateICmpEQ(lenActual, expectedLen); } block = BasicBlock::Create( @@ -353,7 +378,7 @@ static Value *codegenSeqMatchForSeq(const std::vector &patterns, if (patterns[i] == '_') continue; Value *idx = ConstantInt::get(seqIntLLVM(context), i); - Value *sub = builder.CreateLoad(builder.CreateGEP(ptr, idx)); + Value *sub = indexIntoSeq(ptr, lenActual, rc, idx, block); Value *c = ConstantInt::get(IntegerType::getInt8Ty(context), (uint64_t)patterns[i]); Value *subRes = builder.CreateICmpEQ(sub, c); @@ -366,11 +391,11 @@ static Value *codegenSeqMatchForSeq(const std::vector &patterns, if (patterns[i] == '_') continue; Value *idx = ConstantInt::get(seqIntLLVM(context), i); - idx = builder.CreateAdd(idx, len); + idx = builder.CreateAdd(idx, lenActual); idx = builder.CreateSub( idx, ConstantInt::get(seqIntLLVM(context), patterns.size())); - Value *sub = builder.CreateLoad(builder.CreateGEP(ptr, idx)); + Value *sub = indexIntoSeq(ptr, lenActual, rc, idx, block); Value *c = ConstantInt::get(IntegerType::getInt8Ty(context), (uint64_t)patterns[i]); Value *subRes = builder.CreateICmpEQ(sub, c); @@ -383,7 +408,7 @@ static Value *codegenSeqMatchForSeq(const std::vector &patterns, if (patterns[i] == '_') continue; Value *idx = ConstantInt::get(seqIntLLVM(context), i); - Value *sub = builder.CreateLoad(builder.CreateGEP(ptr, idx)); + Value *sub = indexIntoSeq(ptr, lenActual, rc, idx, block); Value *c = ConstantInt::get(IntegerType::getInt8Ty(context), (uint64_t)patterns[i]); Value *subRes = builder.CreateICmpEQ(sub, c); diff --git a/compiler/types/num.cpp b/compiler/types/num.cpp index 4148b8921..5dc246670 100644 --- a/compiler/types/num.cpp +++ b/compiler/types/num.cpp @@ -1334,12 +1334,10 @@ void types::BoolType::initOps() { }; } -// ASCII complement table -static GlobalVariable * -getByteCompTable(Module *module, - const std::string &name = "seq.byte_comp_table") { +GlobalVariable *types::ByteType::getByteCompTable(Module *module, + const std::string &name) { LLVMContext &context = module->getContext(); - Type *ty = IntegerType::getInt8Ty(context); + auto *ty = IntegerType::getInt8Ty(context); GlobalVariable *table = module->getGlobalVariable(name); if (!table) { diff --git a/test/core/match.seq b/test/core/match.seq index 2cb872b20..6a71ba366 100644 --- a/test/core/match.seq +++ b/test/core/match.seq @@ -128,7 +128,9 @@ def f(k): @test def test_seq_match(): s = s'GCGTC' + t = ~s'GACGC' # == ~s assert list(f(s)) == [0, 0, 1, 0, 1, 0, 0, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 0, 0, 0, 1] + assert list(f(t)) == [0, 0, 1, 0, 1, 0, 0, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 0, 0, 0, 1] assert list(f(Kmer[5](s))) == [0, 0, 1, 0, 1, 0, 0, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 0, 0, 0, 1] test_seq_match() From cc31f8ea14a487b2be6586966ebaedc9be4d46f7 Mon Sep 17 00:00:00 2001 From: "A. R. Shajii" Date: Tue, 24 Dec 2019 18:21:05 -0500 Subject: [PATCH 26/30] Faster k-mer matching --- compiler/lang/patterns.cpp | 61 +++++++++++++++++++++++++++++++++++--- compiler/types/seqt.cpp | 16 ++++++++++ 2 files changed, 73 insertions(+), 4 deletions(-) diff --git a/compiler/lang/patterns.cpp b/compiler/lang/patterns.cpp index aa82a54ea..45ad76c0b 100644 --- a/compiler/lang/patterns.cpp +++ b/compiler/lang/patterns.cpp @@ -438,9 +438,13 @@ static Value *codegenSeqMatchForSeq(const std::vector &patterns, static Value *codegenSeqMatchForKmer(const std::vector &patterns, BaseFunc *base, types::KMer *type, Value *val, BasicBlock *&block) { + static const unsigned BRUTE_MATCH_K_CUTOFF = 256; + static const unsigned BRUTE_MATCH_P_CUTOFF = 100; + LLVMContext &context = block->getContext(); Value *fail = ConstantInt::get(IntegerType::getInt1Ty(context), 0); Value *succ = ConstantInt::get(IntegerType::getInt1Ty(context), 1); + Value *falseBool = ConstantInt::get(types::Bool->getLLVMType(context), 0); if (patterns.empty()) return fail; // k-mer length must be at least 1 @@ -476,11 +480,59 @@ static Value *codegenSeqMatchForKmer(const std::vector &patterns, } Type *llvmType = type->getLLVMType(context); + + // if k is small and pattern is small, brute force matching is fastest: + if (hasStar && k <= BRUTE_MATCH_K_CUTOFF && + patterns.size() <= BRUTE_MATCH_P_CUTOFF) { + types::KMer *k1Type = types::KMer::get(1); + + BasicBlock *succBlock = block; + BasicBlock *failBlock = BasicBlock::Create(context, "", block->getParent()); + + IRBuilder<> builder(succBlock); + bool backIndex = false; + + for (unsigned i = 0; i < patterns.size(); i++) { + const char c = patterns[i]; + if (c == '_') + continue; + if (c == '\0') { + assert(!backIndex); + backIndex = true; + continue; + } + unsigned idx = backIndex ? (k + i - patterns.size()) : i; + Value *idxVal = ConstantInt::get(seqIntLLVM(context), idx); + Value *base = type->callMagic("__getitem__", {types::Int}, val, {idxVal}, + succBlock, nullptr); + Value *expected = + k1Type->callMagic("__init__", {types::Byte}, nullptr, + {builder.getInt8(c)}, succBlock, nullptr); + Value *match = builder.CreateICmpEQ(base, expected); + succBlock = BasicBlock::Create(context, "", block->getParent()); + builder.CreateCondBr(match, succBlock, failBlock); + builder.SetInsertPoint(succBlock); + } + + block = BasicBlock::Create(context, "", block->getParent()); + builder.CreateBr(block); + + builder.SetInsertPoint(failBlock); + builder.CreateBr(block); + + builder.SetInsertPoint(block); + PHINode *result = builder.CreatePHI(IntegerType::getInt1Ty(context), 2); + result->addIncoming(succ, succBlock); + result->addIncoming(fail, failBlock); + return result; + } + if (!hasStar) { SeqExpr s(std::string(patterns.begin(), patterns.end())); Value *expectedSeq = s.codegen(base, block); - Value *expectedKmer = type->callMagic("__init__", {types::Seq}, nullptr, - {expectedSeq}, block, nullptr); + Value *expectedKmer = + type->callMagic("__init__", {types::Seq, types::Bool}, nullptr, + {expectedSeq, falseBool}, block, nullptr); IRBuilder<> builder(block); if (!hasWildcard) { @@ -513,8 +565,9 @@ static Value *codegenSeqMatchForKmer(const std::vector &patterns, SeqExpr sLeft(leftString); Value *expectedSeqLeft = sLeft.codegen(base, block); typeLeft = types::KMer::get(leftString.size()); - expectedKmerLeft = typeLeft->callMagic("__init__", {types::Seq}, nullptr, - {expectedSeqLeft}, block, nullptr); + expectedKmerLeft = + typeLeft->callMagic("__init__", {types::Seq, types::Bool}, nullptr, + {expectedSeqLeft, falseBool}, block, nullptr); } if (!rightString.empty()) { diff --git a/compiler/types/seqt.cpp b/compiler/types/seqt.cpp index 9f9140a2b..ee2441ce4 100644 --- a/compiler/types/seqt.cpp +++ b/compiler/types/seqt.cpp @@ -782,6 +782,22 @@ void types::KMer::initOps() { true}, }; + if (k == 1) { + vtable.magic.push_back( + {"__init__", + {Byte}, + this, + [](Value *self, std::vector args, IRBuilder<> &b) { + GlobalVariable *table = + get2bitTable(b.GetInsertBlock()->getModule()); + Value *base = b.CreateZExt(args[0], b.getInt64Ty()); + Value *bits = + b.CreateLoad(b.CreateInBoundsGEP(table, {b.getInt64(0), base})); + return bits; + }, + false}); + } + addMethod( "len", new BaseFuncLite( From 1d3259ad6f9890d9f713073fed4bc2f11c4b3925 Mon Sep 17 00:00:00 2001 From: "A. R. Shajii" Date: Sun, 29 Dec 2019 10:23:20 -0500 Subject: [PATCH 27/30] Add builtin tags --- stdlib/core/builtin.seq | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/stdlib/core/builtin.seq b/stdlib/core/builtin.seq index a1c75e5d5..192f5bc66 100644 --- a/stdlib/core/builtin.seq +++ b/stdlib/core/builtin.seq @@ -32,12 +32,15 @@ class max[T]: x.destroy() raise ValueError("empty sequence") +@builtin def min2(a, b): return a if a < b else b +@builtin def max2(a, b): return a if a > b else b +@builtin def len(x): """ len(x) @@ -46,6 +49,7 @@ def len(x): """ return x.__len__() +@builtin def iter(x): """ iter(x) @@ -54,6 +58,7 @@ def iter(x): """ return x.__iter__() +@builtin def copy(x): """ copy(x) @@ -62,6 +67,7 @@ def copy(x): """ return x.__copy__() +@builtin def abs(x): """ abs(x) @@ -70,6 +76,7 @@ def abs(x): """ return x.__abs__() +@builtin def hash(x): """ hash(x) @@ -78,6 +85,7 @@ def hash(x): """ return x.__hash__() +@builtin def ord(s: str): """ ord(s) @@ -88,6 +96,7 @@ def ord(s: str): raise TypeError("ord() expected a character, but string of length " + str(len(s)) + " found") return int(s.ptr[0]) +@builtin def chr(i: int): """ chr(i) @@ -99,6 +108,7 @@ def chr(i: int): p[0] = byte(i) return str(p, 1) +@builtin def next[T](g: generator[T], default: optional[T] = None): """ next(g) @@ -112,6 +122,7 @@ def next[T](g: generator[T], default: optional[T] = None): raise StopIteration() return g.next() +@builtin def any(x): """ any(x) @@ -124,6 +135,7 @@ def any(x): return True return False +@builtin def all(x): """ all(x) @@ -136,6 +148,7 @@ def all(x): return False return True +@builtin def zip(a, b): """ zip(a, b) @@ -149,6 +162,7 @@ def zip(a, b): yield (i, bi.next()) bi.destroy() +@builtin def filter(f, x): """ filter(f, x) @@ -159,6 +173,7 @@ def filter(f, x): if f(a): yield a +@builtin def map(f, x): """ map(f, x) @@ -168,6 +183,7 @@ def map(f, x): for a in x: yield f(a) +@builtin def enumerate(x, start: int = 0): """ enumerate(x) @@ -180,6 +196,7 @@ def enumerate(x, start: int = 0): yield (i,a) i += 1 +@builtin def echo(x): """ echo(x) @@ -189,6 +206,7 @@ def echo(x): print x return x +@builtin def reversed(x): """ reversed(x) @@ -197,6 +215,7 @@ def reversed(x): """ return x.__reversed__() +@builtin def round(x, n) -> float: """ round(x, n) @@ -207,6 +226,7 @@ def round(x, n) -> float: n := _C.pow(10.0, float(n)) return _C.round(float(x * n)) / n +@builtin def sum(xi): """ sum(xi) From 6d7e7b1812fd7e9ab565f3cd6805158986e72ed9 Mon Sep 17 00:00:00 2001 From: "A. R. Shajii" Date: Sun, 29 Dec 2019 10:24:01 -0500 Subject: [PATCH 28/30] Fix ConstructExpr cloning --- compiler/include/seq/expr.h | 1 + compiler/lang/expr.cpp | 9 ++++++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/compiler/include/seq/expr.h b/compiler/include/seq/expr.h index 894b26fd1..0e91dc49d 100644 --- a/compiler/include/seq/expr.h +++ b/compiler/include/seq/expr.h @@ -541,6 +541,7 @@ class MatchExpr : public Expr { class ConstructExpr : public Expr { private: mutable types::Type *type; + mutable types::Type *type0; // type before deduction, saved for clone std::vector args; public: diff --git a/compiler/lang/expr.cpp b/compiler/lang/expr.cpp index 69ddb8e84..a1b8052b7 100644 --- a/compiler/lang/expr.cpp +++ b/compiler/lang/expr.cpp @@ -1943,7 +1943,7 @@ MatchExpr *MatchExpr::clone(Generic *ref) { } ConstructExpr::ConstructExpr(types::Type *type, std::vector args) - : Expr(), type(type), args(std::move(args)) {} + : Expr(), type(type), type0(nullptr), args(std::move(args)) {} types::Type *ConstructExpr::getConstructType() { return type; } @@ -2035,8 +2035,10 @@ types::Type *ConstructExpr::getType0() const { // type parameter deduction if constructing generic class: auto *ref = dynamic_cast(type); - if (ref && ref->numGenerics() > 0 && !ref->realized()) + if (ref && ref->numGenerics() > 0 && !ref->realized()) { + type0 = type; type = ref->realize(ref->deduceTypesFromArgTypes(types)); + } types::Type *ret = type->magicOut("__init__", types); return ret->is(types::Void) ? type : ret; @@ -2046,7 +2048,8 @@ ConstructExpr *ConstructExpr::clone(Generic *ref) { std::vector argsCloned; for (auto *arg : args) argsCloned.push_back(arg->clone(ref)); - SEQ_RETURN_CLONE(new ConstructExpr(type->clone(ref), argsCloned)); + SEQ_RETURN_CLONE( + new ConstructExpr((type0 ? type0 : type)->clone(ref), argsCloned)); } MethodExpr::MethodExpr(Expr *self, Func *func) From b70db8871e2446f94e8835f1755cc4a004f36761 Mon Sep 17 00:00:00 2001 From: jodiew Date: Tue, 31 Dec 2019 19:01:07 -0800 Subject: [PATCH 29/30] Add partial language server for vscode --- seq-vscode/.gitignore | 2 + seq-vscode/README.md | 9 + seq-vscode/client/package.json | 24 + seq-vscode/client/src/extension.ts | 50 + seq-vscode/client/tsconfig.json | 11 + seq-vscode/client/yarn.lock | 628 +++ seq-vscode/language-configuration.json | 51 + seq-vscode/package.json | 80 + seq-vscode/server/package.json | 23 + seq-vscode/server/src/server.ts | 142 + seq-vscode/server/src/wrapper.ts | 17 + seq-vscode/server/tsconfig.json | 13 + seq-vscode/server/yarn.lock | 116 + seq-vscode/syntaxes/seq.tmLanguage.json | 5334 +++++++++++++++++++++++ seq-vscode/tsconfig.json | 20 + seq-vscode/tslint.json | 6 + seq-vscode/yarn.lock | 261 ++ 17 files changed, 6787 insertions(+) create mode 100644 seq-vscode/.gitignore create mode 100644 seq-vscode/README.md create mode 100644 seq-vscode/client/package.json create mode 100644 seq-vscode/client/src/extension.ts create mode 100644 seq-vscode/client/tsconfig.json create mode 100644 seq-vscode/client/yarn.lock create mode 100644 seq-vscode/language-configuration.json create mode 100644 seq-vscode/package.json create mode 100644 seq-vscode/server/package.json create mode 100644 seq-vscode/server/src/server.ts create mode 100644 seq-vscode/server/src/wrapper.ts create mode 100644 seq-vscode/server/tsconfig.json create mode 100644 seq-vscode/server/yarn.lock create mode 100644 seq-vscode/syntaxes/seq.tmLanguage.json create mode 100644 seq-vscode/tsconfig.json create mode 100644 seq-vscode/tslint.json create mode 100644 seq-vscode/yarn.lock diff --git a/seq-vscode/.gitignore b/seq-vscode/.gitignore new file mode 100644 index 000000000..07d225252 --- /dev/null +++ b/seq-vscode/.gitignore @@ -0,0 +1,2 @@ +node_modules +out \ No newline at end of file diff --git a/seq-vscode/README.md b/seq-vscode/README.md new file mode 100644 index 000000000..14f4d73cf --- /dev/null +++ b/seq-vscode/README.md @@ -0,0 +1,9 @@ +# Seq LSP + +## Running the Client and Server + +- Run `yarn` then `yarn run watch` in this folder. This installs all necessary npm modules in both the client and server folder. Then compiles the typescript. +- Open VS Code on this folder. +- Switch to the Debug viewlet. +- Select `Client + Server` from the drop down. +- Run the launch config. diff --git a/seq-vscode/client/package.json b/seq-vscode/client/package.json new file mode 100644 index 000000000..f11584096 --- /dev/null +++ b/seq-vscode/client/package.json @@ -0,0 +1,24 @@ +{ + "name": "lsp-seq-client", + "description": "VSCode part of a language server", + "author": "seq-lang", + "license": "MIT", + "version": "0.0.1", + "repository": { + "type": "git", + "url": "https://github.com/seq-lang/seq" + }, + "engines": { + "vscode": "^1.33.0" + }, + "scripts": { + "update-vscode": "vscode-install", + "postinstall": "vscode-install" + }, + "dependencies": { + "vscode-languageclient": "^5.2.1" + }, + "devDependencies": { + "vscode": "^1.1.35" + } +} diff --git a/seq-vscode/client/src/extension.ts b/seq-vscode/client/src/extension.ts new file mode 100644 index 000000000..a91495e45 --- /dev/null +++ b/seq-vscode/client/src/extension.ts @@ -0,0 +1,50 @@ +import * as path from 'path'; +import { workspace, ExtensionContext } from 'vscode'; + +import { + LanguageClient, + LanguageClientOptions, + ServerOptions, + TransportKind +} from 'vscode-languageclient'; + +let client: LanguageClient; + +export function activate(context: ExtensionContext) { + let serverModule = context.asAbsolutePath( + path.join('server', 'out', 'server.js') + ); + let debugOptions = { execArgv: ['--nolazy', '--inspect=6009'] }; + + let serverOptions: ServerOptions = { + run: { module: serverModule, transport: TransportKind.ipc }, + debug: { + module: serverModule, + transport: TransportKind.ipc, + options: debugOptions + } + }; + + let clientOptions: LanguageClientOptions = { + documentSelector: [{ scheme: 'file', language: 'seq' }], + synchronize: { + fileEvents: workspace.createFileSystemWatcher('**/.clientrc') + } + }; + + client = new LanguageClient( + 'languageServerSeq', + 'Language Server Seq', + serverOptions, + clientOptions + ); + + client.start(); +} + +export function deactivate(): Thenable | undefined { + if (!client) { + return undefined; + } + return client.stop(); +} diff --git a/seq-vscode/client/tsconfig.json b/seq-vscode/client/tsconfig.json new file mode 100644 index 000000000..521307027 --- /dev/null +++ b/seq-vscode/client/tsconfig.json @@ -0,0 +1,11 @@ +{ + "compilerOptions": { + "module": "commonjs", + "target": "es6", + "outDir": "out", + "rootDir": "src", + "sourceMap": true + }, + "include": ["src"], + "exclude": ["node_modules", ".vscode-test"] +} diff --git a/seq-vscode/client/yarn.lock b/seq-vscode/client/yarn.lock new file mode 100644 index 000000000..bf7b735de --- /dev/null +++ b/seq-vscode/client/yarn.lock @@ -0,0 +1,628 @@ +# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. +# yarn lockfile v1 + + +agent-base@4, agent-base@^4.3.0: + version "4.3.0" + resolved "https://registry.yarnpkg.com/agent-base/-/agent-base-4.3.0.tgz#8165f01c436009bccad0b1d122f05ed770efc6ee" + integrity sha512-salcGninV0nPrwpGNn4VTXBb1SOuXQBiqbrNXoeizJsHrsL6ERFM2Ne3JUSBWRE6aeNJI2ROP/WEEIDUiDe3cg== + dependencies: + es6-promisify "^5.0.0" + +ajv@^6.5.5: + version "6.10.2" + resolved "https://registry.yarnpkg.com/ajv/-/ajv-6.10.2.tgz#d3cea04d6b017b2894ad69040fec8b623eb4bd52" + integrity sha512-TXtUUEYHuaTEbLZWIKUr5pmBuhDLy+8KYtPYdcV8qC+pOZL+NKqYwvWSRrVXHn+ZmRRAu8vJTAznH7Oag6RVRw== + dependencies: + fast-deep-equal "^2.0.1" + fast-json-stable-stringify "^2.0.0" + json-schema-traverse "^0.4.1" + uri-js "^4.2.2" + +asn1@~0.2.3: + version "0.2.4" + resolved "https://registry.yarnpkg.com/asn1/-/asn1-0.2.4.tgz#8d2475dfab553bb33e77b54e59e880bb8ce23136" + integrity sha512-jxwzQpLQjSmWXgwaCZE9Nz+glAG01yF1QnWgbhGwHI5A6FRIEY6IVqtHhIepHqI7/kyEyQEagBC5mBEFlIYvdg== + dependencies: + safer-buffer "~2.1.0" + +assert-plus@1.0.0, assert-plus@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/assert-plus/-/assert-plus-1.0.0.tgz#f12e0f3c5d77b0b1cdd9146942e4e96c1e4dd525" + integrity sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU= + +asynckit@^0.4.0: + version "0.4.0" + resolved "https://registry.yarnpkg.com/asynckit/-/asynckit-0.4.0.tgz#c79ed97f7f34cb8f2ba1bc9790bcc366474b4b79" + integrity sha1-x57Zf380y48robyXkLzDZkdLS3k= + +aws-sign2@~0.7.0: + version "0.7.0" + resolved "https://registry.yarnpkg.com/aws-sign2/-/aws-sign2-0.7.0.tgz#b46e890934a9591f2d2f6f86d7e6a9f1b3fe76a8" + integrity sha1-tG6JCTSpWR8tL2+G1+ap8bP+dqg= + +aws4@^1.8.0: + version "1.9.0" + resolved "https://registry.yarnpkg.com/aws4/-/aws4-1.9.0.tgz#24390e6ad61386b0a747265754d2a17219de862c" + integrity sha512-Uvq6hVe90D0B2WEnUqtdgY1bATGz3mw33nH9Y+dmA+w5DHvUmBgkr5rM/KCHpCsiFNRUfokW/szpPPgMK2hm4A== + +balanced-match@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.0.tgz#89b4d199ab2bee49de164ea02b89ce462d71b767" + integrity sha1-ibTRmasr7kneFk6gK4nORi1xt2c= + +bcrypt-pbkdf@^1.0.0: + version "1.0.2" + resolved "https://registry.yarnpkg.com/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz#a4301d389b6a43f9b67ff3ca11a3f6637e360e9e" + integrity sha1-pDAdOJtqQ/m2f/PKEaP2Y342Dp4= + dependencies: + tweetnacl "^0.14.3" + +brace-expansion@^1.1.7: + version "1.1.11" + resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.11.tgz#3c7fcbf529d87226f3d2f52b966ff5271eb441dd" + integrity sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA== + dependencies: + balanced-match "^1.0.0" + concat-map "0.0.1" + +browser-stdout@1.3.1: + version "1.3.1" + resolved "https://registry.yarnpkg.com/browser-stdout/-/browser-stdout-1.3.1.tgz#baa559ee14ced73452229bad7326467c61fabd60" + integrity sha512-qhAVI1+Av2X7qelOfAIYwXONood6XlZE/fXaBSmW/T5SzLAmCgzi+eiWE7fUvbHaeNBQH13UftjpXxsfLkMpgw== + +buffer-from@^1.0.0: + version "1.1.1" + resolved "https://registry.yarnpkg.com/buffer-from/-/buffer-from-1.1.1.tgz#32713bc028f75c02fdb710d7c7bcec1f2c6070ef" + integrity sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A== + +caseless@~0.12.0: + version "0.12.0" + resolved "https://registry.yarnpkg.com/caseless/-/caseless-0.12.0.tgz#1b681c21ff84033c826543090689420d187151dc" + integrity sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw= + +combined-stream@^1.0.6, combined-stream@~1.0.6: + version "1.0.8" + resolved "https://registry.yarnpkg.com/combined-stream/-/combined-stream-1.0.8.tgz#c3d45a8b34fd730631a110a8a2520682b31d5a7f" + integrity sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg== + dependencies: + delayed-stream "~1.0.0" + +commander@2.15.1: + version "2.15.1" + resolved "https://registry.yarnpkg.com/commander/-/commander-2.15.1.tgz#df46e867d0fc2aec66a34662b406a9ccafff5b0f" + integrity sha512-VlfT9F3V0v+jr4yxPc5gg9s62/fIVWsd2Bk2iD435um1NlGMYdVCq+MjcXnhYq2icNOizHr1kK+5TI6H0Hy0ag== + +concat-map@0.0.1: + version "0.0.1" + resolved "https://registry.yarnpkg.com/concat-map/-/concat-map-0.0.1.tgz#d8a96bd77fd68df7793a73036a3ba0d5405d477b" + integrity sha1-2Klr13/Wjfd5OnMDajug1UBdR3s= + +core-util-is@1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.2.tgz#b5fd54220aa2bc5ab57aab7140c940754503c1a7" + integrity sha1-tf1UIgqivFq1eqtxQMlAdUUDwac= + +dashdash@^1.12.0: + version "1.14.1" + resolved "https://registry.yarnpkg.com/dashdash/-/dashdash-1.14.1.tgz#853cfa0f7cbe2fed5de20326b8dd581035f6e2f0" + integrity sha1-hTz6D3y+L+1d4gMmuN1YEDX24vA= + dependencies: + assert-plus "^1.0.0" + +debug@3.1.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/debug/-/debug-3.1.0.tgz#5bb5a0672628b64149566ba16819e61518c67261" + integrity sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g== + dependencies: + ms "2.0.0" + +debug@^3.1.0: + version "3.2.6" + resolved "https://registry.yarnpkg.com/debug/-/debug-3.2.6.tgz#e83d17de16d8a7efb7717edbe5fb10135eee629b" + integrity sha512-mel+jf7nrtEl5Pn1Qx46zARXKDpBbvzezse7p7LqINmdoIk8PYP5SySaxEmYv6TZ0JyEKA1hsCId6DIhgITtWQ== + dependencies: + ms "^2.1.1" + +delayed-stream@~1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/delayed-stream/-/delayed-stream-1.0.0.tgz#df3ae199acadfb7d440aaae0b29e2272b24ec619" + integrity sha1-3zrhmayt+31ECqrgsp4icrJOxhk= + +diff@3.5.0: + version "3.5.0" + resolved "https://registry.yarnpkg.com/diff/-/diff-3.5.0.tgz#800c0dd1e0a8bfbc95835c202ad220fe317e5a12" + integrity sha512-A46qtFgd+g7pDZinpnwiRJtxbC1hpgf0uzP3iG89scHk0AUC7A1TGxf5OiiOUv/JMZR8GOt8hL900hV0bOy5xA== + +ecc-jsbn@~0.1.1: + version "0.1.2" + resolved "https://registry.yarnpkg.com/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz#3a83a904e54353287874c564b7549386849a98c9" + integrity sha1-OoOpBOVDUyh4dMVkt1SThoSamMk= + dependencies: + jsbn "~0.1.0" + safer-buffer "^2.1.0" + +es6-promise@^4.0.3: + version "4.2.8" + resolved "https://registry.yarnpkg.com/es6-promise/-/es6-promise-4.2.8.tgz#4eb21594c972bc40553d276e510539143db53e0a" + integrity sha512-HJDGx5daxeIvxdBxvG2cb9g4tEvwIk3i8+nhX0yGrYmZUzbkdg8QbDevheDB8gd0//uPj4c1EQua8Q+MViT0/w== + +es6-promisify@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/es6-promisify/-/es6-promisify-5.0.0.tgz#5109d62f3e56ea967c4b63505aef08291c8a5203" + integrity sha1-UQnWLz5W6pZ8S2NQWu8IKRyKUgM= + dependencies: + es6-promise "^4.0.3" + +escape-string-regexp@1.0.5: + version "1.0.5" + resolved "https://registry.yarnpkg.com/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz#1b61c0562190a8dff6ae3bb2cf0200ca130b86d4" + integrity sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ= + +extend@~3.0.2: + version "3.0.2" + resolved "https://registry.yarnpkg.com/extend/-/extend-3.0.2.tgz#f8b1136b4071fbd8eb140aff858b1019ec2915fa" + integrity sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g== + +extsprintf@1.3.0: + version "1.3.0" + resolved "https://registry.yarnpkg.com/extsprintf/-/extsprintf-1.3.0.tgz#96918440e3041a7a414f8c52e3c574eb3c3e1e05" + integrity sha1-lpGEQOMEGnpBT4xS48V06zw+HgU= + +extsprintf@^1.2.0: + version "1.4.0" + resolved "https://registry.yarnpkg.com/extsprintf/-/extsprintf-1.4.0.tgz#e2689f8f356fad62cca65a3a91c5df5f9551692f" + integrity sha1-4mifjzVvrWLMplo6kcXfX5VRaS8= + +fast-deep-equal@^2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/fast-deep-equal/-/fast-deep-equal-2.0.1.tgz#7b05218ddf9667bf7f370bf7fdb2cb15fdd0aa49" + integrity sha1-ewUhjd+WZ79/Nwv3/bLLFf3Qqkk= + +fast-json-stable-stringify@^2.0.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz#874bf69c6f404c2b5d99c481341399fd55892633" + integrity sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw== + +forever-agent@~0.6.1: + version "0.6.1" + resolved "https://registry.yarnpkg.com/forever-agent/-/forever-agent-0.6.1.tgz#fbc71f0c41adeb37f96c577ad1ed42d8fdacca91" + integrity sha1-+8cfDEGt6zf5bFd60e1C2P2sypE= + +form-data@~2.3.2: + version "2.3.3" + resolved "https://registry.yarnpkg.com/form-data/-/form-data-2.3.3.tgz#dcce52c05f644f298c6a7ab936bd724ceffbf3a6" + integrity sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ== + dependencies: + asynckit "^0.4.0" + combined-stream "^1.0.6" + mime-types "^2.1.12" + +fs.realpath@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/fs.realpath/-/fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f" + integrity sha1-FQStJSMVjKpA20onh8sBQRmU6k8= + +getpass@^0.1.1: + version "0.1.7" + resolved "https://registry.yarnpkg.com/getpass/-/getpass-0.1.7.tgz#5eff8e3e684d569ae4cb2b1282604e8ba62149fa" + integrity sha1-Xv+OPmhNVprkyysSgmBOi6YhSfo= + dependencies: + assert-plus "^1.0.0" + +glob@7.1.2: + version "7.1.2" + resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.2.tgz#c19c9df9a028702d678612384a6552404c636d15" + integrity sha512-MJTUg1kjuLeQCJ+ccE4Vpa6kKVXkPYJ2mOCQyUuKLcLQsdrMCpBPUi8qVE6+YuaJkozeA9NusTAw3hLr8Xe5EQ== + dependencies: + fs.realpath "^1.0.0" + inflight "^1.0.4" + inherits "2" + minimatch "^3.0.4" + once "^1.3.0" + path-is-absolute "^1.0.0" + +glob@^7.1.2: + version "7.1.6" + resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.6.tgz#141f33b81a7c2492e125594307480c46679278a6" + integrity sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA== + dependencies: + fs.realpath "^1.0.0" + inflight "^1.0.4" + inherits "2" + minimatch "^3.0.4" + once "^1.3.0" + path-is-absolute "^1.0.0" + +growl@1.10.5: + version "1.10.5" + resolved "https://registry.yarnpkg.com/growl/-/growl-1.10.5.tgz#f2735dc2283674fa67478b10181059355c369e5e" + integrity sha512-qBr4OuELkhPenW6goKVXiv47US3clb3/IbuWF9KNKEijAy9oeHxU9IgzjvJhHkUzhaj7rOUD7+YGWqUjLp5oSA== + +har-schema@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/har-schema/-/har-schema-2.0.0.tgz#a94c2224ebcac04782a0d9035521f24735b7ec92" + integrity sha1-qUwiJOvKwEeCoNkDVSHyRzW37JI= + +har-validator@~5.1.0: + version "5.1.3" + resolved "https://registry.yarnpkg.com/har-validator/-/har-validator-5.1.3.tgz#1ef89ebd3e4996557675eed9893110dc350fa080" + integrity sha512-sNvOCzEQNr/qrvJgc3UG/kD4QtlHycrzwS+6mfTrrSq97BvaYcPZZI1ZSqGSPR73Cxn4LKTD4PttRwfU7jWq5g== + dependencies: + ajv "^6.5.5" + har-schema "^2.0.0" + +has-flag@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/has-flag/-/has-flag-3.0.0.tgz#b5d454dc2199ae225699f3467e5a07f3b955bafd" + integrity sha1-tdRU3CGZriJWmfNGfloH87lVuv0= + +he@1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/he/-/he-1.1.1.tgz#93410fd21b009735151f8868c2f271f3427e23fd" + integrity sha1-k0EP0hsAlzUVH4howvJx80J+I/0= + +http-proxy-agent@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/http-proxy-agent/-/http-proxy-agent-2.1.0.tgz#e4821beef5b2142a2026bd73926fe537631c5405" + integrity sha512-qwHbBLV7WviBl0rQsOzH6o5lwyOIvwp/BdFnvVxXORldu5TmjFfjzBcWUWS5kWAZhmv+JtiDhSuQCp4sBfbIgg== + dependencies: + agent-base "4" + debug "3.1.0" + +http-signature@~1.2.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/http-signature/-/http-signature-1.2.0.tgz#9aecd925114772f3d95b65a60abb8f7c18fbace1" + integrity sha1-muzZJRFHcvPZW2WmCruPfBj7rOE= + dependencies: + assert-plus "^1.0.0" + jsprim "^1.2.2" + sshpk "^1.7.0" + +https-proxy-agent@^2.2.1: + version "2.2.4" + resolved "https://registry.yarnpkg.com/https-proxy-agent/-/https-proxy-agent-2.2.4.tgz#4ee7a737abd92678a293d9b34a1af4d0d08c787b" + integrity sha512-OmvfoQ53WLjtA9HeYP9RNrWMJzzAz1JGaSFr1nijg0PVR1JaD/xbJq1mdEIIlxGpXp9eSe/O2LgU9DJmTPd0Eg== + dependencies: + agent-base "^4.3.0" + debug "^3.1.0" + +inflight@^1.0.4: + version "1.0.6" + resolved "https://registry.yarnpkg.com/inflight/-/inflight-1.0.6.tgz#49bd6331d7d02d0c09bc910a1075ba8165b56df9" + integrity sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk= + dependencies: + once "^1.3.0" + wrappy "1" + +inherits@2: + version "2.0.4" + resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.4.tgz#0fa2c64f932917c3433a0ded55363aae37416b7c" + integrity sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ== + +is-typedarray@~1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/is-typedarray/-/is-typedarray-1.0.0.tgz#e479c80858df0c1b11ddda6940f96011fcda4a9a" + integrity sha1-5HnICFjfDBsR3dppQPlgEfzaSpo= + +isstream@~0.1.2: + version "0.1.2" + resolved "https://registry.yarnpkg.com/isstream/-/isstream-0.1.2.tgz#47e63f7af55afa6f92e1500e690eb8b8529c099a" + integrity sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo= + +jsbn@~0.1.0: + version "0.1.1" + resolved "https://registry.yarnpkg.com/jsbn/-/jsbn-0.1.1.tgz#a5e654c2e5a2deb5f201d96cefbca80c0ef2f513" + integrity sha1-peZUwuWi3rXyAdls77yoDA7y9RM= + +json-schema-traverse@^0.4.1: + version "0.4.1" + resolved "https://registry.yarnpkg.com/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz#69f6a87d9513ab8bb8fe63bdb0979c448e684660" + integrity sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg== + +json-schema@0.2.3: + version "0.2.3" + resolved "https://registry.yarnpkg.com/json-schema/-/json-schema-0.2.3.tgz#b480c892e59a2f05954ce727bd3f2a4e882f9e13" + integrity sha1-tIDIkuWaLwWVTOcnvT8qTogvnhM= + +json-stringify-safe@~5.0.1: + version "5.0.1" + resolved "https://registry.yarnpkg.com/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz#1296a2d58fd45f19a0f6ce01d65701e2c735b6eb" + integrity sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus= + +jsprim@^1.2.2: + version "1.4.1" + resolved "https://registry.yarnpkg.com/jsprim/-/jsprim-1.4.1.tgz#313e66bc1e5cc06e438bc1b7499c2e5c56acb6a2" + integrity sha1-MT5mvB5cwG5Di8G3SZwuXFastqI= + dependencies: + assert-plus "1.0.0" + extsprintf "1.3.0" + json-schema "0.2.3" + verror "1.10.0" + +mime-db@1.42.0: + version "1.42.0" + resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.42.0.tgz#3e252907b4c7adb906597b4b65636272cf9e7bac" + integrity sha512-UbfJCR4UAVRNgMpfImz05smAXK7+c+ZntjaA26ANtkXLlOe947Aag5zdIcKQULAiF9Cq4WxBi9jUs5zkA84bYQ== + +mime-types@^2.1.12, mime-types@~2.1.19: + version "2.1.25" + resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.25.tgz#39772d46621f93e2a80a856c53b86a62156a6437" + integrity sha512-5KhStqB5xpTAeGqKBAMgwaYMnQik7teQN4IAzC7npDv6kzeU6prfkR67bc87J1kWMPGkoaZSq1npmexMgkmEVg== + dependencies: + mime-db "1.42.0" + +minimatch@3.0.4, minimatch@^3.0.4: + version "3.0.4" + resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.0.4.tgz#5166e286457f03306064be5497e8dbb0c3d32083" + integrity sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA== + dependencies: + brace-expansion "^1.1.7" + +minimist@0.0.8: + version "0.0.8" + resolved "https://registry.yarnpkg.com/minimist/-/minimist-0.0.8.tgz#857fcabfc3397d2625b8228262e86aa7a011b05d" + integrity sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0= + +mkdirp@0.5.1: + version "0.5.1" + resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.5.1.tgz#30057438eac6cf7f8c4767f38648d6697d75c903" + integrity sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM= + dependencies: + minimist "0.0.8" + +mocha@^5.2.0: + version "5.2.0" + resolved "https://registry.yarnpkg.com/mocha/-/mocha-5.2.0.tgz#6d8ae508f59167f940f2b5b3c4a612ae50c90ae6" + integrity sha512-2IUgKDhc3J7Uug+FxMXuqIyYzH7gJjXECKe/w43IGgQHTSj3InJi+yAA7T24L9bQMRKiUEHxEX37G5JpVUGLcQ== + dependencies: + browser-stdout "1.3.1" + commander "2.15.1" + debug "3.1.0" + diff "3.5.0" + escape-string-regexp "1.0.5" + glob "7.1.2" + growl "1.10.5" + he "1.1.1" + minimatch "3.0.4" + mkdirp "0.5.1" + supports-color "5.4.0" + +ms@2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/ms/-/ms-2.0.0.tgz#5608aeadfc00be6c2901df5f9861788de0d597c8" + integrity sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g= + +ms@^2.1.1: + version "2.1.2" + resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.2.tgz#d09d1f357b443f493382a8eb3ccd183872ae6009" + integrity sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w== + +oauth-sign@~0.9.0: + version "0.9.0" + resolved "https://registry.yarnpkg.com/oauth-sign/-/oauth-sign-0.9.0.tgz#47a7b016baa68b5fa0ecf3dee08a85c679ac6455" + integrity sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ== + +once@^1.3.0: + version "1.4.0" + resolved "https://registry.yarnpkg.com/once/-/once-1.4.0.tgz#583b1aa775961d4b113ac17d9c50baef9dd76bd1" + integrity sha1-WDsap3WWHUsROsF9nFC6753Xa9E= + dependencies: + wrappy "1" + +path-is-absolute@^1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/path-is-absolute/-/path-is-absolute-1.0.1.tgz#174b9268735534ffbc7ace6bf53a5a9e1b5c5f5f" + integrity sha1-F0uSaHNVNP+8es5r9TpanhtcX18= + +performance-now@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/performance-now/-/performance-now-2.1.0.tgz#6309f4e0e5fa913ec1c69307ae364b4b377c9e7b" + integrity sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns= + +psl@^1.1.24: + version "1.7.0" + resolved "https://registry.yarnpkg.com/psl/-/psl-1.7.0.tgz#f1c4c47a8ef97167dea5d6bbf4816d736e884a3c" + integrity sha512-5NsSEDv8zY70ScRnOTn7bK7eanl2MvFrOrS/R6x+dBt5g1ghnj9Zv90kO8GwT8gxcu2ANyFprnFYB85IogIJOQ== + +punycode@^1.4.1: + version "1.4.1" + resolved "https://registry.yarnpkg.com/punycode/-/punycode-1.4.1.tgz#c0d5a63b2718800ad8e1eb0fa5269c84dd41845e" + integrity sha1-wNWmOycYgArY4esPpSachN1BhF4= + +punycode@^2.1.0: + version "2.1.1" + resolved "https://registry.yarnpkg.com/punycode/-/punycode-2.1.1.tgz#b58b010ac40c22c5657616c8d2c2c02c7bf479ec" + integrity sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A== + +qs@~6.5.2: + version "6.5.2" + resolved "https://registry.yarnpkg.com/qs/-/qs-6.5.2.tgz#cb3ae806e8740444584ef154ce8ee98d403f3e36" + integrity sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA== + +querystringify@^2.1.1: + version "2.1.1" + resolved "https://registry.yarnpkg.com/querystringify/-/querystringify-2.1.1.tgz#60e5a5fd64a7f8bfa4d2ab2ed6fdf4c85bad154e" + integrity sha512-w7fLxIRCRT7U8Qu53jQnJyPkYZIaR4n5151KMfcJlO/A9397Wxb1amJvROTK6TOnp7PfoAmg/qXiNHI+08jRfA== + +request@^2.88.0: + version "2.88.0" + resolved "https://registry.yarnpkg.com/request/-/request-2.88.0.tgz#9c2fca4f7d35b592efe57c7f0a55e81052124fef" + integrity sha512-NAqBSrijGLZdM0WZNsInLJpkJokL72XYjUpnB0iwsRgxh7dB6COrHnTBNwN0E+lHDAJzu7kLAkDeY08z2/A0hg== + dependencies: + aws-sign2 "~0.7.0" + aws4 "^1.8.0" + caseless "~0.12.0" + combined-stream "~1.0.6" + extend "~3.0.2" + forever-agent "~0.6.1" + form-data "~2.3.2" + har-validator "~5.1.0" + http-signature "~1.2.0" + is-typedarray "~1.0.0" + isstream "~0.1.2" + json-stringify-safe "~5.0.1" + mime-types "~2.1.19" + oauth-sign "~0.9.0" + performance-now "^2.1.0" + qs "~6.5.2" + safe-buffer "^5.1.2" + tough-cookie "~2.4.3" + tunnel-agent "^0.6.0" + uuid "^3.3.2" + +requires-port@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/requires-port/-/requires-port-1.0.0.tgz#925d2601d39ac485e091cf0da5c6e694dc3dcaff" + integrity sha1-kl0mAdOaxIXgkc8NpcbmlNw9yv8= + +safe-buffer@^5.0.1, safe-buffer@^5.1.2: + version "5.2.0" + resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.2.0.tgz#b74daec49b1148f88c64b68d49b1e815c1f2f519" + integrity sha512-fZEwUGbVl7kouZs1jCdMLdt95hdIv0ZeHg6L7qPeciMZhZ+/gdesW4wgTARkrFWEpspjEATAzUGPG8N2jJiwbg== + +safer-buffer@^2.0.2, safer-buffer@^2.1.0, safer-buffer@~2.1.0: + version "2.1.2" + resolved "https://registry.yarnpkg.com/safer-buffer/-/safer-buffer-2.1.2.tgz#44fa161b0187b9549dd84bb91802f9bd8385cd6a" + integrity sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg== + +semver@^5.4.1, semver@^5.5.0: + version "5.7.1" + resolved "https://registry.yarnpkg.com/semver/-/semver-5.7.1.tgz#a954f931aeba508d307bbf069eff0c01c96116f7" + integrity sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ== + +source-map-support@^0.5.0: + version "0.5.16" + resolved "https://registry.yarnpkg.com/source-map-support/-/source-map-support-0.5.16.tgz#0ae069e7fe3ba7538c64c98515e35339eac5a042" + integrity sha512-efyLRJDr68D9hBBNIPWFjhpFzURh+KJykQwvMyW5UiZzYwoF6l4YMMDIJJEyFWxWCqfyxLzz6tSfUFR+kXXsVQ== + dependencies: + buffer-from "^1.0.0" + source-map "^0.6.0" + +source-map@^0.6.0: + version "0.6.1" + resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.6.1.tgz#74722af32e9614e9c287a8d0bbde48b5e2f1a263" + integrity sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g== + +sshpk@^1.7.0: + version "1.16.1" + resolved "https://registry.yarnpkg.com/sshpk/-/sshpk-1.16.1.tgz#fb661c0bef29b39db40769ee39fa70093d6f6877" + integrity sha512-HXXqVUq7+pcKeLqqZj6mHFUMvXtOJt1uoUx09pFW6011inTMxqI8BA8PM95myrIyyKwdnzjdFjLiE6KBPVtJIg== + dependencies: + asn1 "~0.2.3" + assert-plus "^1.0.0" + bcrypt-pbkdf "^1.0.0" + dashdash "^1.12.0" + ecc-jsbn "~0.1.1" + getpass "^0.1.1" + jsbn "~0.1.0" + safer-buffer "^2.0.2" + tweetnacl "~0.14.0" + +supports-color@5.4.0: + version "5.4.0" + resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-5.4.0.tgz#1c6b337402c2137605efe19f10fec390f6faab54" + integrity sha512-zjaXglF5nnWpsq470jSv6P9DwPvgLkuapYmfDm3JWOm0vkNTVF2tI4UrN2r6jH1qM/uc/WtxYY1hYoA2dOKj5w== + dependencies: + has-flag "^3.0.0" + +tough-cookie@~2.4.3: + version "2.4.3" + resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-2.4.3.tgz#53f36da3f47783b0925afa06ff9f3b165280f781" + integrity sha512-Q5srk/4vDM54WJsJio3XNn6K2sCG+CQ8G5Wz6bZhRZoAe/+TxjWB/GlFAnYEbkYVlON9FMk/fE3h2RLpPXo4lQ== + dependencies: + psl "^1.1.24" + punycode "^1.4.1" + +tunnel-agent@^0.6.0: + version "0.6.0" + resolved "https://registry.yarnpkg.com/tunnel-agent/-/tunnel-agent-0.6.0.tgz#27a5dea06b36b04a0a9966774b290868f0fc40fd" + integrity sha1-J6XeoGs2sEoKmWZ3SykIaPD8QP0= + dependencies: + safe-buffer "^5.0.1" + +tweetnacl@^0.14.3, tweetnacl@~0.14.0: + version "0.14.5" + resolved "https://registry.yarnpkg.com/tweetnacl/-/tweetnacl-0.14.5.tgz#5ae68177f192d4456269d108afa93ff8743f4f64" + integrity sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q= + +uri-js@^4.2.2: + version "4.2.2" + resolved "https://registry.yarnpkg.com/uri-js/-/uri-js-4.2.2.tgz#94c540e1ff772956e2299507c010aea6c8838eb0" + integrity sha512-KY9Frmirql91X2Qgjry0Wd4Y+YTdrdZheS8TFwvkbLWf/G5KNJDCh6pKL5OZctEW4+0Baa5idK2ZQuELRwPznQ== + dependencies: + punycode "^2.1.0" + +url-parse@^1.4.4: + version "1.4.7" + resolved "https://registry.yarnpkg.com/url-parse/-/url-parse-1.4.7.tgz#a8a83535e8c00a316e403a5db4ac1b9b853ae278" + integrity sha512-d3uaVyzDB9tQoSXFvuSUNFibTd9zxd2bkVrDRvF5TmvWWQwqE4lgYJ5m+x1DbecWkw+LK4RNl2CU1hHuOKPVlg== + dependencies: + querystringify "^2.1.1" + requires-port "^1.0.0" + +uuid@^3.3.2: + version "3.3.3" + resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.3.3.tgz#4568f0216e78760ee1dbf3a4d2cf53e224112866" + integrity sha512-pW0No1RGHgzlpHJO1nsVrHKpOEIxkGg1xB+v0ZmdNH5OAeAwzAVrCnI2/6Mtx+Uys6iaylxa+D3g4j63IKKjSQ== + +verror@1.10.0: + version "1.10.0" + resolved "https://registry.yarnpkg.com/verror/-/verror-1.10.0.tgz#3a105ca17053af55d6e270c1f8288682e18da400" + integrity sha1-OhBcoXBTr1XW4nDB+CiGguGNpAA= + dependencies: + assert-plus "^1.0.0" + core-util-is "1.0.2" + extsprintf "^1.2.0" + +vscode-jsonrpc@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/vscode-jsonrpc/-/vscode-jsonrpc-4.0.0.tgz#a7bf74ef3254d0a0c272fab15c82128e378b3be9" + integrity sha512-perEnXQdQOJMTDFNv+UF3h1Y0z4iSiaN9jIlb0OqIYgosPCZGYh/MCUlkFtV2668PL69lRDO32hmvL2yiidUYg== + +vscode-languageclient@^5.2.1: + version "5.2.1" + resolved "https://registry.yarnpkg.com/vscode-languageclient/-/vscode-languageclient-5.2.1.tgz#7cfc83a294c409f58cfa2b910a8cfeaad0397193" + integrity sha512-7jrS/9WnV0ruqPamN1nE7qCxn0phkH5LjSgSp9h6qoJGoeAKzwKz/PF6M+iGA/aklx4GLZg1prddhEPQtuXI1Q== + dependencies: + semver "^5.5.0" + vscode-languageserver-protocol "3.14.1" + +vscode-languageserver-protocol@3.14.1: + version "3.14.1" + resolved "https://registry.yarnpkg.com/vscode-languageserver-protocol/-/vscode-languageserver-protocol-3.14.1.tgz#b8aab6afae2849c84a8983d39a1cf742417afe2f" + integrity sha512-IL66BLb2g20uIKog5Y2dQ0IiigW0XKrvmWiOvc0yXw80z3tMEzEnHjaGAb3ENuU7MnQqgnYJ1Cl2l9RvNgDi4g== + dependencies: + vscode-jsonrpc "^4.0.0" + vscode-languageserver-types "3.14.0" + +vscode-languageserver-types@3.14.0: + version "3.14.0" + resolved "https://registry.yarnpkg.com/vscode-languageserver-types/-/vscode-languageserver-types-3.14.0.tgz#d3b5952246d30e5241592b6dde8280e03942e743" + integrity sha512-lTmS6AlAlMHOvPQemVwo3CezxBp0sNB95KNPkqp3Nxd5VFEnuG1ByM0zlRWos0zjO3ZWtkvhal0COgiV1xIA4A== + +vscode-test@^0.4.1: + version "0.4.3" + resolved "https://registry.yarnpkg.com/vscode-test/-/vscode-test-0.4.3.tgz#461ebf25fc4bc93d77d982aed556658a2e2b90b8" + integrity sha512-EkMGqBSefZH2MgW65nY05rdRSko15uvzq4VAPM5jVmwYuFQKE7eikKXNJDRxL+OITXHB6pI+a3XqqD32Y3KC5w== + dependencies: + http-proxy-agent "^2.1.0" + https-proxy-agent "^2.2.1" + +vscode@^1.1.35: + version "1.1.36" + resolved "https://registry.yarnpkg.com/vscode/-/vscode-1.1.36.tgz#5e1a0d1bf4977d0c7bc5159a9a13d5b104d4b1b6" + integrity sha512-cGFh9jmGLcTapCpPCKvn8aG/j9zVQ+0x5hzYJq5h5YyUXVGa1iamOaB2M2PZXoumQPES4qeAP1FwkI0b6tL4bQ== + dependencies: + glob "^7.1.2" + mocha "^5.2.0" + request "^2.88.0" + semver "^5.4.1" + source-map-support "^0.5.0" + url-parse "^1.4.4" + vscode-test "^0.4.1" + +wrappy@1: + version "1.0.2" + resolved "https://registry.yarnpkg.com/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f" + integrity sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8= diff --git a/seq-vscode/language-configuration.json b/seq-vscode/language-configuration.json new file mode 100644 index 000000000..4567e504c --- /dev/null +++ b/seq-vscode/language-configuration.json @@ -0,0 +1,51 @@ +{ + "comments": { + "blockComment": [ + "'''", + "'''" + ], + "lineComment": "#" + }, + "brackets": [ + [ + "(", + ")" + ], + [ + "[", + "]" + ], + [ + "{", + "}" + ] + ], + "autoClosingPairs": [ + [ + "(", + ")" + ], + [ + "[", + "]" + ], + [ + "{", + "}" + ] + ], + "surroundingPairs": [ + [ + "(", + ")" + ], + [ + "[", + "]" + ], + [ + "{", + "}" + ] + ] +} \ No newline at end of file diff --git a/seq-vscode/package.json b/seq-vscode/package.json new file mode 100644 index 000000000..484184a50 --- /dev/null +++ b/seq-vscode/package.json @@ -0,0 +1,80 @@ +{ + "name": "seq-lang", + "description": "A language server for seq", + "author": "seq-lang", + "license": "MIT", + "version": "0.0.1", + "repository": { + "type": "git", + "url": "https://github.com/seq-lang/seq" + }, + "categories": [], + "keywords": [ + "multi-root ready" + ], + "engines": { + "vscode": "^1.33.0" + }, + "activationEvents": [ + "onLanguage:seq" + ], + "main": "./client/out/extension", + "contributes": { + "languages": [ + { + "id": "seq", + "aliases": [ + "Seq", + "seq-lang" + ], + "extensions": [ + ".seq" + ], + "configuration": "./language-configuration.json" + } + ], + "grammars": [ + { + "language": "seq", + "scopeName": "source.seq", + "path": "./syntaxes/seq.tmLanguage.json" + } + ], + "configuration": { + "type": "object", + "title": "Seq configuration", + "properties": { + "languageServerSeq.maxNumberOfProblems": { + "scope": "resource", + "type": "number", + "default": 100, + "description": "Controls the maximum number of problems produced by the server." + }, + "languageServerSeq.trace.server": { + "scope": "window", + "type": "string", + "enum": [ + "off", + "messages", + "verbose" + ], + "default": "off", + "description": "Traces the communication between VS Code and the language server." + } + } + } + }, + "scripts": { + "vscode:prepublish": "cd client && yarn run update-vscode && cd .. && yarn run compile", + "compile": "tsc -b", + "watch": "tsc -b -w", + "postinstall": "cd client && yarn && cd ../server && yarn && cd .." + }, + "devDependencies": { + "@types/mocha": "^5.2.7", + "@types/node": "^12.11.7", + "tslint": "^5.16.0", + "typescript": "^3.6.4" + }, + "dependencies": {} +} diff --git a/seq-vscode/server/package.json b/seq-vscode/server/package.json new file mode 100644 index 000000000..4e4350db3 --- /dev/null +++ b/seq-vscode/server/package.json @@ -0,0 +1,23 @@ +{ + "name": "lsp-seq-server", + "description": "The seq language server.", + "version": "0.0.1", + "author": "seq-lang", + "license": "MIT", + "engines": { + "node": "*" + }, + "repository": { + "type": "git", + "url": "https://github.com/seq-lang/seq" + }, + "dependencies": { + "@types/ffi": "^0.2.2", + "@types/ref": "^0.0.28", + "bindings": "^1.5.0", + "ffi": "lxe/node-ffi#node-12", + "ref": "lxe/ref#node-12", + "vscode-languageserver": "^5.2.1" + }, + "scripts": {} +} diff --git a/seq-vscode/server/src/server.ts b/seq-vscode/server/src/server.ts new file mode 100644 index 000000000..9f3454ad8 --- /dev/null +++ b/seq-vscode/server/src/server.ts @@ -0,0 +1,142 @@ +import { + createConnection, + TextDocuments, + InitializeParams, + DidChangeConfigurationNotification, + CompletionItem, + TextDocumentPositionParams, + Hover +} from 'vscode-languageserver'; + +import SeqWrapper from './wrapper'; + +let connection = createConnection(); + +let wrapper: SeqWrapper = new SeqWrapper(); + +let documents: TextDocuments = new TextDocuments(); + +let hasConfigurationCapability: boolean = false; +let hasWorkspaceFolderCapability: boolean = false; + +connection.onInitialize((params: InitializeParams) => { + let capabilities = params.capabilities; + + hasConfigurationCapability = !!( + capabilities.workspace && !!capabilities.workspace.configuration + ); + hasWorkspaceFolderCapability = !!( + capabilities.workspace && !!capabilities.workspace.workspaceFolders + ); + + return { + capabilities: { + textDocumentSync: documents.syncKind, + completionProvider: { + resolveProvider: true + }, + hoverProvider: true + } + }; +}); + +connection.onInitialized(() => { + if (hasConfigurationCapability) { + connection.client.register(DidChangeConfigurationNotification.type, undefined); + } + if (hasWorkspaceFolderCapability) { + connection.workspace.onDidChangeWorkspaceFolders(_event => { + connection.console.log('Workspace folder change event received.'); + }); + } +}); + +interface ExampleSettings { + maxNumberOfProblems: number; +} + +const defaultSettings: ExampleSettings = { maxNumberOfProblems: 1000 }; +let globalSettings: ExampleSettings = defaultSettings; + +let documentSettings: Map> = new Map(); + +connection.onDidChangeConfiguration(change => { + if (hasConfigurationCapability) { + documentSettings.clear(); + } else { + globalSettings = ( + (change.settings.languageServerSeq || defaultSettings) + ); + } +}); + +documents.onDidClose(e => { + documentSettings.delete(e.document.uri); +}); + +connection.onDidChangeWatchedFiles(_change => { + connection.console.log('We received an file change event'); +}); + +connection.onCompletion( + (_textDocumentPosition: TextDocumentPositionParams): CompletionItem[] => { + connection.console.log(`onCompletion ${_textDocumentPosition.textDocument.uri} ${_textDocumentPosition.position.line} ${_textDocumentPosition.position.character}`); + const prefix = documents.get(_textDocumentPosition.textDocument.uri)?.getText({ + start: { line: _textDocumentPosition.position.line, character: 0 }, + end: { line: _textDocumentPosition.position.line, character: _textDocumentPosition.position.character } + }); + if (!prefix) { + return []; + } + connection.console.log(`prefix ${prefix}`); + // TODO: Use the seq wrapper to get the list of completion items. + return []; + } +); + +connection.onCompletionResolve( + (item: CompletionItem): CompletionItem => { + connection.console.log(`onCompletionResolve ${item}`); + return item; + } +); + +connection.onHover( + (_textDocumentPosition: TextDocumentPositionParams): Hover => { + const prefix = documents.get(_textDocumentPosition.textDocument.uri)?.getText({ + start: { line: _textDocumentPosition.position.line, character: 0 }, + end: { line: _textDocumentPosition.position.line, character: _textDocumentPosition.position.character } + }); + if (!prefix) { + return { + contents: { + kind: 'plaintext', + value: '' + } + }; + } + connection.console.log(`prefix ${prefix}`); + // TODO: Use the seq wrapper to get the hover text. + return { + contents: { + kind: 'plaintext', + value: '' + } + }; + } +) + +connection.onDidOpenTextDocument((params) => { + connection.console.log(`${params.textDocument.uri} opened.`); +}); +connection.onDidChangeTextDocument((params) => { + connection.console.log(`${params.textDocument.uri} changed: ${JSON.stringify(params.contentChanges)}`); +}); +connection.onDidCloseTextDocument((params) => { + connection.console.log(`${params.textDocument.uri} closed.`); +}); + + +documents.listen(connection); + +connection.listen(); diff --git a/seq-vscode/server/src/wrapper.ts b/seq-vscode/server/src/wrapper.ts new file mode 100644 index 000000000..f190c57da --- /dev/null +++ b/seq-vscode/server/src/wrapper.ts @@ -0,0 +1,17 @@ +import { ForeignFunction, DynamicLibrary, RTLD_GLOBAL } from 'ffi'; +import { refType, types } from 'ref'; + +export default class SeqWrapper { + // private handle: ForeignFunction; + constructor() { + // const dl = new DynamicLibrary('/path/to/libseqjit.so', RTLD_GLOBAL); + // const init_fn = new ForeignFunction(dl.get('seq_jit_init'), refType(types.void), []); + // this.handle = init_fn(); + } + complete(prefix: string): string[] { + return []; + } + document(idn: string): string { + return ''; + } +} diff --git a/seq-vscode/server/tsconfig.json b/seq-vscode/server/tsconfig.json new file mode 100644 index 000000000..742431d80 --- /dev/null +++ b/seq-vscode/server/tsconfig.json @@ -0,0 +1,13 @@ +{ + "compilerOptions": { + "target": "es6", + "module": "commonjs", + "moduleResolution": "node", + "sourceMap": true, + "strict": true, + "outDir": "out", + "rootDir": "src" + }, + "include": ["src"], + "exclude": ["node_modules", ".vscode-test"] +} diff --git a/seq-vscode/server/yarn.lock b/seq-vscode/server/yarn.lock new file mode 100644 index 000000000..7312b3808 --- /dev/null +++ b/seq-vscode/server/yarn.lock @@ -0,0 +1,116 @@ +# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. +# yarn lockfile v1 + + +"@types/ffi@^0.2.2": + version "0.2.2" + resolved "https://registry.yarnpkg.com/@types/ffi/-/ffi-0.2.2.tgz#a5d78682ecd01abc8759042fe25e5891b223ea15" + integrity sha512-bhGEZ9NeeXGi7zYxaBReGYePKmjeKhI2bUCHze42Jn0NxOwtaP2aTMTo4dpJw3eU6V4zHVyEk4w4TSH+OLkV4w== + dependencies: + "@types/node" "*" + "@types/ref" "*" + "@types/ref-struct" "*" + +"@types/node@*": + version "13.1.2" + resolved "https://registry.yarnpkg.com/@types/node/-/node-13.1.2.tgz#fe94285bf5e0782e1a9e5a8c482b1c34465fa385" + integrity sha512-B8emQA1qeKerqd1dmIsQYnXi+mmAzTB7flExjmy5X1aVAKFNNNDubkavwR13kR6JnpeLp3aLoJhwn9trWPAyFQ== + +"@types/ref-struct@*": + version "0.0.29" + resolved "https://registry.yarnpkg.com/@types/ref-struct/-/ref-struct-0.0.29.tgz#a2d7c113a43d1182be99a22dfca8bf94bae97afa" + integrity sha512-ZOiybExNDG++Rjw7qNAlP8Z9TGe4W1S7UvqHbCYzFMc2cVBhTuA29tzAkOrk3bqY0CdzbOvmU7A5X2VOjTpCkw== + dependencies: + "@types/ref" "*" + +"@types/ref@*", "@types/ref@^0.0.28": + version "0.0.28" + resolved "https://registry.yarnpkg.com/@types/ref/-/ref-0.0.28.tgz#15a61253ed1259038b47499de1c9b0cbca57f55c" + integrity sha1-FaYSU+0SWQOLR0md4cmwy8pX9Vw= + dependencies: + "@types/node" "*" + +bindings@1, bindings@^1.5.0, bindings@~1.5.0: + version "1.5.0" + resolved "https://registry.yarnpkg.com/bindings/-/bindings-1.5.0.tgz#10353c9e945334bc0511a6d90b38fbc7c9c504df" + integrity sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ== + dependencies: + file-uri-to-path "1.0.0" + +debug@4: + version "4.1.1" + resolved "https://registry.yarnpkg.com/debug/-/debug-4.1.1.tgz#3b72260255109c6b589cee050f1d516139664791" + integrity sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw== + dependencies: + ms "^2.1.1" + +ffi@lxe/node-ffi#node-12: + version "3.0.0" + resolved "https://codeload.github.com/lxe/node-ffi/tar.gz/f841a7e11995b61feae38f16e61f25c5a39ecb7e" + dependencies: + bindings "~1.5.0" + debug "4" + nan "^2.13.2" + ref lxe/ref#node-12 + ref-struct lxe/ref-struct#node-12 + +file-uri-to-path@1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz#553a7b8446ff6f684359c445f1e37a05dacc33dd" + integrity sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw== + +ms@^2.1.1: + version "2.1.2" + resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.2.tgz#d09d1f357b443f493382a8eb3ccd183872ae6009" + integrity sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w== + +nan@2, nan@^2.13.2: + version "2.14.0" + resolved "https://registry.yarnpkg.com/nan/-/nan-2.14.0.tgz#7818f722027b2459a86f0295d434d1fc2336c52c" + integrity sha512-INOFj37C7k3AfaNTtX8RhsTw7qRy7eLET14cROi9+5HAVbbHuIWUHEauBv5qT4Av2tWasiTY1Jw6puUNqRJXQg== + +ref-struct@lxe/ref-struct#node-12: + version "2.0.0" + resolved "https://codeload.github.com/lxe/ref-struct/tar.gz/60d0ed18fb968860c06e14248ba92d22b22ae55b" + dependencies: + debug "4" + ref lxe/ref#node-12 + +ref@lxe/ref#node-12: + version "2.0.0" + resolved "https://codeload.github.com/lxe/ref/tar.gz/9379bf23a9ff9a628e989c2fb0b89cd8f3a2e116" + dependencies: + bindings "1" + debug "4" + nan "2" + +vscode-jsonrpc@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/vscode-jsonrpc/-/vscode-jsonrpc-4.0.0.tgz#a7bf74ef3254d0a0c272fab15c82128e378b3be9" + integrity sha512-perEnXQdQOJMTDFNv+UF3h1Y0z4iSiaN9jIlb0OqIYgosPCZGYh/MCUlkFtV2668PL69lRDO32hmvL2yiidUYg== + +vscode-languageserver-protocol@3.14.1: + version "3.14.1" + resolved "https://registry.yarnpkg.com/vscode-languageserver-protocol/-/vscode-languageserver-protocol-3.14.1.tgz#b8aab6afae2849c84a8983d39a1cf742417afe2f" + integrity sha512-IL66BLb2g20uIKog5Y2dQ0IiigW0XKrvmWiOvc0yXw80z3tMEzEnHjaGAb3ENuU7MnQqgnYJ1Cl2l9RvNgDi4g== + dependencies: + vscode-jsonrpc "^4.0.0" + vscode-languageserver-types "3.14.0" + +vscode-languageserver-types@3.14.0: + version "3.14.0" + resolved "https://registry.yarnpkg.com/vscode-languageserver-types/-/vscode-languageserver-types-3.14.0.tgz#d3b5952246d30e5241592b6dde8280e03942e743" + integrity sha512-lTmS6AlAlMHOvPQemVwo3CezxBp0sNB95KNPkqp3Nxd5VFEnuG1ByM0zlRWos0zjO3ZWtkvhal0COgiV1xIA4A== + +vscode-languageserver@^5.2.1: + version "5.2.1" + resolved "https://registry.yarnpkg.com/vscode-languageserver/-/vscode-languageserver-5.2.1.tgz#0d2feddd33f92aadf5da32450df498d52f6f14eb" + integrity sha512-GuayqdKZqAwwaCUjDvMTAVRPJOp/SLON3mJ07eGsx/Iq9HjRymhKWztX41rISqDKhHVVyFM+IywICyZDla6U3A== + dependencies: + vscode-languageserver-protocol "3.14.1" + vscode-uri "^1.0.6" + +vscode-uri@^1.0.6: + version "1.0.8" + resolved "https://registry.yarnpkg.com/vscode-uri/-/vscode-uri-1.0.8.tgz#9769aaececae4026fb6e22359cb38946580ded59" + integrity sha512-obtSWTlbJ+a+TFRYGaUumtVwb+InIUVI0Lu0VBUAPmj2cU5JutEXg3xUE0c2J5Tcy7h2DEKVJBFi+Y9ZSFzzPQ== diff --git a/seq-vscode/syntaxes/seq.tmLanguage.json b/seq-vscode/syntaxes/seq.tmLanguage.json new file mode 100644 index 000000000..0b3d75152 --- /dev/null +++ b/seq-vscode/syntaxes/seq.tmLanguage.json @@ -0,0 +1,5334 @@ +{ + "$schema": "https://raw.githubusercontent.com/martinring/tmlanguage/master/tmlanguage.json", + "name": "Seq", + "scopeName": "source.seq", + "patterns": [ + { + "include": "#statement" + }, + { + "include": "#expression" + } + ], + "repository": { + "impossible": { + "comment": "This is a special rule that should be used where no match is desired. It is not a good idea to match something like '1{0}' because in some cases that can result in infinite loops in token generation. So the rule instead matches and impossible expression to allow a match to fail and move to the next token.", + "match": "$.^" + }, + "statement": { + "patterns": [ + { + "include": "#import" + }, + { + "include": "#class-declaration" + }, + { + "include": "#function-declaration" + }, + { + "include": "#statement-keyword" + }, + { + "include": "#assignment-operator" + }, + { + "include": "#decorator" + }, + { + "include": "#docstring-statement" + }, + { + "include": "#semicolon" + } + ] + }, + "semicolon": { + "patterns": [ + { + "name": "invalid.deprecated.semicolon.seq", + "match": "\\;$" + } + ] + }, + "comments": { + "patterns": [ + { + "name": "comment.line.number-sign.seq", + "contentName": "meta.typehint.comment.seq", + "begin": "(?x)\n (?:\n \\# \\s* (type:)\n \\s*+ (?# we want `\\s*+` which is possessive quantifier since\n we do not actually want to backtrack when matching\n whitespace here)\n (?! $ | \\#)\n )\n", + "end": "(?:$|(?=\\#))", + "beginCaptures": { + "0": { + "name": "meta.typehint.comment.seq" + }, + "1": { + "name": "comment.typehint.directive.notation.seq" + } + }, + "patterns": [ + { + "name": "comment.typehint.ignore.notation.seq", + "match": "(?x)\n \\G ignore\n (?= \\s* (?: $ | \\#))\n" + }, + { + "name": "comment.typehint.type.notation.seq", + "match": "(?x)\n (?))" + }, + { + "name": "comment.typehint.variable.notation.seq", + "match": "([[:alpha:]_]\\w*)" + } + ] + }, + { + "include": "#comments-base" + } + ] + }, + "docstring-statement": { + "begin": "^(?=\\s*[rR]?(\\'\\'\\'|\\\"\\\"\\\"|\\'|\\\"))", + "comment": "the string either terminates correctly or by the beginning of a new line (this is for single line docstrings that aren't terminated) AND it's not followed by another docstring", + "end": "((?<=\\1)|^)(?!\\s*[rR]?(\\'\\'\\'|\\\"\\\"\\\"|\\'|\\\"))", + "patterns": [ + { + "include": "#docstring" + } + ] + }, + "docstring": { + "patterns": [ + { + "name": "string.quoted.docstring.multi.seq", + "begin": "(\\'\\'\\'|\\\"\\\"\\\")", + "end": "(\\1)", + "beginCaptures": { + "1": { + "name": "punctuation.definition.string.begin.seq" + } + }, + "endCaptures": { + "1": { + "name": "punctuation.definition.string.end.seq" + } + }, + "patterns": [ + { + "include": "#docstring-prompt" + }, + { + "include": "#codetags" + }, + { + "include": "#docstring-guts-unicode" + } + ] + }, + { + "name": "string.quoted.docstring.raw.multi.seq", + "begin": "([rR])(\\'\\'\\'|\\\"\\\"\\\")", + "end": "(\\2)", + "beginCaptures": { + "1": { + "name": "storage.type.string.seq" + }, + "2": { + "name": "punctuation.definition.string.begin.seq" + } + }, + "endCaptures": { + "1": { + "name": "punctuation.definition.string.end.seq" + } + }, + "patterns": [ + { + "include": "#string-consume-escape" + }, + { + "include": "#docstring-prompt" + }, + { + "include": "#codetags" + } + ] + }, + { + "name": "string.quoted.docstring.single.seq", + "begin": "(\\'|\\\")", + "end": "(\\1)|(\\n)", + "beginCaptures": { + "1": { + "name": "punctuation.definition.string.begin.seq" + } + }, + "endCaptures": { + "1": { + "name": "punctuation.definition.string.end.seq" + }, + "2": { + "name": "invalid.illegal.newline.seq" + } + }, + "patterns": [ + { + "include": "#codetags" + }, + { + "include": "#docstring-guts-unicode" + } + ] + }, + { + "name": "string.quoted.docstring.raw.single.seq", + "begin": "([rR])(\\'|\\\")", + "end": "(\\2)|(\\n)", + "beginCaptures": { + "1": { + "name": "storage.type.string.seq" + }, + "2": { + "name": "punctuation.definition.string.begin.seq" + } + }, + "endCaptures": { + "1": { + "name": "punctuation.definition.string.end.seq" + }, + "2": { + "name": "invalid.illegal.newline.seq" + } + }, + "patterns": [ + { + "include": "#string-consume-escape" + }, + { + "include": "#codetags" + } + ] + } + ] + }, + "docstring-guts-unicode": { + "patterns": [ + { + "include": "#escape-sequence-unicode" + }, + { + "include": "#escape-sequence" + }, + { + "include": "#string-line-continuation" + } + ] + }, + "docstring-prompt": { + "match": "(?x)\n (?:\n (?:^|\\G) \\s* (?# '\\G' is necessary for ST)\n ((?:>>>|\\.\\.\\.) \\s) (?=\\s*\\S)\n )\n", + "captures": { + "1": { + "name": "keyword.control.flow.seq" + } + } + }, + "statement-keyword": { + "patterns": [ + { + "name": "storage.type.function.seq", + "match": "\\b((async\\s+)?\\s*def)\\b" + }, + { + "name": "keyword.control.flow.seq", + "comment": "if `as` is eventually followed by `:` or line continuation\nit's probably control flow like:\n with foo as bar, \\\n Foo as Bar:\n try:\n do_stuff()\n except Exception as e:\n pass\n", + "match": "\\b(?>= | //= | \\*\\*=\n | \\+= | -= | /= | @=\n | \\*= | %= | ~= | \\^= | &= | \\|=\n | =(?!=)\n" + }, + "operator": { + "match": "(?x)\n \\b(?> | & | \\| | \\^ | ~) (?# 3)\n\n | (\\*\\* | \\* | \\+ | - | % | // | / | @) (?# 4)\n\n | (!= | == | >= | <= | < | >) (?# 5)\n", + "captures": { + "1": { + "name": "keyword.operator.logical.seq" + }, + "2": { + "name": "keyword.control.flow.seq" + }, + "3": { + "name": "keyword.operator.bitwise.seq" + }, + "4": { + "name": "keyword.operator.arithmetic.seq" + }, + "5": { + "name": "keyword.operator.comparison.seq" + } + } + }, + "punctuation": { + "patterns": [ + { + "name": "punctuation.separator.colon.seq", + "match": ":" + }, + { + "name": "punctuation.separator.element.seq", + "match": "," + } + ] + }, + "literal": { + "patterns": [ + { + "name": "constant.language.seq", + "match": "\\b(True|False|None|NotImplemented|Ellipsis)\\b" + }, + { + "include": "#number" + } + ] + }, + "number": { + "name": "constant.numeric.seq", + "patterns": [ + { + "include": "#number-float" + }, + { + "include": "#number-dec" + }, + { + "include": "#number-hex" + }, + { + "include": "#number-oct" + }, + { + "include": "#number-bin" + }, + { + "include": "#number-long" + }, + { + "name": "invalid.illegal.name.seq", + "match": "\\b[0-9]+\\w+" + } + ] + }, + "number-float": { + "name": "constant.numeric.float.seq", + "match": "(?x)\n (?=^]? [-+ ]? \\#?\n \\d* ,? (\\.\\d+)? [bcdeEfFgGnosxX%]? )?\n })\n )\n", + "captures": { + "1": { + "name": "constant.character.format.placeholder.other.seq" + }, + "3": { + "name": "storage.type.format.seq" + }, + "4": { + "name": "storage.type.format.seq" + } + } + }, + { + "name": "meta.format.brace.seq", + "match": "(?x)\n (\n {\n \\w* (\\.[[:alpha:]_]\\w* | \\[[^\\]'\"]+\\])*\n (![rsa])?\n (:)\n [^'\"{}\\n]* (?:\n \\{ [^'\"}\\n]*? \\} [^'\"{}\\n]*\n )*\n }\n )\n", + "captures": { + "1": { + "name": "constant.character.format.placeholder.other.seq" + }, + "3": { + "name": "storage.type.format.seq" + }, + "4": { + "name": "storage.type.format.seq" + } + } + } + ] + }, + "fstring-formatting": { + "patterns": [ + { + "include": "#fstring-formatting-braces" + }, + { + "include": "#fstring-formatting-singe-brace" + } + ] + }, + "fstring-formatting-singe-brace": { + "name": "invalid.illegal.brace.seq", + "match": "(}(?!}))" + }, + "import": { + "comment": "Import statements used to correctly mark `from`, `import`, and `as`\n", + "patterns": [ + { + "begin": "\\b(?)", + "end": "(?=:)", + "beginCaptures": { + "1": { + "name": "punctuation.separator.annotation.result.seq" + } + }, + "patterns": [ + { + "include": "#expression" + } + ] + }, + "item-access": { + "patterns": [ + { + "name": "meta.item-access.seq", + "begin": "(?x)\n \\b(?=\n [[:alpha:]_]\\w* \\s* \\[\n )\n", + "end": "(\\])", + "endCaptures": { + "1": { + "name": "punctuation.definition.arguments.end.seq" + } + }, + "patterns": [ + { + "include": "#item-name" + }, + { + "include": "#item-index" + }, + { + "include": "#expression" + } + ] + } + ] + }, + "item-name": { + "patterns": [ + { + "include": "#special-variables" + }, + { + "include": "#builtin-functions" + }, + { + "include": "#special-names" + }, + { + "match": "(?x)\n \\b ([[:alpha:]_]\\w*) \\b\n" + } + ] + }, + "item-index": { + "begin": "(\\[)", + "end": "(?=\\])", + "beginCaptures": { + "1": { + "name": "punctuation.definition.arguments.begin.seq" + } + }, + "contentName": "meta.item-access.arguments.seq", + "patterns": [ + { + "name": "punctuation.separator.slice.seq", + "match": ":" + }, + { + "include": "#expression" + } + ] + }, + "decorator": { + "name": "meta.function.decorator.seq", + "begin": "(?x)\n ^\\s*\n ((@)) \\s* (?=[[:alpha:]_]\\w*)\n", + "end": "(?x)\n ( \\) )\n # trailing whitespace and comments are legal\n (?: (.*?) (?=\\s*(?:\\#|$)) )\n | (?=\\n|\\#)\n", + "beginCaptures": { + "1": { + "name": "entity.name.function.decorator.seq" + }, + "2": { + "name": "punctuation.definition.decorator.seq" + } + }, + "endCaptures": { + "1": { + "name": "punctuation.definition.arguments.end.seq" + }, + "2": { + "name": "invalid.illegal.decorator.seq" + } + }, + "patterns": [ + { + "include": "#decorator-name" + }, + { + "include": "#function-arguments" + } + ] + }, + "decorator-name": { + "patterns": [ + { + "include": "#builtin-callables" + }, + { + "include": "#illegal-object-name" + }, + { + "name": "entity.name.function.decorator.seq", + "match": "(?x)\n ([[:alpha:]_]\\w*) | (\\.)\n", + "captures": { + "2": { + "name": "punctuation.separator.period.seq" + } + } + }, + { + "include": "#line-continuation" + }, + { + "name": "invalid.illegal.decorator.seq", + "match": "(?x)\n \\s* ([^([:alpha:]\\s_\\.#\\\\] .*?) (?=\\#|$)\n", + "captures": { + "1": { + "name": "invalid.illegal.decorator.seq" + } + } + } + ] + }, + "call-wrapper-inheritance": { + "comment": "same as a function call, but in inheritance context", + "name": "meta.function-call.seq", + "begin": "(?x)\n \\b(?=\n ([[:alpha:]_]\\w*) \\s* (\\()\n )\n", + "end": "(\\))", + "endCaptures": { + "1": { + "name": "punctuation.definition.arguments.end.seq" + } + }, + "patterns": [ + { + "include": "#inheritance-name" + }, + { + "include": "#function-arguments" + } + ] + }, + "inheritance-name": { + "patterns": [ + { + "include": "#lambda-incomplete" + }, + { + "include": "#builtin-possible-callables" + }, + { + "include": "#inheritance-identifier" + } + ] + }, + "function-call": { + "name": "meta.function-call.seq", + "begin": "(?x)\n \\b(?=\n ([[:alpha:]_]\\w*) \\s* (\\()\n )\n", + "end": "(\\))", + "endCaptures": { + "1": { + "name": "punctuation.definition.arguments.end.seq" + } + }, + "patterns": [ + { + "include": "#special-variables" + }, + { + "include": "#function-name" + }, + { + "include": "#function-arguments" + } + ] + }, + "function-name": { + "patterns": [ + { + "include": "#builtin-possible-callables" + }, + { + "comment": "Some color schemas support meta.function-call.generic scope", + "name": "meta.function-call.generic.seq", + "match": "(?x)\n \\b ([[:alpha:]_]\\w*) \\b\n" + } + ] + }, + "function-arguments": { + "begin": "(\\()", + "end": "(?=\\))(?!\\)\\s*\\()", + "beginCaptures": { + "1": { + "name": "punctuation.definition.arguments.begin.seq" + } + }, + "contentName": "meta.function-call.arguments.seq", + "patterns": [ + { + "name": "punctuation.separator.arguments.seq", + "match": "(,)" + }, + { + "match": "(?x)\n (?:(?<=[,(])|^) \\s* (\\*{1,2})\n", + "captures": { + "1": { + "name": "keyword.operator.unpacking.arguments.seq" + } + } + }, + { + "include": "#lambda-incomplete" + }, + { + "include": "#illegal-names" + }, + { + "match": "\\b([[:alpha:]_]\\w*)\\s*(=)(?!=)", + "captures": { + "1": { + "name": "variable.parameter.function-call.seq" + }, + "2": { + "name": "keyword.operator.assignment.seq" + } + } + }, + { + "name": "keyword.operator.assignment.seq", + "match": "=(?!=)" + }, + { + "include": "#expression" + }, + { + "match": "\\s*(\\))\\s*(\\()", + "captures": { + "1": { + "name": "punctuation.definition.arguments.end.seq" + }, + "2": { + "name": "punctuation.definition.arguments.begin.seq" + } + } + } + ] + }, + "builtin-callables": { + "patterns": [ + { + "include": "#illegal-names" + }, + { + "include": "#illegal-object-name" + }, + { + "include": "#builtin-exceptions" + }, + { + "include": "#builtin-functions" + }, + { + "include": "#builtin-types" + } + ] + }, + "builtin-possible-callables": { + "patterns": [ + { + "include": "#builtin-callables" + }, + { + "include": "#magic-names" + } + ] + }, + "builtin-exceptions": { + "name": "support.type.exception.seq", + "match": "(?x) (?" + }, + "regexp-base-expression": { + "patterns": [ + { + "include": "#regexp-quantifier" + }, + { + "include": "#regexp-base-common" + } + ] + }, + "fregexp-base-expression": { + "patterns": [ + { + "include": "#fregexp-quantifier" + }, + { + "include": "#fstring-formatting-braces" + }, + { + "match": "\\{.*?\\}" + }, + { + "include": "#regexp-base-common" + } + ] + }, + "fstring-formatting-braces": { + "patterns": [ + { + "comment": "empty braces are illegal", + "match": "({)(\\s*?)(})", + "captures": { + "1": { + "name": "constant.character.format.placeholder.other.seq" + }, + "2": { + "name": "invalid.illegal.brace.seq" + }, + "3": { + "name": "constant.character.format.placeholder.other.seq" + } + } + }, + { + "name": "constant.character.escape.seq", + "match": "({{|}})" + } + ] + }, + "regexp-base-common": { + "patterns": [ + { + "name": "support.other.match.any.regexp", + "match": "\\." + }, + { + "name": "support.other.match.begin.regexp", + "match": "\\^" + }, + { + "name": "support.other.match.end.regexp", + "match": "\\$" + }, + { + "name": "keyword.operator.quantifier.regexp", + "match": "[+*?]\\??" + }, + { + "name": "keyword.operator.disjunction.regexp", + "match": "\\|" + }, + { + "include": "#regexp-escape-sequence" + } + ] + }, + "regexp-quantifier": { + "name": "keyword.operator.quantifier.regexp", + "match": "(?x)\n \\{(\n \\d+ | \\d+,(\\d+)? | ,\\d+\n )\\}\n" + }, + "fregexp-quantifier": { + "name": "keyword.operator.quantifier.regexp", + "match": "(?x)\n \\{\\{(\n \\d+ | \\d+,(\\d+)? | ,\\d+\n )\\}\\}\n" + }, + "regexp-backreference-number": { + "name": "meta.backreference.regexp", + "match": "(\\\\[1-9]\\d?)", + "captures": { + "1": { + "name": "entity.name.tag.backreference.regexp" + } + } + }, + "regexp-backreference": { + "name": "meta.backreference.named.regexp", + "match": "(?x)\n (\\() (\\?P= \\w+(?:\\s+[[:alnum:]]+)?) (\\))\n", + "captures": { + "1": { + "name": "support.other.parenthesis.regexp punctuation.parenthesis.backreference.named.begin.regexp" + }, + "2": { + "name": "entity.name.tag.named.backreference.regexp" + }, + "3": { + "name": "support.other.parenthesis.regexp punctuation.parenthesis.backreference.named.end.regexp" + } + } + }, + "regexp-flags": { + "name": "storage.modifier.flag.regexp", + "match": "\\(\\?[aiLmsux]+\\)" + }, + "regexp-escape-special": { + "name": "support.other.escape.special.regexp", + "match": "\\\\([AbBdDsSwWZ])" + }, + "regexp-escape-character": { + "name": "constant.character.escape.regexp", + "match": "(?x)\n \\\\ (\n x[0-9A-Fa-f]{2}\n | 0[0-7]{1,2}\n | [0-7]{3}\n )\n" + }, + "regexp-escape-unicode": { + "name": "constant.character.unicode.regexp", + "match": "(?x)\n \\\\ (\n u[0-9A-Fa-f]{4}\n | U[0-9A-Fa-f]{8}\n )\n" + }, + "regexp-escape-catchall": { + "name": "constant.character.escape.regexp", + "match": "\\\\(.|\\n)" + }, + "regexp-escape-sequence": { + "patterns": [ + { + "include": "#regexp-escape-special" + }, + { + "include": "#regexp-escape-character" + }, + { + "include": "#regexp-escape-unicode" + }, + { + "include": "#regexp-backreference-number" + }, + { + "include": "#regexp-escape-catchall" + } + ] + }, + "regexp-charecter-set-escapes": { + "patterns": [ + { + "name": "constant.character.escape.regexp", + "match": "\\\\[abfnrtv\\\\]" + }, + { + "include": "#regexp-escape-special" + }, + { + "name": "constant.character.escape.regexp", + "match": "\\\\([0-7]{1,3})" + }, + { + "include": "#regexp-escape-character" + }, + { + "include": "#regexp-escape-unicode" + }, + { + "include": "#regexp-escape-catchall" + } + ] + }, + "codetags": { + "match": "(?:\\b(NOTE|XXX|HACK|FIXME|BUG|TODO)\\b)", + "captures": { + "1": { + "name": "keyword.codetag.notation.seq" + } + } + }, + "comments-base": { + "name": "comment.line.number-sign.seq", + "begin": "(\\#)", + "beginCaptures": { + "1": { + "name": "punctuation.definition.comment.seq" + } + }, + "end": "($)", + "patterns": [ + { + "include": "#codetags" + } + ] + }, + "comments-string-single-three": { + "name": "comment.line.number-sign.seq", + "begin": "(\\#)", + "beginCaptures": { + "1": { + "name": "punctuation.definition.comment.seq" + } + }, + "end": "($|(?='''))", + "patterns": [ + { + "include": "#codetags" + } + ] + }, + "comments-string-double-three": { + "name": "comment.line.number-sign.seq", + "begin": "(\\#)", + "beginCaptures": { + "1": { + "name": "punctuation.definition.comment.seq" + } + }, + "end": "($|(?=\"\"\"))", + "patterns": [ + { + "include": "#codetags" + } + ] + }, + "single-one-regexp-expression": { + "patterns": [ + { + "include": "#regexp-base-expression" + }, + { + "include": "#single-one-regexp-character-set" + }, + { + "include": "#single-one-regexp-comments" + }, + { + "include": "#regexp-flags" + }, + { + "include": "#single-one-regexp-named-group" + }, + { + "include": "#regexp-backreference" + }, + { + "include": "#single-one-regexp-lookahead" + }, + { + "include": "#single-one-regexp-lookahead-negative" + }, + { + "include": "#single-one-regexp-lookbehind" + }, + { + "include": "#single-one-regexp-lookbehind-negative" + }, + { + "include": "#single-one-regexp-conditional" + }, + { + "include": "#single-one-regexp-parentheses-non-capturing" + }, + { + "include": "#single-one-regexp-parentheses" + } + ] + }, + "single-one-regexp-character-set": { + "patterns": [ + { + "match": "(?x)\n \\[ \\^? \\] (?! .*?\\])\n" + }, + { + "name": "meta.character.set.regexp", + "begin": "(\\[)(\\^)?(\\])?", + "end": "(\\]|(?=\\'))|((?=(?)\n", + "end": "(\\)|(?=\\'))|((?=(?)\n", + "end": "(\\)|(?=\\'\\'\\'))", + "beginCaptures": { + "1": { + "name": "support.other.parenthesis.regexp punctuation.parenthesis.named.begin.regexp" + }, + "2": { + "name": "entity.name.tag.named.group.regexp" + } + }, + "endCaptures": { + "1": { + "name": "support.other.parenthesis.regexp punctuation.parenthesis.named.end.regexp" + }, + "2": { + "name": "invalid.illegal.newline.seq" + } + }, + "patterns": [ + { + "include": "#single-three-regexp-expression" + }, + { + "include": "#comments-string-single-three" + } + ] + }, + "single-three-regexp-comments": { + "name": "comment.regexp", + "begin": "\\(\\?#", + "end": "(\\)|(?=\\'\\'\\'))", + "beginCaptures": { + "0": { + "name": "punctuation.comment.begin.regexp" + } + }, + "endCaptures": { + "1": { + "name": "punctuation.comment.end.regexp" + }, + "2": { + "name": "invalid.illegal.newline.seq" + } + }, + "patterns": [ + { + "include": "#codetags" + } + ] + }, + "single-three-regexp-lookahead": { + "begin": "(\\()\\?=", + "end": "(\\)|(?=\\'\\'\\'))", + "beginCaptures": { + "0": { + "name": "keyword.operator.lookahead.regexp" + }, + "1": { + "name": "punctuation.parenthesis.lookahead.begin.regexp" + } + }, + "endCaptures": { + "1": { + "name": "keyword.operator.lookahead.regexp punctuation.parenthesis.lookahead.end.regexp" + }, + "2": { + "name": "invalid.illegal.newline.seq" + } + }, + "patterns": [ + { + "include": "#single-three-regexp-expression" + }, + { + "include": "#comments-string-single-three" + } + ] + }, + "single-three-regexp-lookahead-negative": { + "begin": "(\\()\\?!", + "end": "(\\)|(?=\\'\\'\\'))", + "beginCaptures": { + "0": { + "name": "keyword.operator.lookahead.negative.regexp" + }, + "1": { + "name": "punctuation.parenthesis.lookahead.begin.regexp" + } + }, + "endCaptures": { + "1": { + "name": "keyword.operator.lookahead.negative.regexp punctuation.parenthesis.lookahead.end.regexp" + }, + "2": { + "name": "invalid.illegal.newline.seq" + } + }, + "patterns": [ + { + "include": "#single-three-regexp-expression" + }, + { + "include": "#comments-string-single-three" + } + ] + }, + "single-three-regexp-lookbehind": { + "begin": "(\\()\\?<=", + "end": "(\\)|(?=\\'\\'\\'))", + "beginCaptures": { + "0": { + "name": "keyword.operator.lookbehind.regexp" + }, + "1": { + "name": "punctuation.parenthesis.lookbehind.begin.regexp" + } + }, + "endCaptures": { + "1": { + "name": "keyword.operator.lookbehind.regexp punctuation.parenthesis.lookbehind.end.regexp" + }, + "2": { + "name": "invalid.illegal.newline.seq" + } + }, + "patterns": [ + { + "include": "#single-three-regexp-expression" + }, + { + "include": "#comments-string-single-three" + } + ] + }, + "single-three-regexp-lookbehind-negative": { + "begin": "(\\()\\?)\n", + "end": "(\\)|(?=\"))|((?=(?)\n", + "end": "(\\)|(?=\"\"\"))", + "beginCaptures": { + "1": { + "name": "support.other.parenthesis.regexp punctuation.parenthesis.named.begin.regexp" + }, + "2": { + "name": "entity.name.tag.named.group.regexp" + } + }, + "endCaptures": { + "1": { + "name": "support.other.parenthesis.regexp punctuation.parenthesis.named.end.regexp" + }, + "2": { + "name": "invalid.illegal.newline.seq" + } + }, + "patterns": [ + { + "include": "#double-three-regexp-expression" + }, + { + "include": "#comments-string-double-three" + } + ] + }, + "double-three-regexp-comments": { + "name": "comment.regexp", + "begin": "\\(\\?#", + "end": "(\\)|(?=\"\"\"))", + "beginCaptures": { + "0": { + "name": "punctuation.comment.begin.regexp" + } + }, + "endCaptures": { + "1": { + "name": "punctuation.comment.end.regexp" + }, + "2": { + "name": "invalid.illegal.newline.seq" + } + }, + "patterns": [ + { + "include": "#codetags" + } + ] + }, + "double-three-regexp-lookahead": { + "begin": "(\\()\\?=", + "end": "(\\)|(?=\"\"\"))", + "beginCaptures": { + "0": { + "name": "keyword.operator.lookahead.regexp" + }, + "1": { + "name": "punctuation.parenthesis.lookahead.begin.regexp" + } + }, + "endCaptures": { + "1": { + "name": "keyword.operator.lookahead.regexp punctuation.parenthesis.lookahead.end.regexp" + }, + "2": { + "name": "invalid.illegal.newline.seq" + } + }, + "patterns": [ + { + "include": "#double-three-regexp-expression" + }, + { + "include": "#comments-string-double-three" + } + ] + }, + "double-three-regexp-lookahead-negative": { + "begin": "(\\()\\?!", + "end": "(\\)|(?=\"\"\"))", + "beginCaptures": { + "0": { + "name": "keyword.operator.lookahead.negative.regexp" + }, + "1": { + "name": "punctuation.parenthesis.lookahead.begin.regexp" + } + }, + "endCaptures": { + "1": { + "name": "keyword.operator.lookahead.negative.regexp punctuation.parenthesis.lookahead.end.regexp" + }, + "2": { + "name": "invalid.illegal.newline.seq" + } + }, + "patterns": [ + { + "include": "#double-three-regexp-expression" + }, + { + "include": "#comments-string-double-three" + } + ] + }, + "double-three-regexp-lookbehind": { + "begin": "(\\()\\?<=", + "end": "(\\)|(?=\"\"\"))", + "beginCaptures": { + "0": { + "name": "keyword.operator.lookbehind.regexp" + }, + "1": { + "name": "punctuation.parenthesis.lookbehind.begin.regexp" + } + }, + "endCaptures": { + "1": { + "name": "keyword.operator.lookbehind.regexp punctuation.parenthesis.lookbehind.end.regexp" + }, + "2": { + "name": "invalid.illegal.newline.seq" + } + }, + "patterns": [ + { + "include": "#double-three-regexp-expression" + }, + { + "include": "#comments-string-double-three" + } + ] + }, + "double-three-regexp-lookbehind-negative": { + "begin": "(\\()\\?)\n", + "end": "(\\)|(?=\\'))|((?=(?)\n", + "end": "(\\)|(?=\\'\\'\\'))", + "beginCaptures": { + "1": { + "name": "support.other.parenthesis.regexp punctuation.parenthesis.named.begin.regexp" + }, + "2": { + "name": "entity.name.tag.named.group.regexp" + } + }, + "endCaptures": { + "1": { + "name": "support.other.parenthesis.regexp punctuation.parenthesis.named.end.regexp" + }, + "2": { + "name": "invalid.illegal.newline.seq" + } + }, + "patterns": [ + { + "include": "#single-three-fregexp-expression" + }, + { + "include": "#comments-string-single-three" + } + ] + }, + "single-three-fregexp-lookahead": { + "begin": "(\\()\\?=", + "end": "(\\)|(?=\\'\\'\\'))", + "beginCaptures": { + "0": { + "name": "keyword.operator.lookahead.regexp" + }, + "1": { + "name": "punctuation.parenthesis.lookahead.begin.regexp" + } + }, + "endCaptures": { + "1": { + "name": "keyword.operator.lookahead.regexp punctuation.parenthesis.lookahead.end.regexp" + }, + "2": { + "name": "invalid.illegal.newline.seq" + } + }, + "patterns": [ + { + "include": "#single-three-fregexp-expression" + }, + { + "include": "#comments-string-single-three" + } + ] + }, + "single-three-fregexp-lookahead-negative": { + "begin": "(\\()\\?!", + "end": "(\\)|(?=\\'\\'\\'))", + "beginCaptures": { + "0": { + "name": "keyword.operator.lookahead.negative.regexp" + }, + "1": { + "name": "punctuation.parenthesis.lookahead.begin.regexp" + } + }, + "endCaptures": { + "1": { + "name": "keyword.operator.lookahead.negative.regexp punctuation.parenthesis.lookahead.end.regexp" + }, + "2": { + "name": "invalid.illegal.newline.seq" + } + }, + "patterns": [ + { + "include": "#single-three-fregexp-expression" + }, + { + "include": "#comments-string-single-three" + } + ] + }, + "single-three-fregexp-lookbehind": { + "begin": "(\\()\\?<=", + "end": "(\\)|(?=\\'\\'\\'))", + "beginCaptures": { + "0": { + "name": "keyword.operator.lookbehind.regexp" + }, + "1": { + "name": "punctuation.parenthesis.lookbehind.begin.regexp" + } + }, + "endCaptures": { + "1": { + "name": "keyword.operator.lookbehind.regexp punctuation.parenthesis.lookbehind.end.regexp" + }, + "2": { + "name": "invalid.illegal.newline.seq" + } + }, + "patterns": [ + { + "include": "#single-three-fregexp-expression" + }, + { + "include": "#comments-string-single-three" + } + ] + }, + "single-three-fregexp-lookbehind-negative": { + "begin": "(\\()\\?)\n", + "end": "(\\)|(?=\"))|((?=(?)\n", + "end": "(\\)|(?=\"\"\"))", + "beginCaptures": { + "1": { + "name": "support.other.parenthesis.regexp punctuation.parenthesis.named.begin.regexp" + }, + "2": { + "name": "entity.name.tag.named.group.regexp" + } + }, + "endCaptures": { + "1": { + "name": "support.other.parenthesis.regexp punctuation.parenthesis.named.end.regexp" + }, + "2": { + "name": "invalid.illegal.newline.seq" + } + }, + "patterns": [ + { + "include": "#double-three-fregexp-expression" + }, + { + "include": "#comments-string-double-three" + } + ] + }, + "double-three-fregexp-lookahead": { + "begin": "(\\()\\?=", + "end": "(\\)|(?=\"\"\"))", + "beginCaptures": { + "0": { + "name": "keyword.operator.lookahead.regexp" + }, + "1": { + "name": "punctuation.parenthesis.lookahead.begin.regexp" + } + }, + "endCaptures": { + "1": { + "name": "keyword.operator.lookahead.regexp punctuation.parenthesis.lookahead.end.regexp" + }, + "2": { + "name": "invalid.illegal.newline.seq" + } + }, + "patterns": [ + { + "include": "#double-three-fregexp-expression" + }, + { + "include": "#comments-string-double-three" + } + ] + }, + "double-three-fregexp-lookahead-negative": { + "begin": "(\\()\\?!", + "end": "(\\)|(?=\"\"\"))", + "beginCaptures": { + "0": { + "name": "keyword.operator.lookahead.negative.regexp" + }, + "1": { + "name": "punctuation.parenthesis.lookahead.begin.regexp" + } + }, + "endCaptures": { + "1": { + "name": "keyword.operator.lookahead.negative.regexp punctuation.parenthesis.lookahead.end.regexp" + }, + "2": { + "name": "invalid.illegal.newline.seq" + } + }, + "patterns": [ + { + "include": "#double-three-fregexp-expression" + }, + { + "include": "#comments-string-double-three" + } + ] + }, + "double-three-fregexp-lookbehind": { + "begin": "(\\()\\?<=", + "end": "(\\)|(?=\"\"\"))", + "beginCaptures": { + "0": { + "name": "keyword.operator.lookbehind.regexp" + }, + "1": { + "name": "punctuation.parenthesis.lookbehind.begin.regexp" + } + }, + "endCaptures": { + "1": { + "name": "keyword.operator.lookbehind.regexp punctuation.parenthesis.lookbehind.end.regexp" + }, + "2": { + "name": "invalid.illegal.newline.seq" + } + }, + "patterns": [ + { + "include": "#double-three-fregexp-expression" + }, + { + "include": "#comments-string-double-three" + } + ] + }, + "double-three-fregexp-lookbehind-negative": { + "begin": "(\\()\\?=^]? [-+ ]? \\#?\n \\d* ,? (\\.\\d+)? [bcdeEfFgGnosxX%]? )(?=})\n", + "captures": { + "1": { + "name": "storage.type.format.seq" + }, + "2": { + "name": "storage.type.format.seq" + } + } + }, + { + "include": "#fstring-terminator-single-tail" + } + ] + }, + "fstring-terminator-single-tail": { + "begin": "(![rsa])?(:)(?=.*?{)", + "end": "(?=})|(?=\\n)", + "beginCaptures": { + "1": { + "name": "storage.type.format.seq" + }, + "2": { + "name": "storage.type.format.seq" + } + }, + "patterns": [ + { + "include": "#fstring-illegal-single-brace" + }, + { + "include": "#fstring-single-brace" + }, + { + "name": "storage.type.format.seq", + "match": "([bcdeEfFgGnosxX%])(?=})" + }, + { + "name": "storage.type.format.seq", + "match": "(\\.\\d+)" + }, + { + "name": "storage.type.format.seq", + "match": "(,)" + }, + { + "name": "storage.type.format.seq", + "match": "(\\d+)" + }, + { + "name": "storage.type.format.seq", + "match": "(\\#)" + }, + { + "name": "storage.type.format.seq", + "match": "([-+ ])" + }, + { + "name": "storage.type.format.seq", + "match": "([<>=^])" + }, + { + "name": "storage.type.format.seq", + "match": "(\\w)" + } + ] + }, + "fstring-fnorm-quoted-multi-line": { + "name": "meta.fstring.seq", + "begin": "(\\b[fF])([bBuU])?('''|\"\"\")", + "end": "(\\3)", + "beginCaptures": { + "1": { + "name": "string.interpolated.python string.quoted.multi.python storage.type.string.seq" + }, + "2": { + "name": "invalid.illegal.prefix.seq" + }, + "3": { + "name": "punctuation.definition.string.begin.python string.interpolated.python string.quoted.multi.seq" + } + }, + "endCaptures": { + "1": { + "name": "punctuation.definition.string.end.python string.interpolated.python string.quoted.multi.seq" + }, + "2": { + "name": "invalid.illegal.newline.seq" + } + }, + "patterns": [ + { + "include": "#fstring-guts" + }, + { + "include": "#fstring-illegal-multi-brace" + }, + { + "include": "#fstring-multi-brace" + }, + { + "include": "#fstring-multi-core" + } + ] + }, + "fstring-normf-quoted-multi-line": { + "name": "meta.fstring.seq", + "begin": "(\\b[bBuU])([fF])('''|\"\"\")", + "end": "(\\3)", + "beginCaptures": { + "1": { + "name": "invalid.illegal.prefix.seq" + }, + "2": { + "name": "string.interpolated.python string.quoted.multi.python storage.type.string.seq" + }, + "3": { + "name": "punctuation.definition.string.begin.python string.quoted.multi.seq" + } + }, + "endCaptures": { + "1": { + "name": "punctuation.definition.string.end.python string.interpolated.python string.quoted.multi.seq" + }, + "2": { + "name": "invalid.illegal.newline.seq" + } + }, + "patterns": [ + { + "include": "#fstring-guts" + }, + { + "include": "#fstring-illegal-multi-brace" + }, + { + "include": "#fstring-multi-brace" + }, + { + "include": "#fstring-multi-core" + } + ] + }, + "fstring-raw-quoted-multi-line": { + "name": "meta.fstring.seq", + "begin": "(\\b(?:[R][fF]|[fF][R]))('''|\"\"\")", + "end": "(\\2)", + "beginCaptures": { + "1": { + "name": "string.interpolated.python string.quoted.raw.multi.python storage.type.string.seq" + }, + "2": { + "name": "punctuation.definition.string.begin.python string.quoted.raw.multi.seq" + } + }, + "endCaptures": { + "1": { + "name": "punctuation.definition.string.end.python string.interpolated.python string.quoted.raw.multi.seq" + }, + "2": { + "name": "invalid.illegal.newline.seq" + } + }, + "patterns": [ + { + "include": "#fstring-raw-guts" + }, + { + "include": "#fstring-illegal-multi-brace" + }, + { + "include": "#fstring-multi-brace" + }, + { + "include": "#fstring-raw-multi-core" + } + ] + }, + "fstring-multi-core": { + "name": "string.interpolated.python string.quoted.multi.seq", + "match": "(?x)\n (.+?)\n (\n (?# .* and .*? in multi-line match need special handling of\n newlines otherwise SublimeText and Atom will match slightly\n differently.\n\n The guard for newlines has to be separate from the\n lookahead because of special $ matching rule.)\n ($\\n?)\n |\n (?=[\\\\\\}\\{]|'''|\"\"\")\n )\n (?# due to how multiline regexps are matched we need a special case\n for matching a newline character)\n | \\n\n" + }, + "fstring-raw-multi-core": { + "name": "string.interpolated.python string.quoted.raw.multi.seq", + "match": "(?x)\n (.+?)\n (\n (?# .* and .*? in multi-line match need special handling of\n newlines otherwise SublimeText and Atom will match slightly\n differently.\n\n The guard for newlines has to be separate from the\n lookahead because of special $ matching rule.)\n ($\\n?)\n |\n (?=[\\\\\\}\\{]|'''|\"\"\")\n )\n (?# due to how multiline regexps are matched we need a special case\n for matching a newline character)\n | \\n\n" + }, + "fstring-multi-brace": { + "comment": "value interpolation using { ... }", + "begin": "(\\{)", + "end": "(?x)\n (\\})\n", + "beginCaptures": { + "1": { + "name": "constant.character.format.placeholder.other.seq" + } + }, + "endCaptures": { + "1": { + "name": "constant.character.format.placeholder.other.seq" + } + }, + "patterns": [ + { + "include": "#fstring-terminator-multi" + }, + { + "include": "#f-expression" + } + ] + }, + "fstring-terminator-multi": { + "patterns": [ + { + "name": "storage.type.format.seq", + "match": "(![rsa])(?=})" + }, + { + "match": "(?x)\n (![rsa])?\n ( : \\w? [<>=^]? [-+ ]? \\#?\n \\d* ,? (\\.\\d+)? [bcdeEfFgGnosxX%]? )(?=})\n", + "captures": { + "1": { + "name": "storage.type.format.seq" + }, + "2": { + "name": "storage.type.format.seq" + } + } + }, + { + "include": "#fstring-terminator-multi-tail" + } + ] + }, + "fstring-terminator-multi-tail": { + "begin": "(![rsa])?(:)(?=.*?{)", + "end": "(?=})", + "beginCaptures": { + "1": { + "name": "storage.type.format.seq" + }, + "2": { + "name": "storage.type.format.seq" + } + }, + "patterns": [ + { + "include": "#fstring-illegal-multi-brace" + }, + { + "include": "#fstring-multi-brace" + }, + { + "name": "storage.type.format.seq", + "match": "([bcdeEfFgGnosxX%])(?=})" + }, + { + "name": "storage.type.format.seq", + "match": "(\\.\\d+)" + }, + { + "name": "storage.type.format.seq", + "match": "(,)" + }, + { + "name": "storage.type.format.seq", + "match": "(\\d+)" + }, + { + "name": "storage.type.format.seq", + "match": "(\\#)" + }, + { + "name": "storage.type.format.seq", + "match": "([-+ ])" + }, + { + "name": "storage.type.format.seq", + "match": "([<>=^])" + }, + { + "name": "storage.type.format.seq", + "match": "(\\w)" + } + ] + } + } +} \ No newline at end of file diff --git a/seq-vscode/tsconfig.json b/seq-vscode/tsconfig.json new file mode 100644 index 000000000..a19d64e5c --- /dev/null +++ b/seq-vscode/tsconfig.json @@ -0,0 +1,20 @@ +{ + "compilerOptions": { + "module": "commonjs", + "target": "es6", + "outDir": "out", + "rootDir": "src", + "sourceMap": true + }, + "include": [ + "src" + ], + "exclude": [ + "node_modules", + ".vscode-test" + ], + "references": [ + { "path": "./client" }, + { "path": "./server" } + ] +} \ No newline at end of file diff --git a/seq-vscode/tslint.json b/seq-vscode/tslint.json new file mode 100644 index 000000000..0ab0ca6e8 --- /dev/null +++ b/seq-vscode/tslint.json @@ -0,0 +1,6 @@ +{ + "rules": { + "indent": [true, "tabs"], + "semicolon": [true, "always"] + } +} \ No newline at end of file diff --git a/seq-vscode/yarn.lock b/seq-vscode/yarn.lock new file mode 100644 index 000000000..fbbe539c4 --- /dev/null +++ b/seq-vscode/yarn.lock @@ -0,0 +1,261 @@ +# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. +# yarn lockfile v1 + + +"@babel/code-frame@^7.0.0": + version "7.5.5" + resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.5.5.tgz#bc0782f6d69f7b7d49531219699b988f669a8f9d" + integrity sha512-27d4lZoomVyo51VegxI20xZPuSHusqbQag/ztrBC7wegWoQ1nLREPVSKSW8byhTlzTKyNE4ifaTA6lCp7JjpFw== + dependencies: + "@babel/highlight" "^7.0.0" + +"@babel/highlight@^7.0.0": + version "7.5.0" + resolved "https://registry.yarnpkg.com/@babel/highlight/-/highlight-7.5.0.tgz#56d11312bd9248fa619591d02472be6e8cb32540" + integrity sha512-7dV4eu9gBxoM0dAnj/BCFDW9LFU0zvTrkq0ugM7pnHEgguOEeOz1so2ZghEdzviYzQEED0r4EAgpsBChKy1TRQ== + dependencies: + chalk "^2.0.0" + esutils "^2.0.2" + js-tokens "^4.0.0" + +"@types/mocha@^5.2.7": + version "5.2.7" + resolved "https://registry.yarnpkg.com/@types/mocha/-/mocha-5.2.7.tgz#315d570ccb56c53452ff8638738df60726d5b6ea" + integrity sha512-NYrtPht0wGzhwe9+/idPaBB+TqkY9AhTvOLMkThm0IoEfLaiVQZwBwyJ5puCkO3AUCWrmcoePjp2mbFocKy4SQ== + +"@types/node@^12.11.7": + version "12.12.22" + resolved "https://registry.yarnpkg.com/@types/node/-/node-12.12.22.tgz#b8d9eae3328b96910a373cf06ac8d3c5abe9c200" + integrity sha512-r5i93jqbPWGXYXxianGATOxTelkp6ih/U0WVnvaqAvTqM+0U6J3kw6Xk6uq/dWNRkEVw/0SLcO5ORXbVNz4FMQ== + +ansi-styles@^3.2.1: + version "3.2.1" + resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-3.2.1.tgz#41fbb20243e50b12be0f04b8dedbf07520ce841d" + integrity sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA== + dependencies: + color-convert "^1.9.0" + +argparse@^1.0.7: + version "1.0.10" + resolved "https://registry.yarnpkg.com/argparse/-/argparse-1.0.10.tgz#bcd6791ea5ae09725e17e5ad988134cd40b3d911" + integrity sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg== + dependencies: + sprintf-js "~1.0.2" + +balanced-match@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.0.tgz#89b4d199ab2bee49de164ea02b89ce462d71b767" + integrity sha1-ibTRmasr7kneFk6gK4nORi1xt2c= + +brace-expansion@^1.1.7: + version "1.1.11" + resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.11.tgz#3c7fcbf529d87226f3d2f52b966ff5271eb441dd" + integrity sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA== + dependencies: + balanced-match "^1.0.0" + concat-map "0.0.1" + +builtin-modules@^1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/builtin-modules/-/builtin-modules-1.1.1.tgz#270f076c5a72c02f5b65a47df94c5fe3a278892f" + integrity sha1-Jw8HbFpywC9bZaR9+Uxf46J4iS8= + +chalk@^2.0.0, chalk@^2.3.0: + version "2.4.2" + resolved "https://registry.yarnpkg.com/chalk/-/chalk-2.4.2.tgz#cd42541677a54333cf541a49108c1432b44c9424" + integrity sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ== + dependencies: + ansi-styles "^3.2.1" + escape-string-regexp "^1.0.5" + supports-color "^5.3.0" + +color-convert@^1.9.0: + version "1.9.3" + resolved "https://registry.yarnpkg.com/color-convert/-/color-convert-1.9.3.tgz#bb71850690e1f136567de629d2d5471deda4c1e8" + integrity sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg== + dependencies: + color-name "1.1.3" + +color-name@1.1.3: + version "1.1.3" + resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.3.tgz#a7d0558bd89c42f795dd42328f740831ca53bc25" + integrity sha1-p9BVi9icQveV3UIyj3QIMcpTvCU= + +commander@^2.12.1: + version "2.20.3" + resolved "https://registry.yarnpkg.com/commander/-/commander-2.20.3.tgz#fd485e84c03eb4881c20722ba48035e8531aeb33" + integrity sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ== + +concat-map@0.0.1: + version "0.0.1" + resolved "https://registry.yarnpkg.com/concat-map/-/concat-map-0.0.1.tgz#d8a96bd77fd68df7793a73036a3ba0d5405d477b" + integrity sha1-2Klr13/Wjfd5OnMDajug1UBdR3s= + +diff@^4.0.1: + version "4.0.1" + resolved "https://registry.yarnpkg.com/diff/-/diff-4.0.1.tgz#0c667cb467ebbb5cea7f14f135cc2dba7780a8ff" + integrity sha512-s2+XdvhPCOF01LRQBC8hf4vhbVmI2CGS5aZnxLJlT5FtdhPCDFq80q++zK2KlrVorVDdL5BOGZ/VfLrVtYNF+Q== + +escape-string-regexp@^1.0.5: + version "1.0.5" + resolved "https://registry.yarnpkg.com/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz#1b61c0562190a8dff6ae3bb2cf0200ca130b86d4" + integrity sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ= + +esprima@^4.0.0: + version "4.0.1" + resolved "https://registry.yarnpkg.com/esprima/-/esprima-4.0.1.tgz#13b04cdb3e6c5d19df91ab6987a8695619b0aa71" + integrity sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A== + +esutils@^2.0.2: + version "2.0.3" + resolved "https://registry.yarnpkg.com/esutils/-/esutils-2.0.3.tgz#74d2eb4de0b8da1293711910d50775b9b710ef64" + integrity sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g== + +fs.realpath@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/fs.realpath/-/fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f" + integrity sha1-FQStJSMVjKpA20onh8sBQRmU6k8= + +glob@^7.1.1: + version "7.1.6" + resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.6.tgz#141f33b81a7c2492e125594307480c46679278a6" + integrity sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA== + dependencies: + fs.realpath "^1.0.0" + inflight "^1.0.4" + inherits "2" + minimatch "^3.0.4" + once "^1.3.0" + path-is-absolute "^1.0.0" + +has-flag@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/has-flag/-/has-flag-3.0.0.tgz#b5d454dc2199ae225699f3467e5a07f3b955bafd" + integrity sha1-tdRU3CGZriJWmfNGfloH87lVuv0= + +inflight@^1.0.4: + version "1.0.6" + resolved "https://registry.yarnpkg.com/inflight/-/inflight-1.0.6.tgz#49bd6331d7d02d0c09bc910a1075ba8165b56df9" + integrity sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk= + dependencies: + once "^1.3.0" + wrappy "1" + +inherits@2: + version "2.0.4" + resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.4.tgz#0fa2c64f932917c3433a0ded55363aae37416b7c" + integrity sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ== + +js-tokens@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/js-tokens/-/js-tokens-4.0.0.tgz#19203fb59991df98e3a287050d4647cdeaf32499" + integrity sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ== + +js-yaml@^3.13.1: + version "3.13.1" + resolved "https://registry.yarnpkg.com/js-yaml/-/js-yaml-3.13.1.tgz#aff151b30bfdfa8e49e05da22e7415e9dfa37847" + integrity sha512-YfbcO7jXDdyj0DGxYVSlSeQNHbD7XPWvrVWeVUujrQEoZzWJIRrCPoyk6kL6IAjAG2IolMK4T0hNUe0HOUs5Jw== + dependencies: + argparse "^1.0.7" + esprima "^4.0.0" + +minimatch@^3.0.4: + version "3.0.4" + resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.0.4.tgz#5166e286457f03306064be5497e8dbb0c3d32083" + integrity sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA== + dependencies: + brace-expansion "^1.1.7" + +minimist@0.0.8: + version "0.0.8" + resolved "https://registry.yarnpkg.com/minimist/-/minimist-0.0.8.tgz#857fcabfc3397d2625b8228262e86aa7a011b05d" + integrity sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0= + +mkdirp@^0.5.1: + version "0.5.1" + resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.5.1.tgz#30057438eac6cf7f8c4767f38648d6697d75c903" + integrity sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM= + dependencies: + minimist "0.0.8" + +once@^1.3.0: + version "1.4.0" + resolved "https://registry.yarnpkg.com/once/-/once-1.4.0.tgz#583b1aa775961d4b113ac17d9c50baef9dd76bd1" + integrity sha1-WDsap3WWHUsROsF9nFC6753Xa9E= + dependencies: + wrappy "1" + +path-is-absolute@^1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/path-is-absolute/-/path-is-absolute-1.0.1.tgz#174b9268735534ffbc7ace6bf53a5a9e1b5c5f5f" + integrity sha1-F0uSaHNVNP+8es5r9TpanhtcX18= + +path-parse@^1.0.6: + version "1.0.6" + resolved "https://registry.yarnpkg.com/path-parse/-/path-parse-1.0.6.tgz#d62dbb5679405d72c4737ec58600e9ddcf06d24c" + integrity sha512-GSmOT2EbHrINBf9SR7CDELwlJ8AENk3Qn7OikK4nFYAu3Ote2+JYNVvkpAEQm3/TLNEJFD/xZJjzyxg3KBWOzw== + +resolve@^1.3.2: + version "1.14.1" + resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.14.1.tgz#9e018c540fcf0c427d678b9931cbf45e984bcaff" + integrity sha512-fn5Wobh4cxbLzuHaE+nphztHy43/b++4M6SsGFC2gB8uYwf0C8LcarfCz1un7UTW8OFQg9iNjZ4xpcFVGebDPg== + dependencies: + path-parse "^1.0.6" + +semver@^5.3.0: + version "5.7.1" + resolved "https://registry.yarnpkg.com/semver/-/semver-5.7.1.tgz#a954f931aeba508d307bbf069eff0c01c96116f7" + integrity sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ== + +sprintf-js@~1.0.2: + version "1.0.3" + resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.0.3.tgz#04e6926f662895354f3dd015203633b857297e2c" + integrity sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw= + +supports-color@^5.3.0: + version "5.5.0" + resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-5.5.0.tgz#e2e69a44ac8772f78a1ec0b35b689df6530efc8f" + integrity sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow== + dependencies: + has-flag "^3.0.0" + +tslib@^1.8.0, tslib@^1.8.1: + version "1.10.0" + resolved "https://registry.yarnpkg.com/tslib/-/tslib-1.10.0.tgz#c3c19f95973fb0a62973fb09d90d961ee43e5c8a" + integrity sha512-qOebF53frne81cf0S9B41ByenJ3/IuH8yJKngAX35CmiZySA0khhkovshKK+jGCaMnVomla7gVlIcc3EvKPbTQ== + +tslint@^5.16.0: + version "5.20.1" + resolved "https://registry.yarnpkg.com/tslint/-/tslint-5.20.1.tgz#e401e8aeda0152bc44dd07e614034f3f80c67b7d" + integrity sha512-EcMxhzCFt8k+/UP5r8waCf/lzmeSyVlqxqMEDQE7rWYiQky8KpIBz1JAoYXfROHrPZ1XXd43q8yQnULOLiBRQg== + dependencies: + "@babel/code-frame" "^7.0.0" + builtin-modules "^1.1.1" + chalk "^2.3.0" + commander "^2.12.1" + diff "^4.0.1" + glob "^7.1.1" + js-yaml "^3.13.1" + minimatch "^3.0.4" + mkdirp "^0.5.1" + resolve "^1.3.2" + semver "^5.3.0" + tslib "^1.8.0" + tsutils "^2.29.0" + +tsutils@^2.29.0: + version "2.29.0" + resolved "https://registry.yarnpkg.com/tsutils/-/tsutils-2.29.0.tgz#32b488501467acbedd4b85498673a0812aca0b99" + integrity sha512-g5JVHCIJwzfISaXpXE1qvNalca5Jwob6FjI4AoPlqMusJ6ftFE7IkkFoMhVLRgK+4Kx3gkzb8UZK5t5yTTvEmA== + dependencies: + tslib "^1.8.1" + +typescript@^3.6.4: + version "3.7.4" + resolved "https://registry.yarnpkg.com/typescript/-/typescript-3.7.4.tgz#1743a5ec5fef6a1fa9f3e4708e33c81c73876c19" + integrity sha512-A25xv5XCtarLwXpcDNZzCGvW2D1S3/bACratYBx2sax8PefsFhlYmkQicKHvpYflFS8if4zne5zT5kpJ7pzuvw== + +wrappy@1: + version "1.0.2" + resolved "https://registry.yarnpkg.com/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f" + integrity sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8= From 45d9a8cce7ed6fb9f9eb479e2482bafe582f7b6d Mon Sep 17 00:00:00 2001 From: jodiew Date: Mon, 6 Jan 2020 10:47:06 -0800 Subject: [PATCH 30/30] Add RTLD flags --- seq-vscode/server/src/server.ts | 15 +++++-- seq-vscode/server/src/wrapper.ts | 77 +++++++++++++++++++++++++++++--- 2 files changed, 81 insertions(+), 11 deletions(-) diff --git a/seq-vscode/server/src/server.ts b/seq-vscode/server/src/server.ts index 9f3454ad8..05b192ad3 100644 --- a/seq-vscode/server/src/server.ts +++ b/seq-vscode/server/src/server.ts @@ -12,7 +12,7 @@ import SeqWrapper from './wrapper'; let connection = createConnection(); -let wrapper: SeqWrapper = new SeqWrapper(); +let wrapper: SeqWrapper = new SeqWrapper(connection); let documents: TextDocuments = new TextDocuments(); @@ -89,14 +89,21 @@ connection.onCompletion( return []; } connection.console.log(`prefix ${prefix}`); - // TODO: Use the seq wrapper to get the list of completion items. - return []; + const result: CompletionItem[] = []; + const textMatches = wrapper.complete(prefix); + connection.console.log(`text ${textMatches}`); + // textMatches.forEach((match) => { + // result.push({ + // label: match + // }); + // }); + return result; } ); connection.onCompletionResolve( (item: CompletionItem): CompletionItem => { - connection.console.log(`onCompletionResolve ${item}`); + connection.console.log(`onCompletionResolve ${item.label}`); return item; } ); diff --git a/seq-vscode/server/src/wrapper.ts b/seq-vscode/server/src/wrapper.ts index f190c57da..41b800bae 100644 --- a/seq-vscode/server/src/wrapper.ts +++ b/seq-vscode/server/src/wrapper.ts @@ -1,16 +1,79 @@ -import { ForeignFunction, DynamicLibrary, RTLD_GLOBAL } from 'ffi'; -import { refType, types } from 'ref'; +import { IConnection } from 'vscode-languageserver'; +import { DynamicLibrary, VariadicForeignFunction, ForeignFunction, types } from 'ffi'; +import { isNull, refType } from 'ref'; + + +function GlobalLibrary (libfile: string | undefined, funcs: any, lib?: any): any { + /** + * GlobalLibrary + * ffi.Library rewritten to use more RTLD flags. + */ + if (process.platform === 'linux') { + libfile += '.so'; + } else if (process.platform === 'darwin') { + libfile += '.dylib'; + } else { + throw new Error('The current platform is not supported'); + } + if (!lib) { + lib = {}; + } + const dl: DynamicLibrary = new DynamicLibrary( + libfile || undefined, + DynamicLibrary.FLAGS.RTLD_NOW | DynamicLibrary.FLAGS.RTLD_GLOBAL + ); + + Object.keys(funcs || {}).forEach((func) => { + const fptr: Buffer = dl.get(func) + , info = funcs[func]; + + if (isNull(fptr)) { + throw new Error('Library: "' + libfile + '" returned NULL function pointer for "' + func + '"'); + } + + const resultType = info[0] + , paramTypes = info[1] + , fopts = info[2] + , abi = fopts && fopts.abi + , async = fopts && fopts.async + , varargs = fopts && fopts.varags; + + if (varargs) { + lib[func] = VariadicForeignFunction(fptr, resultType, paramTypes, abi); + } else { + const ff = ForeignFunction(fptr, resultType, paramTypes, abi); + lib[func] = async ? ff.async : ff; + } + }); + return lib; +} export default class SeqWrapper { - // private handle: ForeignFunction; - constructor() { - // const dl = new DynamicLibrary('/path/to/libseqjit.so', RTLD_GLOBAL); - // const init_fn = new ForeignFunction(dl.get('seq_jit_init'), refType(types.void), []); - // this.handle = init_fn(); + /** + * SeqWrapper + * Wrapper class for Seq library. Uses node-ffi and ref to interface with the '.so' file. + */ + connection: IConnection; + lib: any; + // handle: Buffer; + constructor(connection: IConnection) { + const voidPtr = refType(types.void); + this.connection = connection; + this.connection.console.log('hello from wrapper'); + // TODO: Better way of geting the path to libseqjit.so + this.lib = GlobalLibrary('/path/to/libseqjit', { + 'seq_jit_init': [voidPtr, []], + 'seq_jit_complete': [types.CString, [voidPtr, types.CString]], + 'seq_jit_document': [types.CString, [voidPtr, types.CString]] + }); + // TODO: The following line causes the server to crash. + // this.handle = this.lib.seq_jit_init(); } + complete(prefix: string): string[] { return []; } + document(idn: string): string { return ''; }