From 485e0d07aaf42981bdd10eda1091e6ae60fc10ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamila=20Sou=C4=8Dkov=C3=A1?= Date: Sat, 31 Jan 2015 23:34:44 +0100 Subject: [PATCH 01/46] HTTP API: COMMIT ALL THE THINGS! --- .gitignore | 1 + config.py.example | 11 ++++++ gateserver/__init__.py | 0 gateserver/controller_api.py | 5 +++ gateserver/db.py | 67 ++++++++++++++++++++++++++++++++++++ gateserver/http_api.py | 48 ++++++++++++++++++++++++++ gateserver/models.py | 22 ++++++++++++ requirements.txt | 1 + run.py | 14 ++++++++ 9 files changed, 169 insertions(+) create mode 100644 config.py.example create mode 100644 gateserver/__init__.py create mode 100644 gateserver/controller_api.py create mode 100644 gateserver/db.py create mode 100644 gateserver/http_api.py create mode 100644 gateserver/models.py create mode 100755 run.py diff --git a/.gitignore b/.gitignore index 077a157..5e35888 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ *.py[co] __pycache__/ venv/ +config.py diff --git a/config.py.example b/config.py.example new file mode 100644 index 0000000..dbb5b1a --- /dev/null +++ b/config.py.example @@ -0,0 +1,11 @@ +"""The server configuration.""" + +http_api = { + 'port': 5047, +} + +controller_api = { + 'port': 5042, +} + +db_url = 'postgresql://user:password@host/dbname' diff --git a/gateserver/__init__.py b/gateserver/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/gateserver/controller_api.py b/gateserver/controller_api.py new file mode 100644 index 0000000..bc020d8 --- /dev/null +++ b/gateserver/controller_api.py @@ -0,0 +1,5 @@ +"""The UDP server that provides the API for the controllers.""" + +def serve(config): + # TODO + print('controller_api would start serving') diff --git a/gateserver/db.py b/gateserver/db.py new file mode 100644 index 0000000..6cec8ca --- /dev/null +++ b/gateserver/db.py @@ -0,0 +1,67 @@ +"""Manages the DB -- holds the connection and provides CRUD primitives.""" + +import config + +import psycopg2 + +db = psycopg2.connect(config.db_url) +db.autocommit=True + +def exec_sql(query, args=(), ret=True): + """Execute the query, returning the result as a list if `ret` is set.""" + with db.cursor() as cur: + cur.execute(query, args) + if ret: return cur.fetchall() + +################################################################################ + +def dict_intersect(d, f): + """Only keeps those keys of d which are present in f. Values stay intact.""" + r = {} + for k, v in d.items(): + if k in f: + r[k] = v + return r + +def unzip(lst): + return zip(*lst) + +class StoredModelMixin: + """Defines CRUD methods for an object. + + The object must define `_table` -- the table name, and `_attrs`: a dict of + name: type for the column types. + + String interpolation is used with `_table` and `_attrs`, therefore THESE + PROPERTIES MUST NEVER BE SET WITH USER-SUPPLIED DATA. + """ + + def __init__(self): + cols = ','.join([ k+' '+v for k, v in self._attrs.items() ]) + exec_sql("CREATE TABLE IF NOT EXISTS {} ({})".format(self._table, cols), + ret=False) + + def get(self, id=None, values={}): + if values == {}: values = self._attrs + cols = list(dict_intersect(self._attrs, values).keys()) + vs = ','.join(cols) + w = 'WHERE id = %s' if id else '' + r = exec_sql('SELECT {} FROM {} {}'.format(vs, self._table, w), (id,)) + rr = [ dict(zip(cols, s)) for s in r ] # {key: value} instead of tuples + if id: + if rr == []: return None + else: return rr[0] + else: return rr + + def create(self, id, **kwargs): + args = dict_intersect(kwargs, self._attrs) + args['id'] = id + keys, values = unzip(list(args.items())) + ks = ','.join(keys); vs = ','.join(['%s']*len(values)) + + exec_sql('INSERT INTO {} ({}) VALUES ({})'.format(self._table, ks, vs), + values, ret=False) + + def delete(self, id): + exec_sql('DELETE FROM {} WHERE id = %s'.format(self._table), (id,), + ret=False) diff --git a/gateserver/http_api.py b/gateserver/http_api.py new file mode 100644 index 0000000..230face --- /dev/null +++ b/gateserver/http_api.py @@ -0,0 +1,48 @@ +"""Defines the REST API for CRUD and management.""" + +from .models import Controller # TODO ...and others, once they exist + +import cherrypy +import psycopg2 +import config + +class RestMixin(): + """Mixin to expose the given model via REST.""" + + exposed = True + + @cherrypy.tools.json_out() + def GET(self, id=None): + r = self.get(id) + if r == None: raise cherrypy.NotFound() + else: return r + + @cherrypy.tools.json_in() + def PUT(self, id): + try: + return self.create(id, **cherrypy.request.json) + except psycopg2.IntegrityError as e: + raise cherrypy.HTTPError("409 Conflict", e.pgerror) + + def DELETE(self, id): + self.delete(id) + +def expose_at(path): + def mount(cls): + cherrypy.tree.mount(cls(), path, {'/': cherrypy_model_endpoint_conf}) + return mount + +################################################################################ + +cherrypy_model_endpoint_conf = { + 'request.dispatch': cherrypy.dispatch.MethodDispatcher(), +} + +@expose_at('/controller') +class ControllerAPI(RestMixin, Controller): pass + +################################################################################ + +def serve(config): + cherrypy.config.update({'server.socket_port': config['port'],}) + cherrypy.engine.start() diff --git a/gateserver/models.py b/gateserver/models.py new file mode 100644 index 0000000..93e1b71 --- /dev/null +++ b/gateserver/models.py @@ -0,0 +1,22 @@ +""" +The data models. +""" + +from . import db + +import nacl.raw as nacl + +class Controller(db.StoredModelMixin): + _table = 'controller' + _attrs = { 'id' : 'macaddr PRIMARY KEY', + 'ip' : 'inet UNIQUE', + 'key' : 'bytea', + 'name': 'text', + } + + def get(self, id=None, values={'id', 'ip', 'name'}): + return super().get(id, values) + + def create(self, id, **kwargs): + kwargs['key'] = nacl.randombytes(nacl.crypto_secretbox_KEYBYTES) + super().create(id, **kwargs) diff --git a/requirements.txt b/requirements.txt index 24ed765..145510f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ +CherryPy==3.6.0 psycopg2==2.5.4 https://github.com/warner/python-tweetnacl/tarball/b48a25a33f diff --git a/run.py b/run.py new file mode 100755 index 0000000..1a003a0 --- /dev/null +++ b/run.py @@ -0,0 +1,14 @@ +#!/usr/bin/env python3 +"""Gate server runner.""" + +from gateserver import controller_api, http_api +import config + +import psycopg2 + +def serve(): + controller_api.serve(config.controller_api) + http_api.serve(config.http_api) + +if __name__ == '__main__': + serve() From ce90890fe12b0f90dc0e7f336b1f470c1ddd2e7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamila=20Sou=C4=8Dkov=C3=A1?= Date: Sat, 31 Jan 2015 23:51:22 +0100 Subject: [PATCH 02/46] fix mutable default args ~_~ --- gateserver/db.py | 4 ++-- gateserver/models.py | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/gateserver/db.py b/gateserver/db.py index 6cec8ca..369145d 100644 --- a/gateserver/db.py +++ b/gateserver/db.py @@ -41,8 +41,8 @@ def __init__(self): exec_sql("CREATE TABLE IF NOT EXISTS {} ({})".format(self._table, cols), ret=False) - def get(self, id=None, values={}): - if values == {}: values = self._attrs + def get(self, id=None, values=None): + if values == None: values = self._attrs cols = list(dict_intersect(self._attrs, values).keys()) vs = ','.join(cols) w = 'WHERE id = %s' if id else '' diff --git a/gateserver/models.py b/gateserver/models.py index 93e1b71..1c12d78 100644 --- a/gateserver/models.py +++ b/gateserver/models.py @@ -14,7 +14,8 @@ class Controller(db.StoredModelMixin): 'name': 'text', } - def get(self, id=None, values={'id', 'ip', 'name'}): + def get(self, id=None, values=None): + if values == None: values = {'id', 'ip', 'name'} return super().get(id, values) def create(self, id, **kwargs): From 6a4556a10ddc807708abfa0e44a9f20ae106726c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamila=20Sou=C4=8Dkov=C3=A1?= Date: Sun, 1 Feb 2015 00:04:15 +0100 Subject: [PATCH 03/46] update README --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index c64e5c2..9323724 100644 --- a/README.md +++ b/README.md @@ -15,5 +15,7 @@ Setup - fish: `. venv/bin/activate.fish` - csh, tcsh: `source venv/bin/activate.csh` 3. Install dependencies if necessary: `pip install -r requirements.txt` +4. run with `./run.py` (or `while true; do sleep 0.1; ./run.py; done`, as the + server will stop when the code changes => auto-restart on save ^_^) The rest doesn't exist yet. From f8dc05245f2f9d441c1f2317846866325f0927c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamila=20Sou=C4=8Dkov=C3=A1?= Date: Sun, 1 Feb 2015 09:12:49 +0100 Subject: [PATCH 04/46] db.py: better dict_intersect() --- gateserver/db.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/gateserver/db.py b/gateserver/db.py index 369145d..bfab50a 100644 --- a/gateserver/db.py +++ b/gateserver/db.py @@ -17,11 +17,7 @@ def exec_sql(query, args=(), ret=True): def dict_intersect(d, f): """Only keeps those keys of d which are present in f. Values stay intact.""" - r = {} - for k, v in d.items(): - if k in f: - r[k] = v - return r + return { k: v for k, v in d.items() if k in f } def unzip(lst): return zip(*lst) From a2eb8c9a0b9c209e978cabb4b8a95cc180ded82d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamila=20Sou=C4=8Dkov=C3=A1?= Date: Sun, 1 Feb 2015 09:42:52 +0100 Subject: [PATCH 05/46] update README, add "to do next" --- README.md | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 9323724..1b425d4 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,17 @@ Setup - fish: `. venv/bin/activate.fish` - csh, tcsh: `source venv/bin/activate.csh` 3. Install dependencies if necessary: `pip install -r requirements.txt` -4. run with `./run.py` (or `while true; do sleep 0.1; ./run.py; done`, as the +4. configure: `cp config.py{.example,}; $EDITOR config.py` +5. run with `./run.py` (or `while true; do sleep 0.1; ./run.py; done`, as the server will stop when the code changes => auto-restart on save ^_^) The rest doesn't exist yet. + +Next to do: +----------- + +- tests +- listen on UDP socket +- wrap/unwrap NaCl +- fix DB singleton (who wants a singleton?!) -- inversion of control +- CI From 950e82da3ba9f6773d02b7cc07cea544844be4ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamila=20Sou=C4=8Dkov=C3=A1?= Date: Sun, 1 Feb 2015 16:07:44 +0100 Subject: [PATCH 06/46] cosmetic changes in db.py --- gateserver/db.py | 41 +++++++++++++++++------------------------ 1 file changed, 17 insertions(+), 24 deletions(-) diff --git a/gateserver/db.py b/gateserver/db.py index bfab50a..c4ec2e3 100644 --- a/gateserver/db.py +++ b/gateserver/db.py @@ -7,7 +7,7 @@ db = psycopg2.connect(config.db_url) db.autocommit=True -def exec_sql(query, args=(), ret=True): +def exec_sql(query, args=(), ret=False): """Execute the query, returning the result as a list if `ret` is set.""" with db.cursor() as cur: cur.execute(query, args) @@ -15,10 +15,6 @@ def exec_sql(query, args=(), ret=True): ################################################################################ -def dict_intersect(d, f): - """Only keeps those keys of d which are present in f. Values stay intact.""" - return { k: v for k, v in d.items() if k in f } - def unzip(lst): return zip(*lst) @@ -34,30 +30,27 @@ class StoredModelMixin: def __init__(self): cols = ','.join([ k+' '+v for k, v in self._attrs.items() ]) - exec_sql("CREATE TABLE IF NOT EXISTS {} ({})".format(self._table, cols), - ret=False) - - def get(self, id=None, values=None): - if values == None: values = self._attrs - cols = list(dict_intersect(self._attrs, values).keys()) - vs = ','.join(cols) - w = 'WHERE id = %s' if id else '' - r = exec_sql('SELECT {} FROM {} {}'.format(vs, self._table, w), (id,)) - rr = [ dict(zip(cols, s)) for s in r ] # {key: value} instead of tuples + exec_sql("CREATE TABLE IF NOT EXISTS {} ({})".format(self._table, cols)) + + def get(self, id=None, cols=None): + if cols == None: cols = self._attrs + assert(set(cols) <= set(self._attrs)) + cols = list(cols) + q = 'SELECT {} FROM {}'.format(','.join(cols), self._table) + if id: q += ' WHERE id = %s' + r = [ dict(zip(cols, s)) for s in exec_sql(q, (id,), ret=True) ] if id: - if rr == []: return None - else: return rr[0] - else: return rr + if r == []: return None + else: return r[0] + else: return r - def create(self, id, **kwargs): - args = dict_intersect(kwargs, self._attrs) + def create(self, id, **args): + assert(set(args) <= set(self._attrs)) args['id'] = id keys, values = unzip(list(args.items())) ks = ','.join(keys); vs = ','.join(['%s']*len(values)) - exec_sql('INSERT INTO {} ({}) VALUES ({})'.format(self._table, ks, vs), - values, ret=False) + values) def delete(self, id): - exec_sql('DELETE FROM {} WHERE id = %s'.format(self._table), (id,), - ret=False) + exec_sql('DELETE FROM {} WHERE id = %s'.format(self._table), (id,)) From 0b4e66e88d29d6b7038bdf711a7d9f945809fc71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamila=20Sou=C4=8Dkov=C3=A1?= Date: Sun, 1 Feb 2015 20:03:58 +0100 Subject: [PATCH 07/46] http_api: add stop() function --- gateserver/http_api.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/gateserver/http_api.py b/gateserver/http_api.py index 230face..070fb9d 100644 --- a/gateserver/http_api.py +++ b/gateserver/http_api.py @@ -46,3 +46,6 @@ class ControllerAPI(RestMixin, Controller): pass def serve(config): cherrypy.config.update({'server.socket_port': config['port'],}) cherrypy.engine.start() + +def stop(): + cherrypy.engine.exit() From 8abc8305882230335619f8b9f07632e0e1c53890 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamila=20Sou=C4=8Dkov=C3=A1?= Date: Fri, 6 Feb 2015 13:56:09 +0100 Subject: [PATCH 08/46] screw my mini ORM, this stuff is simple --- bootstrap.py | 19 ++++++++++ gateserver/db.py | 57 ++++++------------------------ gateserver/http_api.py | 80 +++++++++++++++++++++++++++++------------- gateserver/models.py | 23 ------------ run.py | 5 ++- tables.sql | 1 + 6 files changed, 88 insertions(+), 97 deletions(-) create mode 100755 bootstrap.py delete mode 100644 gateserver/models.py create mode 100644 tables.sql diff --git a/bootstrap.py b/bootstrap.py new file mode 100755 index 0000000..11e79e0 --- /dev/null +++ b/bootstrap.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python3 + +import config +from gateserver import db + +def db_create_tables(): + print('Creating DB tables...', end='') + with open('./tables.sql', 'r') as f: + if not db.conn: db.connect(config.db_url) + for line in f: + line = line.split('#')[0].strip() + if line: db.exec_sql('CREATE TABLE IF NOT EXISTS ' + line) + print(' OK') + +def all(): + db_create_tables() + +if __name__ == '__main__': + all() diff --git a/gateserver/db.py b/gateserver/db.py index c4ec2e3..423214a 100644 --- a/gateserver/db.py +++ b/gateserver/db.py @@ -1,56 +1,21 @@ -"""Manages the DB -- holds the connection and provides CRUD primitives.""" +"""Holds the (global) connection to the DB.""" +# TODO maybe use a connection pool one beautiful day import config - import psycopg2 -db = psycopg2.connect(config.db_url) -db.autocommit=True +conn = None + +def connect(db_url): + global conn + conn = psycopg2.connect(db_url) + conn.autocommit=True def exec_sql(query, args=(), ret=False): """Execute the query, returning the result as a list if `ret` is set.""" - with db.cursor() as cur: + with conn.cursor() as cur: cur.execute(query, args) if ret: return cur.fetchall() -################################################################################ - -def unzip(lst): - return zip(*lst) - -class StoredModelMixin: - """Defines CRUD methods for an object. - - The object must define `_table` -- the table name, and `_attrs`: a dict of - name: type for the column types. - - String interpolation is used with `_table` and `_attrs`, therefore THESE - PROPERTIES MUST NEVER BE SET WITH USER-SUPPLIED DATA. - """ - - def __init__(self): - cols = ','.join([ k+' '+v for k, v in self._attrs.items() ]) - exec_sql("CREATE TABLE IF NOT EXISTS {} ({})".format(self._table, cols)) - - def get(self, id=None, cols=None): - if cols == None: cols = self._attrs - assert(set(cols) <= set(self._attrs)) - cols = list(cols) - q = 'SELECT {} FROM {}'.format(','.join(cols), self._table) - if id: q += ' WHERE id = %s' - r = [ dict(zip(cols, s)) for s in exec_sql(q, (id,), ret=True) ] - if id: - if r == []: return None - else: return r[0] - else: return r - - def create(self, id, **args): - assert(set(args) <= set(self._attrs)) - args['id'] = id - keys, values = unzip(list(args.items())) - ks = ','.join(keys); vs = ','.join(['%s']*len(values)) - exec_sql('INSERT INTO {} ({}) VALUES ({})'.format(self._table, ks, vs), - values) - - def delete(self, id): - exec_sql('DELETE FROM {} WHERE id = %s'.format(self._table), (id,)) +# thrown when constraints aren't satisfied +IntegrityError = psycopg2.IntegrityError diff --git a/gateserver/http_api.py b/gateserver/http_api.py index 070fb9d..578ede1 100644 --- a/gateserver/http_api.py +++ b/gateserver/http_api.py @@ -1,50 +1,80 @@ """Defines the REST API for CRUD and management.""" -from .models import Controller # TODO ...and others, once they exist - +from . import db +import nacl.raw as nacl import cherrypy -import psycopg2 -import config -class RestMixin(): - """Mixin to expose the given model via REST.""" +class MountPoint: + """Represents a mount point, or path prefix, for attaching resources to.""" + pass +class Resource(MountPoint): + """Represents a REST resource.""" exposed = True +class CRUDResource(Resource): + """Represents a REST resource that exposes a DB table's CRUD methods.""" + def __init__(self, tbl, put_columns, get_columns, on_save=lambda x: x): + assert(tbl.isidentifier()) + self.table = tbl + self.put_columns = put_columns + self.get_columns = get_columns + self.on_save = on_save + @cherrypy.tools.json_out() def GET(self, id=None): - r = self.get(id) - if r == None: raise cherrypy.NotFound() - else: return r + cols = list(self.get_columns) + q = 'SELECT {} FROM {}'.format(','.join(cols), self.table) + if id: q += ' WHERE id = %s' + rs = [ dict(zip(cols, r)) for r in db.exec_sql(q, (id,), ret=True) ] + if id: + if len(rs) < 1: raise cherrypy.HTTPError('404 Not Found') + return rs[0] + else: return rs @cherrypy.tools.json_in() + @cherrypy.tools.json_out() def PUT(self, id): + json = self.on_save(dict(cherrypy.request.json, id=id)) + print(json) + cols, values, ps = [], [], [] + for c in self.put_columns: + cols.append(c) + values.append(json.get(c)) + ps.append('%s') + q = 'INSERT INTO controller ({}) VALUES ({})'.format(','.join(cols), + ','.join(ps)) try: - return self.create(id, **cherrypy.request.json) - except psycopg2.IntegrityError as e: - raise cherrypy.HTTPError("409 Conflict", e.pgerror) + db.exec_sql(q, values) + except db.IntegrityError as e: + raise cherrypy.HTTPError('400 Bad Request', e.pgerror) + return { 'url': cherrypy.url() } - def DELETE(self, id): - self.delete(id) + # TODO POST -def expose_at(path): - def mount(cls): - cherrypy.tree.mount(cls(), path, {'/': cherrypy_model_endpoint_conf}) - return mount + def DELETE(self, id): + db.exec_sql('DELETE FROM {} WHERE id = %s'.format(self.table), (id,)) ################################################################################ -cherrypy_model_endpoint_conf = { - 'request.dispatch': cherrypy.dispatch.MethodDispatcher(), -} - -@expose_at('/controller') -class ControllerAPI(RestMixin, Controller): pass +api_root = MountPoint() +api_root.controller = CRUDResource('controller', + get_columns={'id', 'ip', 'name'}, + put_columns={'id', 'ip', 'key', 'name'}, + on_save=lambda ctrl: + dict(ctrl, key=nacl.randombytes(nacl.crypto_secretbox_KEYBYTES))) ################################################################################ +cherrypy_conf = { + '/': { + 'request.dispatch': cherrypy.dispatch.MethodDispatcher(), + } +} +cherrypy.tree.mount(api_root, '/', cherrypy_conf) + def serve(config): - cherrypy.config.update({'server.socket_port': config['port'],}) + cherrypy.config.update({'server.socket_port': config['port']}) cherrypy.engine.start() def stop(): diff --git a/gateserver/models.py b/gateserver/models.py deleted file mode 100644 index 1c12d78..0000000 --- a/gateserver/models.py +++ /dev/null @@ -1,23 +0,0 @@ -""" -The data models. -""" - -from . import db - -import nacl.raw as nacl - -class Controller(db.StoredModelMixin): - _table = 'controller' - _attrs = { 'id' : 'macaddr PRIMARY KEY', - 'ip' : 'inet UNIQUE', - 'key' : 'bytea', - 'name': 'text', - } - - def get(self, id=None, values=None): - if values == None: values = {'id', 'ip', 'name'} - return super().get(id, values) - - def create(self, id, **kwargs): - kwargs['key'] = nacl.randombytes(nacl.crypto_secretbox_KEYBYTES) - super().create(id, **kwargs) diff --git a/run.py b/run.py index 1a003a0..849f233 100755 --- a/run.py +++ b/run.py @@ -1,12 +1,11 @@ #!/usr/bin/env python3 """Gate server runner.""" -from gateserver import controller_api, http_api +from gateserver import db, controller_api, http_api import config -import psycopg2 - def serve(): + db.connect(config.db_url) controller_api.serve(config.controller_api) http_api.serve(config.http_api) diff --git a/tables.sql b/tables.sql new file mode 100644 index 0000000..c212a66 --- /dev/null +++ b/tables.sql @@ -0,0 +1 @@ +controller (id macaddr PRIMARY KEY, ip inet UNIQUE NOT NULL, key bytea NOT NULL, name text) From cc2ee8aa918bf2168bc24b97533ef6ef909ba34b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamila=20Sou=C4=8Dkov=C3=A1?= Date: Fri, 6 Feb 2015 16:51:56 +0100 Subject: [PATCH 09/46] add logs DB table + resource --- gateserver/http_api.py | 7 +++++++ tables.sql | 1 + 2 files changed, 8 insertions(+) diff --git a/gateserver/http_api.py b/gateserver/http_api.py index 578ede1..e3d6292 100644 --- a/gateserver/http_api.py +++ b/gateserver/http_api.py @@ -12,6 +12,12 @@ class Resource(MountPoint): """Represents a REST resource.""" exposed = True +class Log(Resource): + @cherrypy.tools.json_out() + def GET(self, entries=100): + return db.exec_sql('SELECT * FROM log ORDER BY time DESC LIMIT %s', + (entries,), ret=True) + class CRUDResource(Resource): """Represents a REST resource that exposes a DB table's CRUD methods.""" def __init__(self, tbl, put_columns, get_columns, on_save=lambda x: x): @@ -63,6 +69,7 @@ def DELETE(self, id): put_columns={'id', 'ip', 'key', 'name'}, on_save=lambda ctrl: dict(ctrl, key=nacl.randombytes(nacl.crypto_secretbox_KEYBYTES))) +api_root.log = Log() ################################################################################ diff --git a/tables.sql b/tables.sql index c212a66..cc27ad5 100644 --- a/tables.sql +++ b/tables.sql @@ -1 +1,2 @@ controller (id macaddr PRIMARY KEY, ip inet UNIQUE NOT NULL, key bytea NOT NULL, name text) +log (time timestamp NOT NULL, ctrl_id macaddr REFERENCES controller, message text) From d8a8095588cfbe7a876e50482bad60cfafcb30a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamila=20Sou=C4=8Dkov=C3=A1?= Date: Fri, 6 Feb 2015 16:55:15 +0100 Subject: [PATCH 10/46] add tests (yay!) --- .gitignore | 1 + config.py.example | 7 +++++++ requirements.txt | 6 ++++++ tests/__init__.py | 0 tests/test_http_api.py | 40 ++++++++++++++++++++++++++++++++++++++++ 5 files changed, 54 insertions(+) create mode 100644 tests/__init__.py create mode 100644 tests/test_http_api.py diff --git a/.gitignore b/.gitignore index 5e35888..c65d7c3 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ __pycache__/ venv/ config.py +.coverage diff --git a/config.py.example b/config.py.example index dbb5b1a..7897c74 100644 --- a/config.py.example +++ b/config.py.example @@ -9,3 +9,10 @@ controller_api = { } db_url = 'postgresql://user:password@host/dbname' + +tests = { + 'http_api': { + 'port': 9001, + }, + 'db_url': 'postgresql://user:password@localhost/gate_test', +} diff --git a/requirements.txt b/requirements.txt index 145510f..d6ad191 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,9 @@ CherryPy==3.6.0 +cov-core==1.15.0 +coverage==3.7.1 psycopg2==2.5.4 +py==1.4.26 +pytest==2.6.4 +pytest-cov==1.8.1 +requests==2.5.1 https://github.com/warner/python-tweetnacl/tarball/b48a25a33f diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_http_api.py b/tests/test_http_api.py new file mode 100644 index 0000000..d5d00f1 --- /dev/null +++ b/tests/test_http_api.py @@ -0,0 +1,40 @@ +import config +import bootstrap +import gateserver.db +import gateserver.http_api +import requests + +url = 'http://localhost:{}'.format(config.tests['http_api']['port']) + +def setup_module(module): + gateserver.db.connect(config.tests['db_url']) + bootstrap.db_create_tables() + gateserver.http_api.serve(config.tests['http_api']) + +def teardown_module(module): + gateserver.http_api.stop() + +def assert_req(method, url, status=200, expected_data=None, **kwargs): + print(kwargs) + r = requests.request(method, url, **kwargs) + assert r.status_code == status + if expected_data: assert r.json() == expected_data + +def test_controller_crud(): + cid = '00:00:00:00:00:00' + data = { 'id': cid, 'ip': '0.0.0.0', 'name': 'Test Controller' } + requests.delete(url+'/controller/'+cid) # just in case the last run didn't + + assert_req('PUT', url+'/controller/'+cid, json={'ip': data['ip'], + 'name': data['name']}, + expected_data={'url': url+'/controller/'+cid}) + assert_req('GET', url+'/controller/'+cid, expected_data=data) + assert_req('GET', url+'/controller/', expected_data=[data]) + assert_req('PUT', url+'/controller/'+cid, json={}, status=400) + assert_req('DELETE', url+'/controller/'+cid) + assert_req('GET', url+'/controller/'+cid, status=404) + +def test_log(): + r = requests.get(url+'/log/') + assert r.status_code == 200 + assert isinstance(r.json(), list) From f0d0fcaef3d441bb44e34c59358d64d8b66be737 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamila=20Sou=C4=8Dkov=C3=A1?= Date: Fri, 6 Feb 2015 17:01:14 +0100 Subject: [PATCH 11/46] mention bootstrap in README --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 1b425d4..cd48e06 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,8 @@ Setup - csh, tcsh: `source venv/bin/activate.csh` 3. Install dependencies if necessary: `pip install -r requirements.txt` 4. configure: `cp config.py{.example,}; $EDITOR config.py` -5. run with `./run.py` (or `while true; do sleep 0.1; ./run.py; done`, as the +5. bootstrap (create DB tables and such): `./bootstrap.py` +6. run with `./run.py` (or `while true; do sleep 0.1; ./run.py; done`, as the server will stop when the code changes => auto-restart on save ^_^) The rest doesn't exist yet. From f7967a4d8dd8719d280d6fc64b221ae067a704b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamila=20Sou=C4=8Dkov=C3=A1?= Date: Fri, 6 Feb 2015 18:52:34 +0100 Subject: [PATCH 12/46] fix .gitignore; remove unnecessary import; update README --- .gitignore | 4 ++-- README.md | 1 - gateserver/db.py | 1 - 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index c65d7c3..fa2e944 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,5 @@ *.py[co] -__pycache__/ venv/ config.py -.coverage +.* +!.git* diff --git a/README.md b/README.md index cd48e06..4563aa8 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,6 @@ The rest doesn't exist yet. Next to do: ----------- -- tests - listen on UDP socket - wrap/unwrap NaCl - fix DB singleton (who wants a singleton?!) -- inversion of control diff --git a/gateserver/db.py b/gateserver/db.py index 423214a..897866d 100644 --- a/gateserver/db.py +++ b/gateserver/db.py @@ -1,7 +1,6 @@ """Holds the (global) connection to the DB.""" # TODO maybe use a connection pool one beautiful day -import config import psycopg2 conn = None From eff9f7bbe10fc54099321fa924fcdaded9d9cb0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamila=20Sou=C4=8Dkov=C3=A1?= Date: Sat, 7 Feb 2015 14:05:00 +0100 Subject: [PATCH 13/46] new config format; 2 executables; UDP echo server; README update --- README.md | 17 ++++++++++++----- config.py.example | 19 +++---------------- gateserver/controller_api.py | 5 ----- gateserver/controller_server.py | 24 ++++++++++++++++++++++++ gateserver/http_api.py | 2 +- run.py | 13 ------------- runhttp.py | 9 +++++++++ runserver.py | 17 +++++++++++++++++ tests/config.py.example | 5 +++++ tests/test_http_api.py | 8 ++++---- 10 files changed, 75 insertions(+), 44 deletions(-) delete mode 100644 gateserver/controller_api.py create mode 100644 gateserver/controller_server.py delete mode 100755 run.py create mode 100755 runhttp.py create mode 100755 runserver.py create mode 100644 tests/config.py.example diff --git a/README.md b/README.md index 4563aa8..f0cffcc 100644 --- a/README.md +++ b/README.md @@ -17,15 +17,22 @@ Setup 3. Install dependencies if necessary: `pip install -r requirements.txt` 4. configure: `cp config.py{.example,}; $EDITOR config.py` 5. bootstrap (create DB tables and such): `./bootstrap.py` -6. run with `./run.py` (or `while true; do sleep 0.1; ./run.py; done`, as the - server will stop when the code changes => auto-restart on save ^_^) +6. run with `./runserver.py`; + run the HTTP API server with `./runhttp.py` -The rest doesn't exist yet. +Running tests +------------- + +1. Edit configuration: `cp tests/config.py{.example,}; $EDITOR tests/config.py` + A real Postgres DB is used, you need to specify the connection string. +2. run with `py.test tests/` + or `py.test --cov gateserver/ --cov-report term-missing tests/` for coverage report Next to do: ----------- -- listen on UDP socket +- the controller end - wrap/unwrap NaCl -- fix DB singleton (who wants a singleton?!) -- inversion of control +- HTTP: rewrite to use Werkzeug instead of CherryPy +- fix DB singleton (who wants a singleton?!) - CI diff --git a/config.py.example b/config.py.example index 7897c74..3da37ef 100644 --- a/config.py.example +++ b/config.py.example @@ -1,18 +1,5 @@ """The server configuration.""" -http_api = { - 'port': 5047, -} - -controller_api = { - 'port': 5042, -} - -db_url = 'postgresql://user:password@host/dbname' - -tests = { - 'http_api': { - 'port': 9001, - }, - 'db_url': 'postgresql://user:password@localhost/gate_test', -} +http_port = 5047 +udp_port = 5042 +db_url = 'postgresql://user:password@localhost/gate' diff --git a/gateserver/controller_api.py b/gateserver/controller_api.py deleted file mode 100644 index bc020d8..0000000 --- a/gateserver/controller_api.py +++ /dev/null @@ -1,5 +0,0 @@ -"""The UDP server that provides the API for the controllers.""" - -def serve(config): - # TODO - print('controller_api would start serving') diff --git a/gateserver/controller_server.py b/gateserver/controller_server.py new file mode 100644 index 0000000..301fec9 --- /dev/null +++ b/gateserver/controller_server.py @@ -0,0 +1,24 @@ +"""The UDP server that provides the API for the controllers.""" + +#from . import controller_api +import socketserver +import logging + +log = logging.getLogger('server') + +class MessageHandler(socketserver.BaseRequestHandler): + """Handles a message from the controller. + + Behaves according to + https://github.com/fmfi-svt/gate/wiki/Controller-%E2%86%94-Server-Protocol . + """ + + def handle(self): + data, socket = self.request + socket.sendto(data, self.client_address) + log.info(data, extra=dict(ip=self.client_address[0], status='OK')) + +def serve(config): + bind_addr = '0.0.0.0', config.udp_port + server = socketserver.ThreadingUDPServer(bind_addr, MessageHandler) + server.serve_forever() diff --git a/gateserver/http_api.py b/gateserver/http_api.py index e3d6292..abc85de 100644 --- a/gateserver/http_api.py +++ b/gateserver/http_api.py @@ -81,7 +81,7 @@ def DELETE(self, id): cherrypy.tree.mount(api_root, '/', cherrypy_conf) def serve(config): - cherrypy.config.update({'server.socket_port': config['port']}) + cherrypy.config.update({'server.socket_port': config.http_port}) cherrypy.engine.start() def stop(): diff --git a/run.py b/run.py deleted file mode 100755 index 849f233..0000000 --- a/run.py +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env python3 -"""Gate server runner.""" - -from gateserver import db, controller_api, http_api -import config - -def serve(): - db.connect(config.db_url) - controller_api.serve(config.controller_api) - http_api.serve(config.http_api) - -if __name__ == '__main__': - serve() diff --git a/runhttp.py b/runhttp.py new file mode 100755 index 0000000..97ee792 --- /dev/null +++ b/runhttp.py @@ -0,0 +1,9 @@ +#!/usr/bin/env python3 +"""Gate HTTP API server runner.""" + +from gateserver import db, http_api +import config + +if __name__ == '__main__': + db.connect(config.db_url) + http_api.serve(config) diff --git a/runserver.py b/runserver.py new file mode 100755 index 0000000..2d1b6ba --- /dev/null +++ b/runserver.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python3 +"""Gate server runner.""" + +from gateserver import db, controller_server +import config +import logging + +logging.basicConfig(level=logging.DEBUG, + format='%(asctime)s %(levelname)s %(name)s %(ip)s %(message)s %(status)s', + datefmt='%Y-%m-%d %H:%M:%S') + +if __name__ == '__main__': + try: + db.connect(config.db_url) + controller_server.serve(config) + except (SystemExit, KeyboardInterrupt): + logging.shutdown() diff --git a/tests/config.py.example b/tests/config.py.example new file mode 100644 index 0000000..c082e5c --- /dev/null +++ b/tests/config.py.example @@ -0,0 +1,5 @@ +"""Tests configuration.""" + +http_port = 9047 +udp_port = 9042 +db_url = 'postgresql://user:password@localhost/gate_test' diff --git a/tests/test_http_api.py b/tests/test_http_api.py index d5d00f1..e539992 100644 --- a/tests/test_http_api.py +++ b/tests/test_http_api.py @@ -1,15 +1,15 @@ -import config +import tests.config as config import bootstrap import gateserver.db import gateserver.http_api import requests -url = 'http://localhost:{}'.format(config.tests['http_api']['port']) +url = 'http://localhost:{}'.format(config.http_port) def setup_module(module): - gateserver.db.connect(config.tests['db_url']) + gateserver.db.connect(config.db_url) bootstrap.db_create_tables() - gateserver.http_api.serve(config.tests['http_api']) + gateserver.http_api.serve(config) def teardown_module(module): gateserver.http_api.stop() From 250e72224b4cff9086eae7dc5d81e3e9644c9c48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamila=20Sou=C4=8Dkov=C3=A1?= Date: Sat, 7 Feb 2015 14:07:02 +0100 Subject: [PATCH 14/46] small changes in db and http_api --- gateserver/db.py | 5 +++-- gateserver/http_api.py | 5 ++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/gateserver/db.py b/gateserver/db.py index 897866d..34ce816 100644 --- a/gateserver/db.py +++ b/gateserver/db.py @@ -2,13 +2,14 @@ # TODO maybe use a connection pool one beautiful day import psycopg2 +from psycopg2.extras import RealDictCursor # results as dict instead of tuple conn = None def connect(db_url): global conn - conn = psycopg2.connect(db_url) - conn.autocommit=True + conn = psycopg2.connect(db_url, cursor_factory=RealDictCursor) + conn.autocommit = True def exec_sql(query, args=(), ret=False): """Execute the query, returning the result as a list if `ret` is set.""" diff --git a/gateserver/http_api.py b/gateserver/http_api.py index abc85de..8c78511 100644 --- a/gateserver/http_api.py +++ b/gateserver/http_api.py @@ -6,7 +6,6 @@ class MountPoint: """Represents a mount point, or path prefix, for attaching resources to.""" - pass class Resource(MountPoint): """Represents a REST resource.""" @@ -32,7 +31,7 @@ def GET(self, id=None): cols = list(self.get_columns) q = 'SELECT {} FROM {}'.format(','.join(cols), self.table) if id: q += ' WHERE id = %s' - rs = [ dict(zip(cols, r)) for r in db.exec_sql(q, (id,), ret=True) ] + rs = db.exec_sql(q, (id,), ret=True) if id: if len(rs) < 1: raise cherrypy.HTTPError('404 Not Found') return rs[0] @@ -53,7 +52,7 @@ def PUT(self, id): try: db.exec_sql(q, values) except db.IntegrityError as e: - raise cherrypy.HTTPError('400 Bad Request', e.pgerror) + raise cherrypy.HTTPError('400 Bad Request', e.pgerror) from e return { 'url': cherrypy.url() } # TODO POST From 721fc62954e7b6c4eb08bcbbac31bbd80273781e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamila=20Sou=C4=8Dkov=C3=A1?= Date: Sat, 7 Feb 2015 14:13:43 +0100 Subject: [PATCH 15/46] mv tables.{sql,txt} --- bootstrap.py | 2 +- tables.sql => tables.txt | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) rename tables.sql => tables.txt (67%) diff --git a/bootstrap.py b/bootstrap.py index 11e79e0..660592c 100755 --- a/bootstrap.py +++ b/bootstrap.py @@ -5,7 +5,7 @@ def db_create_tables(): print('Creating DB tables...', end='') - with open('./tables.sql', 'r') as f: + with open('./tables.txt', 'r') as f: if not db.conn: db.connect(config.db_url) for line in f: line = line.split('#')[0].strip() diff --git a/tables.sql b/tables.txt similarity index 67% rename from tables.sql rename to tables.txt index cc27ad5..a1bf116 100644 --- a/tables.sql +++ b/tables.txt @@ -1,2 +1,4 @@ +# SQL (CREATE TABLE) syntax (these are executed during bootstrap) +# one line per table controller (id macaddr PRIMARY KEY, ip inet UNIQUE NOT NULL, key bytea NOT NULL, name text) log (time timestamp NOT NULL, ctrl_id macaddr REFERENCES controller, message text) From eb91e0a11897a5be946f239eb547547b7ea2ad00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamila=20Sou=C4=8Dkov=C3=A1?= Date: Thu, 12 Feb 2015 23:27:19 +0100 Subject: [PATCH 16/46] add packet parsing --- gateserver/controller_api.py | 65 ++++++++++++++++++++++++++++++++ gateserver/controller_server.py | 9 +++-- gateserver/utils/structparse.py | 43 +++++++++++++++++++++ packet_example.bin | Bin 0 -> 42 bytes runserver.py | 7 +--- udpclient.py | 13 +++++++ 6 files changed, 127 insertions(+), 10 deletions(-) create mode 100644 gateserver/controller_api.py create mode 100644 gateserver/utils/structparse.py create mode 100644 packet_example.bin create mode 100644 udpclient.py diff --git a/gateserver/controller_api.py b/gateserver/controller_api.py new file mode 100644 index 0000000..79fa2c6 --- /dev/null +++ b/gateserver/controller_api.py @@ -0,0 +1,65 @@ +from .utils.structparse import * +from . import db +import nacl.raw as nacl + +PROTOCOL_VERSION = bytes([0x00,0x01]) + +class MsgType(Enum): + OPEN = 1 + +class ReplyStatus(Enum): + OK = 0x01 + ERR = 0x10 + TRY_AGAIN = 0x11 +S = ReplyStatus + +PacketHead = mystruct('PacketHead', + ['protocol_version', 'controllerID', 'nonce' ], + [ t.bytes(2) , t.bytes(6) , t.bytes(18) ]) + +RequestHead = mystruct('RequestHead', + ['msg_type'], + [ t.uint8 ]) + +ReplyHead = mystruct('ReplyHead', + ['msg_type', 'status' ], + [ t.uint8 , t.uint8 ]) + +def parse_request_packet(buf): + p, payload = PacketHead.unpack_from(buf) + #cntrl = db.exec_sql('SELECT key FROM controller WHERE id = %s', to_mac(p.controllerID)) + #if len(cntrl) != 1: raise BadMessageError('unknown controllerID ' + p.controllerID) + #key = cntrl[0].key + #payload = nacl.crypto_secretbox_open(payload, p.nonce, key) + checkmsg(p.protocol_version == PROTOCOL_VERSION, 'Invalid protocol version') + checkmsg(p.nonce[-1] & 0x1 == 0, 'Last bit of request nonce must be 0') + r, data = RequestHead.unpack_from(payload) + try: + t = MsgType(r.msg_type) + except ValueError: + raise BadMessageError('Unknown request type {}'.format(r.msg_type)) + return p, r, t, data + +def make_reply_packet(packet_head, request_head, status, data): + """Requires `packet_head` and `request_head` to be valid.""" + nnonce = bytearray(packet_head.nonce); nnonce[-1] |= 0x1 + p = PacketHead(protocol_version=PROTOCOL_VERSION, + controllerID=packet_head.controllerID, + nonce=nnonce) + r = ReplyHead(msg_type=request_head.msg_type, status=status.value) + return p.pack() + r.pack() + (data or bytes(0)) + +def fstring(buf): + length, string = int(buf[0]), buf[1:] + checkmsg(length <= len(string), 'fstring length > buffer size') + return string[:length] + +process_request = { + MsgType.OPEN: + lambda data: ((S.OK if fstring(data) == b'Hello' else S.ERR), None), +} + +def handle_request(buf): + packet_head, request_head, request_type, indata = parse_request_packet(buf) + status, outdata = process_request[request_type](indata) + return make_reply_packet(packet_head, request_head, status, outdata) diff --git a/gateserver/controller_server.py b/gateserver/controller_server.py index 301fec9..23dcd7a 100644 --- a/gateserver/controller_server.py +++ b/gateserver/controller_server.py @@ -1,6 +1,6 @@ """The UDP server that provides the API for the controllers.""" -#from . import controller_api +from . import controller_api import socketserver import logging @@ -14,9 +14,10 @@ class MessageHandler(socketserver.BaseRequestHandler): """ def handle(self): - data, socket = self.request - socket.sendto(data, self.client_address) - log.info(data, extra=dict(ip=self.client_address[0], status='OK')) + indata, socket = self.request + outdata = controller_api.handle_request(indata) + socket.sendto(outdata, self.client_address) + log.info(outdata, extra=dict(ip=self.client_address[0])) def serve(config): bind_addr = '0.0.0.0', config.udp_port diff --git a/gateserver/utils/structparse.py b/gateserver/utils/structparse.py new file mode 100644 index 0000000..56d7c0c --- /dev/null +++ b/gateserver/utils/structparse.py @@ -0,0 +1,43 @@ +from collections import namedtuple +from struct import Struct +from enum import Enum + +STRUCT_FORMAT = '<' # little-endian, no alignment (i.e. packed) + +class BadMessageError(Exception): pass + +def checkmsg(expression, err): + if not expression: raise BadMessageError(err) + +class t: + """pieces of struct format strings: docs.python.org/3/library/struct.html""" + uint8 = 'B' + bytes = lambda sz: '{}s'.format(sz) + +class MyStructMixin: + _struct = None + + @classmethod + def set_struct(cls, formatstring): + cls._struct = Struct(formatstring) + + @classmethod + def unpack_from(cls, buf): + """Constructs a new instance by unpacking the given buffer. + + Returns the new instance and the rest of the buffer. + """ + sz = cls._struct.size + head, tail = buf[:sz], buf[sz:] + return cls(*cls._struct.unpack(head)), tail + + def pack(self): + """Returns itself packed as `bytes`.""" + return self._struct.pack(*self) + +def mystruct(name, fields, types): + """Creates a namedtuple that can be packed to and unpacked from `bytes`.""" + class Cls(namedtuple(name, fields), MyStructMixin): pass + Cls.__name__ = name + Cls.set_struct(STRUCT_FORMAT + ''.join(types)) + return Cls diff --git a/packet_example.bin b/packet_example.bin new file mode 100644 index 0000000000000000000000000000000000000000..9e670037452709921a08f13a36c3f67d98149dae GIT binary patch literal 42 dcmZQz>?w$9E-ZF(M}tm`tRAU3Ir$7Q008+1355Uv literal 0 HcmV?d00001 diff --git a/runserver.py b/runserver.py index 2d1b6ba..12938c2 100755 --- a/runserver.py +++ b/runserver.py @@ -3,15 +3,10 @@ from gateserver import db, controller_server import config -import logging - -logging.basicConfig(level=logging.DEBUG, - format='%(asctime)s %(levelname)s %(name)s %(ip)s %(message)s %(status)s', - datefmt='%Y-%m-%d %H:%M:%S') if __name__ == '__main__': try: db.connect(config.db_url) controller_server.serve(config) except (SystemExit, KeyboardInterrupt): - logging.shutdown() + pass diff --git a/udpclient.py b/udpclient.py new file mode 100644 index 0000000..bae42f5 --- /dev/null +++ b/udpclient.py @@ -0,0 +1,13 @@ +import socket + +HOST = '127.0.0.1' +PORT = 5042 + +def msg(buf): + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + sock.sendto(buf, (HOST, PORT)) + return sock.recv(1024) + +if __name__ == '__main__': + with open('./packet_example.bin', 'rb') as f: + print(msg(f.read())) From 9e2b79242c8542bc8b3a0e89430b827205268698 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamila=20Sou=C4=8Dkov=C3=A1?= Date: Sun, 15 Feb 2015 09:14:43 +0100 Subject: [PATCH 17/46] add {http,udp}_host to config --- config.py.example | 4 ++++ tests/config.py.example | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/config.py.example b/config.py.example index 3da37ef..5191f57 100644 --- a/config.py.example +++ b/config.py.example @@ -1,5 +1,9 @@ """The server configuration.""" +http_host = '0.0.0.0' # Use the actual IP address, binding to 0.0.0.0 sucks! http_port = 5047 + +udp_host = '0.0.0.0' # Use the actual IP address, binding to 0.0.0.0 sucks! udp_port = 5042 + db_url = 'postgresql://user:password@localhost/gate' diff --git a/tests/config.py.example b/tests/config.py.example index c082e5c..3e21c46 100644 --- a/tests/config.py.example +++ b/tests/config.py.example @@ -1,5 +1,9 @@ """Tests configuration.""" +http_host = 'localhost' http_port = 9047 + +udp_host = 'localhost' udp_port = 9042 + db_url = 'postgresql://user:password@localhost/gate_test' From 792bd9a23e54c95ff57277847960077afef2dfea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamila=20Sou=C4=8Dkov=C3=A1?= Date: Sun, 15 Feb 2015 09:24:10 +0100 Subject: [PATCH 18/46] add NaCl (yay!) + rewrite udpclient --- README.md | 4 +-- gateserver/controller_api.py | 49 +++++++++++++++++++++++++-------- gateserver/utils/__init__.py | 8 ++++++ gateserver/utils/structparse.py | 5 ---- udpclient.py | 39 ++++++++++++++++++++++---- 5 files changed, 80 insertions(+), 25 deletions(-) create mode 100644 gateserver/utils/__init__.py diff --git a/README.md b/README.md index f0cffcc..7fd406f 100644 --- a/README.md +++ b/README.md @@ -31,8 +31,8 @@ Running tests Next to do: ----------- -- the controller end -- wrap/unwrap NaCl +- split DB table to controller and door +- on request arrival check if client IP matches the one in DB for this ID - HTTP: rewrite to use Werkzeug instead of CherryPy - fix DB singleton (who wants a singleton?!) - CI diff --git a/gateserver/controller_api.py b/gateserver/controller_api.py index 79fa2c6..9143977 100644 --- a/gateserver/controller_api.py +++ b/gateserver/controller_api.py @@ -1,9 +1,15 @@ from .utils.structparse import * +from . import utils from . import db import nacl.raw as nacl PROTOCOL_VERSION = bytes([0x00,0x01]) +class BadMessageError(Exception): pass + +def checkmsg(expression, err): + if not expression: raise BadMessageError(err) + class MsgType(Enum): OPEN = 1 @@ -25,29 +31,48 @@ class ReplyStatus(Enum): ['msg_type', 'status' ], [ t.uint8 , t.uint8 ]) -def parse_request_packet(buf): +def get_key_for_mac(mac): + rs = db.exec_sql('SELECT key FROM controller WHERE id = %s', + (utils.bytes2mac(mac),), ret=True) + checkmsg(len(rs) == 1, 'unknown controllerID '+utils.bytes2mac(mac)) + return rs[0]['key'].tobytes() + +def crypto_unwrap(packet_head, payload): + key = get_key_for_mac(packet_head.controllerID) + return nacl.crypto_secretbox_open(payload, + packet_head.controllerID + packet_head.nonce, key) + +def crypto_wrap(packet_head, payload): + key = get_key_for_mac(packet_head.controllerID) + return nacl.crypto_secretbox(payload, + packet_head.controllerID + packet_head.nonce, key) + +def parse_packet(buf, struct=None): + struct = struct or RequestHead + assert struct in [RequestHead, ReplyHead] p, payload = PacketHead.unpack_from(buf) - #cntrl = db.exec_sql('SELECT key FROM controller WHERE id = %s', to_mac(p.controllerID)) - #if len(cntrl) != 1: raise BadMessageError('unknown controllerID ' + p.controllerID) - #key = cntrl[0].key - #payload = nacl.crypto_secretbox_open(payload, p.nonce, key) checkmsg(p.protocol_version == PROTOCOL_VERSION, 'Invalid protocol version') - checkmsg(p.nonce[-1] & 0x1 == 0, 'Last bit of request nonce must be 0') - r, data = RequestHead.unpack_from(payload) + payload = crypto_unwrap(p, payload) + r, data = struct.unpack_from(payload) try: t = MsgType(r.msg_type) except ValueError: raise BadMessageError('Unknown request type {}'.format(r.msg_type)) return p, r, t, data -def make_reply_packet(packet_head, request_head, status, data): +def make_packet(packet_head, r_head, data=None): + """Requires `packet_head` and `r_head` to be valid.""" + payload = r_head.pack() + (data or bytes(0)) + return packet_head.pack() + crypto_wrap(packet_head, payload) + +def make_reply_for(packet_head, request_head, status, data=None): """Requires `packet_head` and `request_head` to be valid.""" - nnonce = bytearray(packet_head.nonce); nnonce[-1] |= 0x1 + nnonce = bytearray(packet_head.nonce); nnonce[-1] ^= 0x1 p = PacketHead(protocol_version=PROTOCOL_VERSION, controllerID=packet_head.controllerID, nonce=nnonce) r = ReplyHead(msg_type=request_head.msg_type, status=status.value) - return p.pack() + r.pack() + (data or bytes(0)) + return make_packet(p, r, data) def fstring(buf): length, string = int(buf[0]), buf[1:] @@ -60,6 +85,6 @@ def fstring(buf): } def handle_request(buf): - packet_head, request_head, request_type, indata = parse_request_packet(buf) + packet_head, request_head, request_type, indata = parse_packet(buf) status, outdata = process_request[request_type](indata) - return make_reply_packet(packet_head, request_head, status, outdata) + return make_reply_for(packet_head, request_head, status, outdata) diff --git a/gateserver/utils/__init__.py b/gateserver/utils/__init__.py new file mode 100644 index 0000000..b92f437 --- /dev/null +++ b/gateserver/utils/__init__.py @@ -0,0 +1,8 @@ +"""Various utility functions.""" + +def bytes2mac(mac): + return ':'.join('{:02x}'.format(x) for x in mac) + +def mac2bytes(s): + return bytes.fromhex(s.replace(':', '')) + diff --git a/gateserver/utils/structparse.py b/gateserver/utils/structparse.py index 56d7c0c..f27013f 100644 --- a/gateserver/utils/structparse.py +++ b/gateserver/utils/structparse.py @@ -4,11 +4,6 @@ STRUCT_FORMAT = '<' # little-endian, no alignment (i.e. packed) -class BadMessageError(Exception): pass - -def checkmsg(expression, err): - if not expression: raise BadMessageError(err) - class t: """pieces of struct format strings: docs.python.org/3/library/struct.html""" uint8 = 'B' diff --git a/udpclient.py b/udpclient.py index bae42f5..4782d48 100644 --- a/udpclient.py +++ b/udpclient.py @@ -1,13 +1,40 @@ +import config +from gateserver import db +from gateserver.controller_api import * +from gateserver.utils import mac2bytes import socket - -HOST = '127.0.0.1' -PORT = 5042 +import os +import sys def msg(buf): sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - sock.sendto(buf, (HOST, PORT)) + sock.sendto(buf, (config.udp_host, config.udp_port)) return sock.recv(1024) +def request(mac, msgtype, data): + nonce = os.urandom(18) + buf = make_packet(PacketHead(PROTOCOL_VERSION, mac, nonce), + RequestHead(msgtype.value), + data) + r = msg(buf) + return parse_packet(r, ReplyHead) + +def prettyprint_reply(p, r, t, data): + try: + s = ReplyStatus(r.status) + except ValueError: + raise BadMessageError('Unknown status {}'.format(r.status)) + return '{} {}: {}'.format(t.name, s.name, data or '(no data)') + if __name__ == '__main__': - with open('./packet_example.bin', 'rb') as f: - print(msg(f.read())) + _, mac, msgtype = sys.argv + try: + t = MsgType[msgtype.upper()] + except KeyError: + sys.exit('No such message type: '+msgtype) + + indata = sys.stdin.buffer.read() + + db.connect(config.db_url) + reply = request(mac2bytes(mac), t, indata) + print(prettyprint_reply(*reply)) From ffe9d69b6d3514c25f8f6bcb8e0268fb8a05f841 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamila=20Sou=C4=8Dkov=C3=A1?= Date: Sun, 15 Feb 2015 09:44:15 +0100 Subject: [PATCH 19/46] startup check: all message types have handlers --- gateserver/controller_api.py | 1 + 1 file changed, 1 insertion(+) diff --git a/gateserver/controller_api.py b/gateserver/controller_api.py index 9143977..c7836df 100644 --- a/gateserver/controller_api.py +++ b/gateserver/controller_api.py @@ -83,6 +83,7 @@ def fstring(buf): MsgType.OPEN: lambda data: ((S.OK if fstring(data) == b'Hello' else S.ERR), None), } +assert set(MsgType) == set(process_request), 'Not all message types are handled' def handle_request(buf): packet_head, request_head, request_type, indata = parse_packet(buf) From 33e9beeba04725c9d6d65317a1e2a627f5b18eae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamila=20Sou=C4=8Dkov=C3=A1?= Date: Sun, 15 Feb 2015 12:12:11 +0100 Subject: [PATCH 20/46] split controller_api into controller_{api,protocol} --- gateserver/controller_api.py | 93 +++++++------------------------ gateserver/controller_protocol.py | 67 ++++++++++++++++++++++ gateserver/controller_server.py | 6 +- udpclient.py | 13 +++-- 4 files changed, 97 insertions(+), 82 deletions(-) create mode 100644 gateserver/controller_protocol.py diff --git a/gateserver/controller_api.py b/gateserver/controller_api.py index c7836df..30c57e8 100644 --- a/gateserver/controller_api.py +++ b/gateserver/controller_api.py @@ -1,80 +1,15 @@ -from .utils.structparse import * -from . import utils +from .controller_protocol import * from . import db -import nacl.raw as nacl - -PROTOCOL_VERSION = bytes([0x00,0x01]) - -class BadMessageError(Exception): pass - -def checkmsg(expression, err): - if not expression: raise BadMessageError(err) - -class MsgType(Enum): - OPEN = 1 - -class ReplyStatus(Enum): - OK = 0x01 - ERR = 0x10 - TRY_AGAIN = 0x11 -S = ReplyStatus - -PacketHead = mystruct('PacketHead', - ['protocol_version', 'controllerID', 'nonce' ], - [ t.bytes(2) , t.bytes(6) , t.bytes(18) ]) - -RequestHead = mystruct('RequestHead', - ['msg_type'], - [ t.uint8 ]) - -ReplyHead = mystruct('ReplyHead', - ['msg_type', 'status' ], - [ t.uint8 , t.uint8 ]) +from . import utils def get_key_for_mac(mac): rs = db.exec_sql('SELECT key FROM controller WHERE id = %s', (utils.bytes2mac(mac),), ret=True) - checkmsg(len(rs) == 1, 'unknown controllerID '+utils.bytes2mac(mac)) + checkmsg(len(rs) == 1, 'unknown controllerID ') return rs[0]['key'].tobytes() -def crypto_unwrap(packet_head, payload): - key = get_key_for_mac(packet_head.controllerID) - return nacl.crypto_secretbox_open(payload, - packet_head.controllerID + packet_head.nonce, key) - -def crypto_wrap(packet_head, payload): - key = get_key_for_mac(packet_head.controllerID) - return nacl.crypto_secretbox(payload, - packet_head.controllerID + packet_head.nonce, key) - -def parse_packet(buf, struct=None): - struct = struct or RequestHead - assert struct in [RequestHead, ReplyHead] - p, payload = PacketHead.unpack_from(buf) - checkmsg(p.protocol_version == PROTOCOL_VERSION, 'Invalid protocol version') - payload = crypto_unwrap(p, payload) - r, data = struct.unpack_from(payload) - try: - t = MsgType(r.msg_type) - except ValueError: - raise BadMessageError('Unknown request type {}'.format(r.msg_type)) - return p, r, t, data - -def make_packet(packet_head, r_head, data=None): - """Requires `packet_head` and `r_head` to be valid.""" - payload = r_head.pack() + (data or bytes(0)) - return packet_head.pack() + crypto_wrap(packet_head, payload) - -def make_reply_for(packet_head, request_head, status, data=None): - """Requires `packet_head` and `request_head` to be valid.""" - nnonce = bytearray(packet_head.nonce); nnonce[-1] ^= 0x1 - p = PacketHead(protocol_version=PROTOCOL_VERSION, - controllerID=packet_head.controllerID, - nonce=nnonce) - r = ReplyHead(msg_type=request_head.msg_type, status=status.value) - return make_packet(p, r, data) - def fstring(buf): + """See https://github.com/fmfi-svt-gate/server/wiki/Controller-%E2%86%94-Server-Protocol#note-isic-ids-representation""" length, string = int(buf[0]), buf[1:] checkmsg(length <= len(string), 'fstring length > buffer size') return string[:length] @@ -85,7 +20,21 @@ def fstring(buf): } assert set(MsgType) == set(process_request), 'Not all message types are handled' +def log_message(controllerID, req_type, indata, status): + """TODO""" + print(utils.bytes2mac(controllerID), req_type.name, indata, '->', status.name) + +def log_bad_packet(buf, e): + """TODO""" + raise e + def handle_request(buf): - packet_head, request_head, request_type, indata = parse_packet(buf) - status, outdata = process_request[request_type](indata) - return make_reply_for(packet_head, request_head, status, outdata) + try: + packet_head, payload = parse_packet_head(buf) + key = get_key_for_mac(packet_head.controllerID) + req_head, req_type, indata = parse_r(RequestHead, packet_head, key, payload) + status, outdata = process_request[req_type](indata) + log_message(packet_head.controllerID, req_type, indata, status) + return make_reply_for(packet_head, req_head, key, status, outdata) + except BadMessageError as e: + log_bad_packet(buf, e) diff --git a/gateserver/controller_protocol.py b/gateserver/controller_protocol.py new file mode 100644 index 0000000..aeaa199 --- /dev/null +++ b/gateserver/controller_protocol.py @@ -0,0 +1,67 @@ +from .utils.structparse import * +import nacl.raw as nacl + +PROTOCOL_VERSION = bytes([0x00,0x01]) + +class BadMessageError(Exception): pass + +def checkmsg(expression, err): + if not expression: raise BadMessageError(err) + +class MsgType(Enum): + OPEN = 1 + +class ReplyStatus(Enum): + OK = 0x01 + ERR = 0x10 + TRY_AGAIN = 0x11 +S = ReplyStatus + +PacketHead = mystruct('PacketHead', + ['protocol_version', 'controllerID', 'nonce' ], + [ t.bytes(2) , t.bytes(6) , t.bytes(18) ]) + +RequestHead = mystruct('RequestHead', + ['msg_type'], + [ t.uint8 ]) + +ReplyHead = mystruct('ReplyHead', + ['msg_type', 'status' ], + [ t.uint8 , t.uint8 ]) + +def crypto_unwrap(packet_head, key, payload): + return nacl.crypto_secretbox_open(payload, + packet_head.controllerID + packet_head.nonce, key) + +def crypto_wrap(packet_head, key, payload): + return nacl.crypto_secretbox(payload, + packet_head.controllerID + packet_head.nonce, key) + +def parse_packet_head(buf): + p, payload = PacketHead.unpack_from(buf) + checkmsg(p.protocol_version == PROTOCOL_VERSION, 'Invalid protocol version') + return p, payload + +def parse_r(struct, packet_head, key, payload): + assert struct in [RequestHead, ReplyHead] + try: payload = crypto_unwrap(packet_head, key, payload) + except ValueError as e: raise BadMessageError('Decryption failed') from e + r, data = struct.unpack_from(payload) + try: t = MsgType(r.msg_type) + except ValueError as e: raise BadMessageError('Unknown message type') from e + return r, t, data + +def make_packet(packet_head, r_head, key, data=None): + """Requires `packet_head` and `r_head` to be valid.""" + payload = r_head.pack() + (data or bytes(0)) + + return packet_head.pack() + crypto_wrap(packet_head, key, payload) + +def make_reply_for(packet_head, request_head, key, status, data=None): + """Requires `packet_head` and `request_head` to be valid.""" + nnonce = bytearray(packet_head.nonce); nnonce[-1] ^= 0x1 + p = PacketHead(protocol_version=PROTOCOL_VERSION, + controllerID=packet_head.controllerID, + nonce=nnonce) + r = ReplyHead(msg_type=request_head.msg_type, status=status.value) + return make_packet(p, r, key, data) diff --git a/gateserver/controller_server.py b/gateserver/controller_server.py index 23dcd7a..6587cd4 100644 --- a/gateserver/controller_server.py +++ b/gateserver/controller_server.py @@ -2,9 +2,6 @@ from . import controller_api import socketserver -import logging - -log = logging.getLogger('server') class MessageHandler(socketserver.BaseRequestHandler): """Handles a message from the controller. @@ -17,9 +14,8 @@ def handle(self): indata, socket = self.request outdata = controller_api.handle_request(indata) socket.sendto(outdata, self.client_address) - log.info(outdata, extra=dict(ip=self.client_address[0])) def serve(config): - bind_addr = '0.0.0.0', config.udp_port + bind_addr = config.udp_host, config.udp_port server = socketserver.ThreadingUDPServer(bind_addr, MessageHandler) server.serve_forever() diff --git a/udpclient.py b/udpclient.py index 4782d48..3338415 100644 --- a/udpclient.py +++ b/udpclient.py @@ -12,12 +12,15 @@ def msg(buf): return sock.recv(1024) def request(mac, msgtype, data): + key = get_key_for_mac(mac) nonce = os.urandom(18) - buf = make_packet(PacketHead(PROTOCOL_VERSION, mac, nonce), - RequestHead(msgtype.value), - data) - r = msg(buf) - return parse_packet(r, ReplyHead) + res = msg(make_packet(PacketHead(PROTOCOL_VERSION, mac, nonce), + RequestHead(msgtype.value), + key, + data)) + p, payload = parse_packet_head(res) + r, t, data = parse_r(ReplyHead, p, key, payload) + return p, r, t, data def prettyprint_reply(p, r, t, data): try: From 26e84c6b6fd051ff575f7c4a0e84e2af2f43c8a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamila=20Sou=C4=8Dkov=C3=A1?= Date: Sun, 15 Feb 2015 12:32:19 +0100 Subject: [PATCH 21/46] cleanup: add docstrings & stuff --- gateserver/controller_api.py | 1 + gateserver/controller_protocol.py | 26 +++++++++++++++++++++----- gateserver/controller_server.py | 6 +----- 3 files changed, 23 insertions(+), 10 deletions(-) diff --git a/gateserver/controller_api.py b/gateserver/controller_api.py index 30c57e8..b880c19 100644 --- a/gateserver/controller_api.py +++ b/gateserver/controller_api.py @@ -3,6 +3,7 @@ from . import utils def get_key_for_mac(mac): + """Loads the key for this controller from the DB.""" rs = db.exec_sql('SELECT key FROM controller WHERE id = %s', (utils.bytes2mac(mac),), ret=True) checkmsg(len(rs) == 1, 'unknown controllerID ') diff --git a/gateserver/controller_protocol.py b/gateserver/controller_protocol.py index aeaa199..164d3ce 100644 --- a/gateserver/controller_protocol.py +++ b/gateserver/controller_protocol.py @@ -1,3 +1,8 @@ +"""Implements the Controller <-> Server protocol. + +See https://github.com/fmfi-svt-gate/server/wiki/Controller-%E2%86%94-Server-Protocol . +""" + from .utils.structparse import * import nacl.raw as nacl @@ -5,8 +10,8 @@ class BadMessageError(Exception): pass -def checkmsg(expression, err): - if not expression: raise BadMessageError(err) +def checkmsg(expression, errmsg): + if not expression: raise BadMessageError(errmsg) class MsgType(Enum): OPEN = 1 @@ -38,11 +43,17 @@ def crypto_wrap(packet_head, key, payload): packet_head.controllerID + packet_head.nonce, key) def parse_packet_head(buf): + """Parses the packet header, returning that and the rest of the data.""" p, payload = PacketHead.unpack_from(buf) checkmsg(p.protocol_version == PROTOCOL_VERSION, 'Invalid protocol version') return p, payload def parse_r(struct, packet_head, key, payload): + """Decrypts the payload and parses the request/reply header. + + Returns the parsed header (as struct), message type (as MsgType) and the + rest of the data. + """ assert struct in [RequestHead, ReplyHead] try: payload = crypto_unwrap(packet_head, key, payload) except ValueError as e: raise BadMessageError('Decryption failed') from e @@ -52,13 +63,18 @@ def parse_r(struct, packet_head, key, payload): return r, t, data def make_packet(packet_head, r_head, key, data=None): - """Requires `packet_head` and `r_head` to be valid.""" - payload = r_head.pack() + (data or bytes(0)) + """Packs and encrypts the packet headers, request/reply headers and data. + Requires `packet_head` and `r_head` to be valid.""" + payload = r_head.pack() + (data or bytes(0)) return packet_head.pack() + crypto_wrap(packet_head, key, payload) def make_reply_for(packet_head, request_head, key, status, data=None): - """Requires `packet_head` and `request_head` to be valid.""" + """Creates a reply for the given packet and request headers. + + Packs status and data into a reply, encrypting according to `packet_head` + and `key`. Requires `packet_head` and `request_head` to be valid. + """ nnonce = bytearray(packet_head.nonce); nnonce[-1] ^= 0x1 p = PacketHead(protocol_version=PROTOCOL_VERSION, controllerID=packet_head.controllerID, diff --git a/gateserver/controller_server.py b/gateserver/controller_server.py index 6587cd4..c208705 100644 --- a/gateserver/controller_server.py +++ b/gateserver/controller_server.py @@ -4,11 +4,7 @@ import socketserver class MessageHandler(socketserver.BaseRequestHandler): - """Handles a message from the controller. - - Behaves according to - https://github.com/fmfi-svt/gate/wiki/Controller-%E2%86%94-Server-Protocol . - """ + """Handles a message from the controller.""" def handle(self): indata, socket = self.request From 11fcf2e91790ab6de43ce2d27999e7a7bb2a0fe2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamila=20Sou=C4=8Dkov=C3=A1?= Date: Sun, 15 Feb 2015 12:37:00 +0100 Subject: [PATCH 22/46] rm packet_example.bin No longer correct, as now encryption is in place. Cannot be made correct either, as the encryption key is locally generated on controller create. --- packet_example.bin | Bin 42 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 packet_example.bin diff --git a/packet_example.bin b/packet_example.bin deleted file mode 100644 index 9e670037452709921a08f13a36c3f67d98149dae..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 42 dcmZQz>?w$9E-ZF(M}tm`tRAU3Ir$7Q008+1355Uv From eee39b26161cdf22bdd5d6b3031a0ab18770a9ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamila=20Sou=C4=8Dkov=C3=A1?= Date: Sun, 15 Feb 2015 12:53:36 +0100 Subject: [PATCH 23/46] fix formatting in controller_api --- gateserver/controller_api.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/gateserver/controller_api.py b/gateserver/controller_api.py index b880c19..a04e25e 100644 --- a/gateserver/controller_api.py +++ b/gateserver/controller_api.py @@ -6,7 +6,7 @@ def get_key_for_mac(mac): """Loads the key for this controller from the DB.""" rs = db.exec_sql('SELECT key FROM controller WHERE id = %s', (utils.bytes2mac(mac),), ret=True) - checkmsg(len(rs) == 1, 'unknown controllerID ') + checkmsg(len(rs) == 1, 'unknown controllerID') return rs[0]['key'].tobytes() def fstring(buf): @@ -21,9 +21,9 @@ def fstring(buf): } assert set(MsgType) == set(process_request), 'Not all message types are handled' -def log_message(controllerID, req_type, indata, status): +def log_message(controllerID, mtype, indata, status): """TODO""" - print(utils.bytes2mac(controllerID), req_type.name, indata, '->', status.name) + print(utils.bytes2mac(controllerID), mtype.name, indata, '->', status.name) def log_bad_packet(buf, e): """TODO""" @@ -31,11 +31,11 @@ def log_bad_packet(buf, e): def handle_request(buf): try: - packet_head, payload = parse_packet_head(buf) - key = get_key_for_mac(packet_head.controllerID) - req_head, req_type, indata = parse_r(RequestHead, packet_head, key, payload) - status, outdata = process_request[req_type](indata) - log_message(packet_head.controllerID, req_type, indata, status) - return make_reply_for(packet_head, req_head, key, status, outdata) + pkt_head, payload = parse_packet_head(buf) + key = get_key_for_mac(pkt_head.controllerID) + req_head, mtype, indata = parse_r(RequestHead, pkt_head, key, payload) + status, outdata = process_request[mtype](indata) + log_message(pkt_head.controllerID, mtype, indata, status) + return make_reply_for(pkt_head, req_head, key, status, outdata) except BadMessageError as e: log_bad_packet(buf, e) From aefd60d5113495b5909036cf3edfc6db7810e827 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamila=20Sou=C4=8Dkov=C3=A1?= Date: Mon, 16 Feb 2015 16:04:12 +0100 Subject: [PATCH 24/46] DB init: .sql instead of bootstrap.py + weird .txt --- README.md | 7 ++++--- bootstrap.py | 19 ------------------- dbinit.sql | 11 +++++++++++ tables.txt | 4 ---- 4 files changed, 15 insertions(+), 26 deletions(-) delete mode 100755 bootstrap.py create mode 100644 dbinit.sql delete mode 100644 tables.txt diff --git a/README.md b/README.md index 7fd406f..dce13fb 100644 --- a/README.md +++ b/README.md @@ -16,16 +16,17 @@ Setup - csh, tcsh: `source venv/bin/activate.csh` 3. Install dependencies if necessary: `pip install -r requirements.txt` 4. configure: `cp config.py{.example,}; $EDITOR config.py` -5. bootstrap (create DB tables and such): `./bootstrap.py` +5. create DB tables: `psql -U -f dbinit.sql` 6. run with `./runserver.py`; run the HTTP API server with `./runhttp.py` Running tests ------------- -1. Edit configuration: `cp tests/config.py{.example,}; $EDITOR tests/config.py` +1. Edit configuration: `cp tests/config.py{.example,}; $EDITOR tests/config.py` A real Postgres DB is used, you need to specify the connection string. -2. run with `py.test tests/` +2. create DB tables: `psql -U -f dbinit.sql` +3. run with `py.test tests/` or `py.test --cov gateserver/ --cov-report term-missing tests/` for coverage report Next to do: diff --git a/bootstrap.py b/bootstrap.py deleted file mode 100755 index 660592c..0000000 --- a/bootstrap.py +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env python3 - -import config -from gateserver import db - -def db_create_tables(): - print('Creating DB tables...', end='') - with open('./tables.txt', 'r') as f: - if not db.conn: db.connect(config.db_url) - for line in f: - line = line.split('#')[0].strip() - if line: db.exec_sql('CREATE TABLE IF NOT EXISTS ' + line) - print(' OK') - -def all(): - db_create_tables() - -if __name__ == '__main__': - all() diff --git a/dbinit.sql b/dbinit.sql new file mode 100644 index 0000000..71a12cc --- /dev/null +++ b/dbinit.sql @@ -0,0 +1,11 @@ +CREATE TABLE IF NOT EXISTS controller ( + id macaddr PRIMARY KEY, + ip inet UNIQUE NOT NULL, + key bytea NOT NULL, + name text +); +CREATE TABLE IF NOT EXISTS log ( + time timestamp NOT NULL, + ctrl_id macaddr REFERENCES controller, + message text +); diff --git a/tables.txt b/tables.txt deleted file mode 100644 index a1bf116..0000000 --- a/tables.txt +++ /dev/null @@ -1,4 +0,0 @@ -# SQL (CREATE TABLE) syntax (these are executed during bootstrap) -# one line per table -controller (id macaddr PRIMARY KEY, ip inet UNIQUE NOT NULL, key bytea NOT NULL, name text) -log (time timestamp NOT NULL, ctrl_id macaddr REFERENCES controller, message text) From a8984d1dd45d66d0dab5798f26d2a32e3cdda633 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamila=20Sou=C4=8Dkov=C3=A1?= Date: Mon, 16 Feb 2015 21:18:32 +0100 Subject: [PATCH 25/46] remove http_api & co. --- gateserver/http_api.py | 87 ------------------------------------------ runhttp.py | 11 +++--- tests/test_http_api.py | 40 ------------------- 3 files changed, 6 insertions(+), 132 deletions(-) delete mode 100644 gateserver/http_api.py delete mode 100644 tests/test_http_api.py diff --git a/gateserver/http_api.py b/gateserver/http_api.py deleted file mode 100644 index 8c78511..0000000 --- a/gateserver/http_api.py +++ /dev/null @@ -1,87 +0,0 @@ -"""Defines the REST API for CRUD and management.""" - -from . import db -import nacl.raw as nacl -import cherrypy - -class MountPoint: - """Represents a mount point, or path prefix, for attaching resources to.""" - -class Resource(MountPoint): - """Represents a REST resource.""" - exposed = True - -class Log(Resource): - @cherrypy.tools.json_out() - def GET(self, entries=100): - return db.exec_sql('SELECT * FROM log ORDER BY time DESC LIMIT %s', - (entries,), ret=True) - -class CRUDResource(Resource): - """Represents a REST resource that exposes a DB table's CRUD methods.""" - def __init__(self, tbl, put_columns, get_columns, on_save=lambda x: x): - assert(tbl.isidentifier()) - self.table = tbl - self.put_columns = put_columns - self.get_columns = get_columns - self.on_save = on_save - - @cherrypy.tools.json_out() - def GET(self, id=None): - cols = list(self.get_columns) - q = 'SELECT {} FROM {}'.format(','.join(cols), self.table) - if id: q += ' WHERE id = %s' - rs = db.exec_sql(q, (id,), ret=True) - if id: - if len(rs) < 1: raise cherrypy.HTTPError('404 Not Found') - return rs[0] - else: return rs - - @cherrypy.tools.json_in() - @cherrypy.tools.json_out() - def PUT(self, id): - json = self.on_save(dict(cherrypy.request.json, id=id)) - print(json) - cols, values, ps = [], [], [] - for c in self.put_columns: - cols.append(c) - values.append(json.get(c)) - ps.append('%s') - q = 'INSERT INTO controller ({}) VALUES ({})'.format(','.join(cols), - ','.join(ps)) - try: - db.exec_sql(q, values) - except db.IntegrityError as e: - raise cherrypy.HTTPError('400 Bad Request', e.pgerror) from e - return { 'url': cherrypy.url() } - - # TODO POST - - def DELETE(self, id): - db.exec_sql('DELETE FROM {} WHERE id = %s'.format(self.table), (id,)) - -################################################################################ - -api_root = MountPoint() -api_root.controller = CRUDResource('controller', - get_columns={'id', 'ip', 'name'}, - put_columns={'id', 'ip', 'key', 'name'}, - on_save=lambda ctrl: - dict(ctrl, key=nacl.randombytes(nacl.crypto_secretbox_KEYBYTES))) -api_root.log = Log() - -################################################################################ - -cherrypy_conf = { - '/': { - 'request.dispatch': cherrypy.dispatch.MethodDispatcher(), - } -} -cherrypy.tree.mount(api_root, '/', cherrypy_conf) - -def serve(config): - cherrypy.config.update({'server.socket_port': config.http_port}) - cherrypy.engine.start() - -def stop(): - cherrypy.engine.exit() diff --git a/runhttp.py b/runhttp.py index 97ee792..0f97084 100755 --- a/runhttp.py +++ b/runhttp.py @@ -1,9 +1,10 @@ #!/usr/bin/env python3 """Gate HTTP API server runner.""" -from gateserver import db, http_api -import config +#from gateserver import db, http_api +#import config -if __name__ == '__main__': - db.connect(config.db_url) - http_api.serve(config) +#if __name__ == '__main__': + #db.connect(config.db_url) + #http_api.serve(config) +print('Not implemented yet.') diff --git a/tests/test_http_api.py b/tests/test_http_api.py deleted file mode 100644 index e539992..0000000 --- a/tests/test_http_api.py +++ /dev/null @@ -1,40 +0,0 @@ -import tests.config as config -import bootstrap -import gateserver.db -import gateserver.http_api -import requests - -url = 'http://localhost:{}'.format(config.http_port) - -def setup_module(module): - gateserver.db.connect(config.db_url) - bootstrap.db_create_tables() - gateserver.http_api.serve(config) - -def teardown_module(module): - gateserver.http_api.stop() - -def assert_req(method, url, status=200, expected_data=None, **kwargs): - print(kwargs) - r = requests.request(method, url, **kwargs) - assert r.status_code == status - if expected_data: assert r.json() == expected_data - -def test_controller_crud(): - cid = '00:00:00:00:00:00' - data = { 'id': cid, 'ip': '0.0.0.0', 'name': 'Test Controller' } - requests.delete(url+'/controller/'+cid) # just in case the last run didn't - - assert_req('PUT', url+'/controller/'+cid, json={'ip': data['ip'], - 'name': data['name']}, - expected_data={'url': url+'/controller/'+cid}) - assert_req('GET', url+'/controller/'+cid, expected_data=data) - assert_req('GET', url+'/controller/', expected_data=[data]) - assert_req('PUT', url+'/controller/'+cid, json={}, status=400) - assert_req('DELETE', url+'/controller/'+cid) - assert_req('GET', url+'/controller/'+cid, status=404) - -def test_log(): - r = requests.get(url+'/log/') - assert r.status_code == 200 - assert isinstance(r.json(), list) From b622e513558a5e93ae8c3db5312850292640725b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamila=20Sou=C4=8Dkov=C3=A1?= Date: Mon, 16 Feb 2015 21:34:31 +0100 Subject: [PATCH 26/46] fix formatting: Align! Align! Align! --- gateserver/controller_protocol.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/gateserver/controller_protocol.py b/gateserver/controller_protocol.py index 164d3ce..816164a 100644 --- a/gateserver/controller_protocol.py +++ b/gateserver/controller_protocol.py @@ -17,22 +17,22 @@ class MsgType(Enum): OPEN = 1 class ReplyStatus(Enum): - OK = 0x01 - ERR = 0x10 + OK = 0x01 + ERR = 0x10 TRY_AGAIN = 0x11 S = ReplyStatus PacketHead = mystruct('PacketHead', - ['protocol_version', 'controllerID', 'nonce' ], - [ t.bytes(2) , t.bytes(6) , t.bytes(18) ]) + ['protocol_version', 'controllerID', 'nonce' ], + [ t.bytes(2) , t.bytes(6) , t.bytes(18) ]) RequestHead = mystruct('RequestHead', ['msg_type'], - [ t.uint8 ]) + [ t.uint8 ]) ReplyHead = mystruct('ReplyHead', ['msg_type', 'status' ], - [ t.uint8 , t.uint8 ]) + [ t.uint8 , t.uint8 ]) def crypto_unwrap(packet_head, key, payload): return nacl.crypto_secretbox_open(payload, From 7d8811fa34ccbf0c1ef6fea40a6fde9d3816f98e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamila=20Sou=C4=8Dkov=C3=A1?= Date: Tue, 17 Feb 2015 16:22:52 +0100 Subject: [PATCH 27/46] remove CherryPy from requirements.txt --- requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index d6ad191..965e7dd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,3 @@ -CherryPy==3.6.0 cov-core==1.15.0 coverage==3.7.1 psycopg2==2.5.4 From cf47406c15f27a21a52447554b85189755318954 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamila=20Sou=C4=8Dkov=C3=A1?= Date: Tue, 17 Feb 2015 20:30:30 +0100 Subject: [PATCH 28/46] review-induced fixes --- gateserver/controller_api.py | 32 ++++++++++++++++--------------- gateserver/controller_protocol.py | 26 ++++++++++++++----------- gateserver/controller_server.py | 6 +++--- runserver.py | 7 ++----- 4 files changed, 37 insertions(+), 34 deletions(-) diff --git a/gateserver/controller_api.py b/gateserver/controller_api.py index a04e25e..1242bb3 100644 --- a/gateserver/controller_api.py +++ b/gateserver/controller_api.py @@ -1,4 +1,4 @@ -from .controller_protocol import * +from . import controller_protocol as p from . import db from . import utils @@ -6,20 +6,21 @@ def get_key_for_mac(mac): """Loads the key for this controller from the DB.""" rs = db.exec_sql('SELECT key FROM controller WHERE id = %s', (utils.bytes2mac(mac),), ret=True) - checkmsg(len(rs) == 1, 'unknown controllerID') - return rs[0]['key'].tobytes() + p.checkmsg(len(rs) == 1, 'unknown controllerID') + return bytes(rs[0]['key']) -def fstring(buf): +def isic_id_repr(buf): """See https://github.com/fmfi-svt-gate/server/wiki/Controller-%E2%86%94-Server-Protocol#note-isic-ids-representation""" - length, string = int(buf[0]), buf[1:] - checkmsg(length <= len(string), 'fstring length > buffer size') + length, string = buf[0], buf[1:] + p.checkmsg(length <= len(string), 'isic_id_repr length > buffer size') return string[:length] process_request = { - MsgType.OPEN: - lambda data: ((S.OK if fstring(data) == b'Hello' else S.ERR), None), + p.MsgType.OPEN: + lambda data: + ((p.S.OK if isic_id_repr(data) == b'Hello' else p.S.ERR), None), } -assert set(MsgType) == set(process_request), 'Not all message types are handled' +assert set(p.MsgType) == set(process_request), 'Not all message types handled' def log_message(controllerID, mtype, indata, status): """TODO""" @@ -31,11 +32,12 @@ def log_bad_packet(buf, e): def handle_request(buf): try: - pkt_head, payload = parse_packet_head(buf) - key = get_key_for_mac(pkt_head.controllerID) - req_head, mtype, indata = parse_r(RequestHead, pkt_head, key, payload) + packet_head, payload = p.parse_packet_head(buf) + key = get_key_for_mac(packet_head.controllerID) + request_head, mtype, indata = p.parse_r( + p.RequestHead, packet_head, key, payload) status, outdata = process_request[mtype](indata) - log_message(pkt_head.controllerID, mtype, indata, status) - return make_reply_for(pkt_head, req_head, key, status, outdata) - except BadMessageError as e: + log_message(packet_head.controllerID, mtype, indata, status) + return p.make_reply_for(packet_head, request_head, key, status, outdata) + except p.BadMessageError as e: log_bad_packet(buf, e) diff --git a/gateserver/controller_protocol.py b/gateserver/controller_protocol.py index 816164a..373d4ca 100644 --- a/gateserver/controller_protocol.py +++ b/gateserver/controller_protocol.py @@ -55,18 +55,22 @@ def parse_r(struct, packet_head, key, payload): rest of the data. """ assert struct in [RequestHead, ReplyHead] - try: payload = crypto_unwrap(packet_head, key, payload) - except ValueError as e: raise BadMessageError('Decryption failed') from e + try: + payload = crypto_unwrap(packet_head, key, payload) + except ValueError as e: + raise BadMessageError('Decryption failed') from e r, data = struct.unpack_from(payload) - try: t = MsgType(r.msg_type) - except ValueError as e: raise BadMessageError('Unknown message type') from e + try: + t = MsgType(r.msg_type) + except ValueError as e: + raise BadMessageError('Unknown message type') from e return r, t, data def make_packet(packet_head, r_head, key, data=None): """Packs and encrypts the packet headers, request/reply headers and data. Requires `packet_head` and `r_head` to be valid.""" - payload = r_head.pack() + (data or bytes(0)) + payload = r_head.pack() + (data or b'') return packet_head.pack() + crypto_wrap(packet_head, key, payload) def make_reply_for(packet_head, request_head, key, status, data=None): @@ -75,9 +79,9 @@ def make_reply_for(packet_head, request_head, key, status, data=None): Packs status and data into a reply, encrypting according to `packet_head` and `key`. Requires `packet_head` and `request_head` to be valid. """ - nnonce = bytearray(packet_head.nonce); nnonce[-1] ^= 0x1 - p = PacketHead(protocol_version=PROTOCOL_VERSION, - controllerID=packet_head.controllerID, - nonce=nnonce) - r = ReplyHead(msg_type=request_head.msg_type, status=status.value) - return make_packet(p, r, key, data) + reply_nonce = bytearray(packet_head.nonce); reply_nonce[-1] ^= 0x1 + reply_packet_head = PacketHead(protocol_version=PROTOCOL_VERSION, + controllerID=packet_head.controllerID, + nonce=reply_nonce) + reply_head = ReplyHead(msg_type=request_head.msg_type, status=status.value) + return make_packet(reply_packet_head, reply_head, key, data) diff --git a/gateserver/controller_server.py b/gateserver/controller_server.py index c208705..481f79d 100644 --- a/gateserver/controller_server.py +++ b/gateserver/controller_server.py @@ -7,9 +7,9 @@ class MessageHandler(socketserver.BaseRequestHandler): """Handles a message from the controller.""" def handle(self): - indata, socket = self.request - outdata = controller_api.handle_request(indata) - socket.sendto(outdata, self.client_address) + in_packet, socket = self.request + out_packet = controller_api.handle_request(in_packet) + socket.sendto(out_packet, self.client_address) def serve(config): bind_addr = config.udp_host, config.udp_port diff --git a/runserver.py b/runserver.py index 12938c2..1d5b736 100755 --- a/runserver.py +++ b/runserver.py @@ -5,8 +5,5 @@ import config if __name__ == '__main__': - try: - db.connect(config.db_url) - controller_server.serve(config) - except (SystemExit, KeyboardInterrupt): - pass + db.connect(config.db_url) + controller_server.serve(config) From 823f7890173fc63bc33e6ce61c9bb6c0b76ae7e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamila=20Sou=C4=8Dkov=C3=A1?= Date: Wed, 18 Feb 2015 16:02:13 +0100 Subject: [PATCH 29/46] change struct declaration format to "vertical" --- gateserver/controller_protocol.py | 12 ++++++------ gateserver/utils/__init__.py | 2 ++ gateserver/utils/structparse.py | 14 ++++++-------- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/gateserver/controller_protocol.py b/gateserver/controller_protocol.py index 373d4ca..fed193d 100644 --- a/gateserver/controller_protocol.py +++ b/gateserver/controller_protocol.py @@ -23,16 +23,16 @@ class ReplyStatus(Enum): S = ReplyStatus PacketHead = mystruct('PacketHead', - ['protocol_version', 'controllerID', 'nonce' ], - [ t.bytes(2) , t.bytes(6) , t.bytes(18) ]) + (t.bytes(2) , 'protocol_version'), + (t.bytes(6) , 'controllerID' ), + (t.bytes(18), 'nonce' )) RequestHead = mystruct('RequestHead', - ['msg_type'], - [ t.uint8 ]) + (t.uint8, 'msg_type')) ReplyHead = mystruct('ReplyHead', - ['msg_type', 'status' ], - [ t.uint8 , t.uint8 ]) + (t.uint8, 'msg_type'), + (t.uint8, 'status' )) def crypto_unwrap(packet_head, key, payload): return nacl.crypto_secretbox_open(payload, diff --git a/gateserver/utils/__init__.py b/gateserver/utils/__init__.py index b92f437..4dfdee7 100644 --- a/gateserver/utils/__init__.py +++ b/gateserver/utils/__init__.py @@ -6,3 +6,5 @@ def bytes2mac(mac): def mac2bytes(s): return bytes.fromhex(s.replace(':', '')) +def unzip(lst): + return zip(*lst) # yay :D diff --git a/gateserver/utils/structparse.py b/gateserver/utils/structparse.py index f27013f..3fc94cf 100644 --- a/gateserver/utils/structparse.py +++ b/gateserver/utils/structparse.py @@ -1,8 +1,9 @@ from collections import namedtuple from struct import Struct from enum import Enum +from . import unzip -STRUCT_FORMAT = '<' # little-endian, no alignment (i.e. packed) +ENDIANITY = '<' # little-endian, no alignment (i.e. packed) class t: """pieces of struct format strings: docs.python.org/3/library/struct.html""" @@ -12,10 +13,6 @@ class t: class MyStructMixin: _struct = None - @classmethod - def set_struct(cls, formatstring): - cls._struct = Struct(formatstring) - @classmethod def unpack_from(cls, buf): """Constructs a new instance by unpacking the given buffer. @@ -30,9 +27,10 @@ def pack(self): """Returns itself packed as `bytes`.""" return self._struct.pack(*self) -def mystruct(name, fields, types): +def mystruct(name, *fields): """Creates a namedtuple that can be packed to and unpacked from `bytes`.""" - class Cls(namedtuple(name, fields), MyStructMixin): pass + fieldtypes, fieldnames = unzip(fields) + class Cls(namedtuple(name, fieldnames), MyStructMixin): pass Cls.__name__ = name - Cls.set_struct(STRUCT_FORMAT + ''.join(types)) + Cls._struct = Struct(ENDIANITY + ''.join(fieldtypes)) return Cls From 9a7e598f894a820d7ffed1bd05d2943c8a8fb88a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamila=20Sou=C4=8Dkov=C3=A1?= Date: Wed, 18 Feb 2015 16:04:11 +0100 Subject: [PATCH 30/46] remove requests from requirements.txt --- requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 965e7dd..b66f131 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,5 +4,4 @@ psycopg2==2.5.4 py==1.4.26 pytest==2.6.4 pytest-cov==1.8.1 -requests==2.5.1 https://github.com/warner/python-tweetnacl/tarball/b48a25a33f From 23502eff551e9c69acb969d6318c3d7d56447850 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamila=20Sou=C4=8Dkov=C3=A1?= Date: Wed, 18 Feb 2015 19:39:53 +0100 Subject: [PATCH 31/46] rename parse_r -> parse_payload --- gateserver/controller_api.py | 2 +- gateserver/controller_protocol.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/gateserver/controller_api.py b/gateserver/controller_api.py index 1242bb3..ee80abf 100644 --- a/gateserver/controller_api.py +++ b/gateserver/controller_api.py @@ -34,7 +34,7 @@ def handle_request(buf): try: packet_head, payload = p.parse_packet_head(buf) key = get_key_for_mac(packet_head.controllerID) - request_head, mtype, indata = p.parse_r( + request_head, mtype, indata = p.parse_payload( p.RequestHead, packet_head, key, payload) status, outdata = process_request[mtype](indata) log_message(packet_head.controllerID, mtype, indata, status) diff --git a/gateserver/controller_protocol.py b/gateserver/controller_protocol.py index fed193d..5aede9e 100644 --- a/gateserver/controller_protocol.py +++ b/gateserver/controller_protocol.py @@ -48,7 +48,7 @@ def parse_packet_head(buf): checkmsg(p.protocol_version == PROTOCOL_VERSION, 'Invalid protocol version') return p, payload -def parse_r(struct, packet_head, key, payload): +def parse_payload(struct, packet_head, key, payload): """Decrypts the payload and parses the request/reply header. Returns the parsed header (as struct), message type (as MsgType) and the From ad78ac00fb87cd95467b46c8dd1df0a1aaa53d55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamila=20Sou=C4=8Dkov=C3=A1?= Date: Mon, 7 Dec 2015 11:07:48 +0100 Subject: [PATCH 32/46] update config.py.example --- config.py.example | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/config.py.example b/config.py.example index 5191f57..884cf31 100644 --- a/config.py.example +++ b/config.py.example @@ -1,9 +1,9 @@ """The server configuration.""" -http_host = '0.0.0.0' # Use the actual IP address, binding to 0.0.0.0 sucks! +http_host = '0.0.0.0' http_port = 5047 -udp_host = '0.0.0.0' # Use the actual IP address, binding to 0.0.0.0 sucks! +udp_host = '0.0.0.0' # Use the actual IP address for UDP! udp_port = 5042 -db_url = 'postgresql://user:password@localhost/gate' +db_url = 'postgresql://user:password@localhost/deadlock' From 2b164e9e608f02b226d9c306dcca139a05278458 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamila=20Sou=C4=8Dkov=C3=A1?= Date: Sat, 23 May 2015 11:44:59 +0200 Subject: [PATCH 33/46] rename reply to response and such --- gateserver/controller_api.py | 24 +++++++++---------- gateserver/controller_protocol.py | 39 +++++++++++++++++-------------- udpclient.py | 5 ++-- 3 files changed, 37 insertions(+), 31 deletions(-) diff --git a/gateserver/controller_api.py b/gateserver/controller_api.py index ee80abf..215b133 100644 --- a/gateserver/controller_api.py +++ b/gateserver/controller_api.py @@ -1,4 +1,5 @@ -from . import controller_protocol as p +from .controller_protocol import (checkmsg, MsgType, ResponseStatus, + parse_packet_head, parse_request, BadMessageError) from . import db from . import utils @@ -6,21 +7,21 @@ def get_key_for_mac(mac): """Loads the key for this controller from the DB.""" rs = db.exec_sql('SELECT key FROM controller WHERE id = %s', (utils.bytes2mac(mac),), ret=True) - p.checkmsg(len(rs) == 1, 'unknown controllerID') + checkmsg(len(rs) == 1, 'unknown controllerID') return bytes(rs[0]['key']) def isic_id_repr(buf): """See https://github.com/fmfi-svt-gate/server/wiki/Controller-%E2%86%94-Server-Protocol#note-isic-ids-representation""" length, string = buf[0], buf[1:] - p.checkmsg(length <= len(string), 'isic_id_repr length > buffer size') + checkmsg(length <= len(string), 'isic_id_repr length > buffer size') return string[:length] process_request = { - p.MsgType.OPEN: - lambda data: - ((p.S.OK if isic_id_repr(data) == b'Hello' else p.S.ERR), None), + MsgType.OPEN: + lambda data: ((ResponseStatus.OK if isic_id_repr(data) == b'Hello' + else ResponseStatus.ERR), None), } -assert set(p.MsgType) == set(process_request), 'Not all message types handled' +assert set(MsgType) == set(process_request), 'Not all message types handled' def log_message(controllerID, mtype, indata, status): """TODO""" @@ -32,12 +33,11 @@ def log_bad_packet(buf, e): def handle_request(buf): try: - packet_head, payload = p.parse_packet_head(buf) + packet_head, payload = parse_packet_head(buf) key = get_key_for_mac(packet_head.controllerID) - request_head, mtype, indata = p.parse_payload( - p.RequestHead, packet_head, key, payload) + request_head, mtype, indata = parse_request(packet_head, key, payload) status, outdata = process_request[mtype](indata) log_message(packet_head.controllerID, mtype, indata, status) - return p.make_reply_for(packet_head, request_head, key, status, outdata) - except p.BadMessageError as e: + return make_response_for(packet_head, request_head, key, status, outdata) + except BadMessageError as e: log_bad_packet(buf, e) diff --git a/gateserver/controller_protocol.py b/gateserver/controller_protocol.py index 5aede9e..1dffbbc 100644 --- a/gateserver/controller_protocol.py +++ b/gateserver/controller_protocol.py @@ -16,11 +16,10 @@ def checkmsg(expression, errmsg): class MsgType(Enum): OPEN = 1 -class ReplyStatus(Enum): +class ResponseStatus(Enum): OK = 0x01 ERR = 0x10 TRY_AGAIN = 0x11 -S = ReplyStatus PacketHead = mystruct('PacketHead', (t.bytes(2) , 'protocol_version'), @@ -30,9 +29,9 @@ class ReplyStatus(Enum): RequestHead = mystruct('RequestHead', (t.uint8, 'msg_type')) -ReplyHead = mystruct('ReplyHead', - (t.uint8, 'msg_type'), - (t.uint8, 'status' )) +ResponseHead = mystruct('ResponseHead', + (t.uint8, 'msg_type'), + (t.uint8, 'status' )) def crypto_unwrap(packet_head, key, payload): return nacl.crypto_secretbox_open(payload, @@ -49,12 +48,12 @@ def parse_packet_head(buf): return p, payload def parse_payload(struct, packet_head, key, payload): - """Decrypts the payload and parses the request/reply header. + """Decrypts the payload and parses the request/response header. Returns the parsed header (as struct), message type (as MsgType) and the rest of the data. """ - assert struct in [RequestHead, ReplyHead] + assert struct in [RequestHead, ResponseHead] try: payload = crypto_unwrap(packet_head, key, payload) except ValueError as e: @@ -66,22 +65,28 @@ def parse_payload(struct, packet_head, key, payload): raise BadMessageError('Unknown message type') from e return r, t, data +def parse_request(packet_head, key, payload): + return parse_payload(RequestHead, packet_head, key, payload) + +def parse_response(packet_head, key, payload): + return parse_payload(ResponseHead, packet_head, key, payload) + def make_packet(packet_head, r_head, key, data=None): - """Packs and encrypts the packet headers, request/reply headers and data. + """Packs and encrypts the packet headers, request/response headers and data. Requires `packet_head` and `r_head` to be valid.""" payload = r_head.pack() + (data or b'') return packet_head.pack() + crypto_wrap(packet_head, key, payload) -def make_reply_for(packet_head, request_head, key, status, data=None): - """Creates a reply for the given packet and request headers. +def make_response_for(packet_head, request_head, key, status, data=None): + """Creates a response for the given packet and request headers. - Packs status and data into a reply, encrypting according to `packet_head` + Packs status and data into a response, encrypting according to `packet_head` and `key`. Requires `packet_head` and `request_head` to be valid. """ - reply_nonce = bytearray(packet_head.nonce); reply_nonce[-1] ^= 0x1 - reply_packet_head = PacketHead(protocol_version=PROTOCOL_VERSION, - controllerID=packet_head.controllerID, - nonce=reply_nonce) - reply_head = ReplyHead(msg_type=request_head.msg_type, status=status.value) - return make_packet(reply_packet_head, reply_head, key, data) + response_nonce = bytearray(packet_head.nonce); response_nonce[-1] ^= 0x1 + response_packet_head = PacketHead(protocol_version=PROTOCOL_VERSION, + controllerID=packet_head.controllerID, + nonce=response_nonce) + response_head = ResponseHead(msg_type=request_head.msg_type, status=status.value) + return make_packet(response_packet_head, response_head, key, data) diff --git a/udpclient.py b/udpclient.py index 3338415..8e5a5c4 100644 --- a/udpclient.py +++ b/udpclient.py @@ -1,6 +1,7 @@ import config from gateserver import db from gateserver.controller_api import * +from gateserver.controller_protocol import * from gateserver.utils import mac2bytes import socket import os @@ -19,7 +20,7 @@ def request(mac, msgtype, data): key, data)) p, payload = parse_packet_head(res) - r, t, data = parse_r(ReplyHead, p, key, payload) + r, t, data = parse_payload(ReplyHead, p, key, payload) return p, r, t, data def prettyprint_reply(p, r, t, data): @@ -30,7 +31,7 @@ def prettyprint_reply(p, r, t, data): return '{} {}: {}'.format(t.name, s.name, data or '(no data)') if __name__ == '__main__': - _, mac, msgtype = sys.argv + mac, msgtype = sys.argv[1:] try: t = MsgType[msgtype.upper()] except KeyError: From 817e5dec2092aaa783f049e1e8975b6f8f588209 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamila=20Sou=C4=8Dkov=C3=A1?= Date: Tue, 8 Dec 2015 16:15:52 +0100 Subject: [PATCH 34/46] change controller_protocol based on review --- gateserver/controller_api.py | 33 ++++++----- gateserver/controller_protocol.py | 93 +++++++++++++++++-------------- gateserver/controller_server.py | 7 +-- gateserver/utils/structparse.py | 55 +++++++++++------- udpclient.py | 23 ++++---- 5 files changed, 120 insertions(+), 91 deletions(-) diff --git a/gateserver/controller_api.py b/gateserver/controller_api.py index 215b133..e898085 100644 --- a/gateserver/controller_api.py +++ b/gateserver/controller_api.py @@ -1,5 +1,5 @@ from .controller_protocol import (checkmsg, MsgType, ResponseStatus, - parse_packet_head, parse_request, BadMessageError) + parse_packet, parse_request, make_response_packet_for, BadMessageError) from . import db from . import utils @@ -7,7 +7,7 @@ def get_key_for_mac(mac): """Loads the key for this controller from the DB.""" rs = db.exec_sql('SELECT key FROM controller WHERE id = %s', (utils.bytes2mac(mac),), ret=True) - checkmsg(len(rs) == 1, 'unknown controllerID') + checkmsg(len(rs) == 1, 'unknown controller_id') return bytes(rs[0]['key']) def isic_id_repr(buf): @@ -16,28 +16,33 @@ def isic_id_repr(buf): checkmsg(length <= len(string), 'isic_id_repr length > buffer size') return string[:length] +# Note: once this function does something useful, it will be defined elsewhere +def handle_open(data): + s = ResponseStatus.OK if isic_id_repr(data) == b'Hello' else ResponseStatus.ERR + return s, None + process_request = { - MsgType.OPEN: - lambda data: ((ResponseStatus.OK if isic_id_repr(data) == b'Hello' - else ResponseStatus.ERR), None), + MsgType.OPEN: handle_open, } assert set(MsgType) == set(process_request), 'Not all message types handled' -def log_message(controllerID, mtype, indata, status): +def log_message(controller_id, request, status): """TODO""" - print(utils.bytes2mac(controllerID), mtype.name, indata, '->', status.name) + # print(utils.bytes2mac(controller_id), mtype.name, indata, '->', status.name) + print(utils.bytes2mac(controller_id), request, '->', status.name) def log_bad_packet(buf, e): """TODO""" raise e -def handle_request(buf): +def handle_packet(buf): try: - packet_head, payload = parse_packet_head(buf) - key = get_key_for_mac(packet_head.controllerID) - request_head, mtype, indata = parse_request(packet_head, key, payload) - status, outdata = process_request[mtype](indata) - log_message(packet_head.controllerID, mtype, indata, status) - return make_response_for(packet_head, request_head, key, status, outdata) + in_packet = parse_packet(buf) + key = get_key_for_mac(in_packet.controller_id) + request = parse_request(in_packet, key) + status, outdata = process_request[MsgType(request.msg_type)](request.data) + out_packet = make_response_packet_for(in_packet, key, request, status, outdata) + log_message(in_packet.controller_id, request, status) + return out_packet.pack() except BadMessageError as e: log_bad_packet(buf, e) diff --git a/gateserver/controller_protocol.py b/gateserver/controller_protocol.py index 1dffbbc..e2b1801 100644 --- a/gateserver/controller_protocol.py +++ b/gateserver/controller_protocol.py @@ -5,6 +5,7 @@ from .utils.structparse import * import nacl.raw as nacl +from enum import Enum PROTOCOL_VERSION = bytes([0x00,0x01]) @@ -21,72 +22,80 @@ class ResponseStatus(Enum): ERR = 0x10 TRY_AGAIN = 0x11 -PacketHead = mystruct('PacketHead', - (t.bytes(2) , 'protocol_version'), - (t.bytes(6) , 'controllerID' ), - (t.bytes(18), 'nonce' )) +Packet = mystruct('Packet', + (t.bytes(2) , 'protocol_version'), + (t.bytes(6) , 'controller_id' ), + (t.bytes(18), 'nonce' ), + (t.tail , 'payload' )) -RequestHead = mystruct('RequestHead', - (t.uint8, 'msg_type')) +Request = mystruct('Request', + (t.uint8, 'msg_type'), + (t.tail , 'data' )) -ResponseHead = mystruct('ResponseHead', - (t.uint8, 'msg_type'), - (t.uint8, 'status' )) +Response = mystruct('Response', + (t.uint8, 'msg_type'), + (t.uint8, 'status' ), + (t.tail , 'data' )) -def crypto_unwrap(packet_head, key, payload): - return nacl.crypto_secretbox_open(payload, - packet_head.controllerID + packet_head.nonce, key) +def crypto_unwrap_payload(packet, key): + return nacl.crypto_secretbox_open(packet.payload, + packet.controller_id + packet.nonce, key) -def crypto_wrap(packet_head, key, payload): +def crypto_wrap_payload(payload, controller_id, nonce, key): return nacl.crypto_secretbox(payload, - packet_head.controllerID + packet_head.nonce, key) + controller_id + nonce, key) -def parse_packet_head(buf): +def parse_packet(buf): """Parses the packet header, returning that and the rest of the data.""" - p, payload = PacketHead.unpack_from(buf) + try: + p = Packet.unpack(buf) + except ValueError as e: + raise BadMessageError(e.args) checkmsg(p.protocol_version == PROTOCOL_VERSION, 'Invalid protocol version') - return p, payload + return p -def parse_payload(struct, packet_head, key, payload): +def parse_payload(struct, packet, key): """Decrypts the payload and parses the request/response header. - Returns the parsed header (as struct), message type (as MsgType) and the - rest of the data. + struct: Request or Response + Returns the parsed payload (as struct). """ - assert struct in [RequestHead, ResponseHead] + assert struct in [Request, Response] try: - payload = crypto_unwrap(packet_head, key, payload) + payload = crypto_unwrap_payload(packet, key) except ValueError as e: raise BadMessageError('Decryption failed') from e - r, data = struct.unpack_from(payload) + r = struct.unpack(payload) try: - t = MsgType(r.msg_type) + _ = MsgType(r.msg_type) # check it is valid; TODO change to MsgType during unpack() except ValueError as e: raise BadMessageError('Unknown message type') from e - return r, t, data + return r -def parse_request(packet_head, key, payload): - return parse_payload(RequestHead, packet_head, key, payload) +def parse_request(packet, key): + return parse_payload(Request, packet, key) -def parse_response(packet_head, key, payload): - return parse_payload(ResponseHead, packet_head, key, payload) +def parse_response(packet, key): + return parse_payload(Response, packet, key) -def make_packet(packet_head, r_head, key, data=None): +def make_packet(controller_id, key, nonce, r): """Packs and encrypts the packet headers, request/response headers and data. + """ + encrypted = crypto_wrap_payload(r.pack(), controller_id, nonce, key) + return Packet(protocol_version=PROTOCOL_VERSION, + controller_id=controller_id, + nonce=nonce, + payload=encrypted) - Requires `packet_head` and `r_head` to be valid.""" - payload = r_head.pack() + (data or b'') - return packet_head.pack() + crypto_wrap(packet_head, key, payload) - -def make_response_for(packet_head, request_head, key, status, data=None): - """Creates a response for the given packet and request headers. +def make_response_packet_for(in_packet, key, request, status, data=None): + """Creates a response packet from the status and data for the given request. Packs status and data into a response, encrypting according to `packet_head` and `key`. Requires `packet_head` and `request_head` to be valid. """ - response_nonce = bytearray(packet_head.nonce); response_nonce[-1] ^= 0x1 - response_packet_head = PacketHead(protocol_version=PROTOCOL_VERSION, - controllerID=packet_head.controllerID, - nonce=response_nonce) - response_head = ResponseHead(msg_type=request_head.msg_type, status=status.value) - return make_packet(response_packet_head, response_head, key, data) + if not data: data = b'' + response = Response(msg_type=request.msg_type, + status=status.value, + data=data) + response_nonce = bytearray(in_packet.nonce); response_nonce[-1] ^= 0x1 + return make_packet(in_packet.controller_id, key, response_nonce, response) diff --git a/gateserver/controller_server.py b/gateserver/controller_server.py index 481f79d..178fedd 100644 --- a/gateserver/controller_server.py +++ b/gateserver/controller_server.py @@ -4,12 +4,11 @@ import socketserver class MessageHandler(socketserver.BaseRequestHandler): - """Handles a message from the controller.""" - def handle(self): + """Handles a request from the controller.""" in_packet, socket = self.request - out_packet = controller_api.handle_request(in_packet) - socket.sendto(out_packet, self.client_address) + out_packet = controller_api.handle_packet(in_packet) + if out_packet: socket.sendto(out_packet, self.client_address) def serve(config): bind_addr = config.udp_host, config.udp_port diff --git a/gateserver/utils/structparse.py b/gateserver/utils/structparse.py index 3fc94cf..245375e 100644 --- a/gateserver/utils/structparse.py +++ b/gateserver/utils/structparse.py @@ -1,36 +1,53 @@ from collections import namedtuple -from struct import Struct -from enum import Enum -from . import unzip -ENDIANITY = '<' # little-endian, no alignment (i.e. packed) +def unzip(x): return zip(*x) + +class _Type: + """Defines a type that can be serialized and unserialized.""" + def __init__(self, pack, unpack): + """ + pack: data -> buffer + unpack: buffer -> (parsed data, rest of the buffer) + """ + self.pack = pack + self.unpack = unpack + +def idf(x): return x class t: - """pieces of struct format strings: docs.python.org/3/library/struct.html""" - uint8 = 'B' - bytes = lambda sz: '{}s'.format(sz) + """Defines a few useful types.""" + # Note: endianity must be handled once multi-byte numbers are needed + tail = _Type(idf, lambda buf: (buf, bytes([]))) + uint8 = _Type(lambda x: bytes([x]), lambda buf: (int(buf[0]), buf[1:])) + bytes = lambda n: _Type(idf, lambda buf: (buf[:n], buf[n:])) + #pstr = lambda n: _Type(lambda s: [n]+bytes(s) if len(s) <= n else raise ValueError('error when encoding TODO'), lambda buf: (buf[1:buf[0]]), buf[n+1:]) class MyStructMixin: - _struct = None - + """Mixin providing the `pack` and `unpack` methods.""" + @classmethod - def unpack_from(cls, buf): - """Constructs a new instance by unpacking the given buffer. + def unpack(cls, data): + """Constructs a new instance by unpacking the given buffer.""" + def _unpack(data): + for t in cls._fieldtypes: + val, data = t.unpack(data) + yield val + if len(data) > 0: raise ValueError('buffer size != struct size') - Returns the new instance and the rest of the buffer. - """ - sz = cls._struct.size - head, tail = buf[:sz], buf[sz:] - return cls(*cls._struct.unpack(head)), tail + return cls(*_unpack(data)) def pack(self): """Returns itself packed as `bytes`.""" - return self._struct.pack(*self) + return b''.join([ t.pack(x) for t,x in zip(self._fieldtypes, self) ]) def mystruct(name, *fields): - """Creates a namedtuple that can be packed to and unpacked from `bytes`.""" + """Creates a namedtuple that can be packed to and unpacked from `bytes`. + + `unpack(buf)` will raise `ValueError` if `len(buf)` doesn't exactly match + the struct's size. + """ fieldtypes, fieldnames = unzip(fields) class Cls(namedtuple(name, fieldnames), MyStructMixin): pass Cls.__name__ = name - Cls._struct = Struct(ENDIANITY + ''.join(fieldtypes)) + Cls._fieldtypes = fieldtypes return Cls diff --git a/udpclient.py b/udpclient.py index 8e5a5c4..9894282 100644 --- a/udpclient.py +++ b/udpclient.py @@ -15,20 +15,19 @@ def msg(buf): def request(mac, msgtype, data): key = get_key_for_mac(mac) nonce = os.urandom(18) - res = msg(make_packet(PacketHead(PROTOCOL_VERSION, mac, nonce), - RequestHead(msgtype.value), - key, - data)) - p, payload = parse_packet_head(res) - r, t, data = parse_payload(ReplyHead, p, key, payload) - return p, r, t, data + req = Request(msgtype.value, data) + res = msg(make_packet(mac, key, nonce, req).pack()) + p = parse_packet(res) + r = parse_response(p, key) + return r -def prettyprint_reply(p, r, t, data): +def prettyprint_reply(r): try: - s = ReplyStatus(r.status) + t = MsgType(r.msg_type) + s = ResponseStatus(r.status) except ValueError: - raise BadMessageError('Unknown status {}'.format(r.status)) - return '{} {}: {}'.format(t.name, s.name, data or '(no data)') + raise BadMessageError('Unknown type or status {}'.format(r.status)) + return '{} {}: {}'.format(t.name, s.name, r.data or '(no data)') if __name__ == '__main__': mac, msgtype = sys.argv[1:] @@ -41,4 +40,4 @@ def prettyprint_reply(p, r, t, data): db.connect(config.db_url) reply = request(mac2bytes(mac), t, indata) - print(prettyprint_reply(*reply)) + print(prettyprint_reply(reply)) From d02e545adec9b307b5179d5d1415d8793fe3ebee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamila=20Sou=C4=8Dkov=C3=A1?= Date: Sun, 3 Apr 2016 13:41:18 +0200 Subject: [PATCH 35/46] more packet-parsing refactoring; structparse now separate --- controller_client.py | 40 +++++++++ {gateserver => deadserver}/__init__.py | 0 deadserver/controller_api.py | 48 +++++++++++ deadserver/controller_protocol.py | 96 +++++++++++++++++++++ deadserver/controller_server.py | 33 ++++++++ deadserver/db.py | 19 +++++ deadserver/utils/__init__.py | 4 + gateserver/controller_api.py | 48 ----------- gateserver/controller_protocol.py | 101 ----------------------- gateserver/controller_server.py | 16 ---- gateserver/db.py | 21 ----- gateserver/utils/__init__.py | 10 --- gateserver/utils/structparse.py | 53 ------------ runserver.py | 8 +- structparse/__init__.py | 63 ++++++++++++++ {tests => structparse/tests}/__init__.py | 0 structparse/tests/test_structparse.py | 41 +++++++++ structparse/tests/test_types.py | 75 +++++++++++++++++ structparse/types.py | 100 ++++++++++++++++++++++ tests/config.py.example | 9 -- udpclient.py | 43 ---------- 21 files changed, 523 insertions(+), 305 deletions(-) create mode 100644 controller_client.py rename {gateserver => deadserver}/__init__.py (100%) create mode 100644 deadserver/controller_api.py create mode 100644 deadserver/controller_protocol.py create mode 100644 deadserver/controller_server.py create mode 100644 deadserver/db.py create mode 100644 deadserver/utils/__init__.py delete mode 100644 gateserver/controller_api.py delete mode 100644 gateserver/controller_protocol.py delete mode 100644 gateserver/controller_server.py delete mode 100644 gateserver/db.py delete mode 100644 gateserver/utils/__init__.py delete mode 100644 gateserver/utils/structparse.py create mode 100644 structparse/__init__.py rename {tests => structparse/tests}/__init__.py (100%) create mode 100644 structparse/tests/test_structparse.py create mode 100644 structparse/tests/test_types.py create mode 100644 structparse/types.py delete mode 100644 tests/config.py.example delete mode 100644 udpclient.py diff --git a/controller_client.py b/controller_client.py new file mode 100644 index 0000000..9dec868 --- /dev/null +++ b/controller_client.py @@ -0,0 +1,40 @@ +"""Quick & dirty client (i.e. the controller end), used for manual testing of the server.""" + +import config +from deadserver import db +from deadserver.controller_api import * +from deadserver.controller_protocol import * +import socket +import os +import sys + +api = API(db_conn=db.Connection(config.db_url)) + +def msg(buf): + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + sock.sendto(buf, (config.udp_host, config.udp_port)) + return sock.recv(1024) + +def send(id, msgtype, data): + nonce = os.urandom(18) + req = Request(msgtype.value, data) + req_packet = make_packet(id, nonce, req, get_key=api.get_key) + res_packet = msg(req_packet.pack()) + return parse_packet(Response, res_packet, get_key=api.get_key) + +if __name__ == '__main__': + mac, msgtype = sys.argv[1:] + try: + t = MsgType[msgtype.upper()] + except KeyError: + sys.exit('No such message type: '+msgtype) + + indata = sys.stdin.buffer.read() + + hdr, res = send(str2id(mac), t, indata) + + print(' * * * as {} sent request: {}'.format(mac, str(t))) + print(indata) + print(' * * * received response: {}'.format(str(t))) + print(str(res.status)) + print(res.data) diff --git a/gateserver/__init__.py b/deadserver/__init__.py similarity index 100% rename from gateserver/__init__.py rename to deadserver/__init__.py diff --git a/deadserver/controller_api.py b/deadserver/controller_api.py new file mode 100644 index 0000000..45214de --- /dev/null +++ b/deadserver/controller_api.py @@ -0,0 +1,48 @@ +"""The controller ↔ server API -- the business logic. + +This knows what should happen for a given request. See `controller_protocol` for +the message format details. +""" + +from . import controller_protocol as protocol +from .controller_protocol import MsgType, ResponseStatus as Status + +from structparse.types import Tail # TODO remove + +class API: + def __init__(self, db_conn): + self.db = db_conn + + def handle_packet(self, in_buf): + try: + request_header, request = protocol.parse_packet(protocol.Request, in_buf, get_key=self.get_key) + status, response_data = self.process_request[request.msg_type](request_header.controller_id, request.data) # TODO this will be rewritten + self.log_message(request_header.controller_id, request, status) + response_packet = protocol.make_response_packet_for(request_header, request.msg_type, status, response_data, get_key=self.get_key) + return response_packet.pack() + except protocol.BadMessageError as e: + log_bad_message(buf, e) + + # TODO this table will be dynamic via handler registration -- pretend it doesn't exist + process_request = { + # TODO this should get data.isic_id, except that's not implemented yet and structparse needs unions and stuff + MsgType.OPEN: (lambda id, data: ((Status.OK if data == Tail(b'Hello') else Status.ERR), None)) + } + + def get_key(self, id): + """Loads the key for this controller from the DB.""" + rs = self.db.exec_sql('SELECT key FROM controller WHERE id = %s', + (protocol.id2str(id),), ret=True) + protocol.check(len(rs) == 1, 'unknown controller ID') + return bytes(rs[0]['key']) + + def log_message(self, controller_id, request, status): + """TODO""" + # print(utils.bytes2mac(controller_id), mtype.name, indata, '->', status.name) + print(protocol.id2str(controller_id), request, '->', status.name) + + def log_bad_message(self, buf, e): + """TODO""" + raise e + +assert set(protocol.MsgType) == set(API.process_request), 'Not all message types handled' diff --git a/deadserver/controller_protocol.py b/deadserver/controller_protocol.py new file mode 100644 index 0000000..c8c7522 --- /dev/null +++ b/deadserver/controller_protocol.py @@ -0,0 +1,96 @@ +"""The controller ↔ server protocol message structure. + +This knows the data format for the various structures in the protocol. See +`controller_api` for the behavior / business logic. +""" + +# TODO: consider [CBOR](http://cbor.io/). +# TODO: With the faster processor, we probably can afford assymetric crypto. Switch if possible. + +from structparse import mystruct, types +import nacl.raw as nacl +from enum import Enum + +class BadMessageError(Exception): pass + +def check(expression, errmsg): + if not expression: raise BadMessageError(errmsg) + +PROTOCOL_VERSION = types.Bytes(2)([0,1]) + +class MsgType(types.Uint8, Enum): + OPEN = 1 + +class ResponseStatus(types.Uint8, Enum): + OK = 0x01 + ERR = 0x10 + TRY_AGAIN = 0x11 + +Request = mystruct('Request', + (MsgType, 'msg_type'), + (types.Tail, 'data' )) + +Response = mystruct('Response', + (MsgType, 'msg_type'), + (ResponseStatus, 'status' ), + (types.Tail, 'data' )) + +PacketHeader = mystruct('PacketHeader', + (types.Bytes(2), 'protocol_version'), + (types.Bytes(6), 'controller_id' ), + (types.Bytes(18), 'nonce' )) + +Packet = mystruct('Packet', + (PacketHeader, 'header'), + (types.Tail, 'payload')) + +def id2str(id): + return ':'.join('{:02x}'.format(x) for x in id.val) + +def str2id(s): + return types.Bytes(6)(bytes.fromhex(s.replace(':', ''))) + +def crypto_unwrap_payload(nonce, payload, key): + return nacl.crypto_secretbox_open(payload, nonce, key) + +def crypto_wrap_payload(nonce, payload, key): + return nacl.crypto_secretbox(payload, nonce, key) + +def parse_packet(struct, buf, get_key): + """Parses the buffer into PacketHeader and `struct`, which must be Request or Response. + + Decrypts the payload with the key returned by the `get_key` function. + """ + assert struct in [Request, Response] + + try: + hdr, tail = PacketHeader.unpack(buf) + check(hdr.protocol_version == PROTOCOL_VERSION, 'Invalid protocol version') + payload_buf = crypto_unwrap_payload(hdr.controller_id.val + hdr.nonce.val, + tail, get_key(hdr.controller_id)) + payload = struct.unpack_all(payload_buf) + except ValueError as e: + raise BadMessageError('parse_packet failed') from e + + return hdr, payload + +def make_packet(controller_id, nonce, payload, get_key): + """Packs and encrypts the packet headers, request/response headers and data.""" + encrypted = crypto_wrap_payload(controller_id.val + nonce, payload.pack(), get_key(controller_id)) + return Packet(PacketHeader(protocol_version=PROTOCOL_VERSION, + controller_id=controller_id, + nonce=nonce), + encrypted) + +def make_response_packet_for(request_header, msg_type, status, response_data, get_key): + """Creates a response packet from the status and data for the given request. + + Packs status and data into a response, encrypting according to the key returned by `get_key`. + Requires `request_packet` to be valid. + """ + if not response_data: response_data = b'' + response = Response(msg_type=msg_type, + status=status, + data=response_data) + response_nonce = bytearray(request_header.nonce.val); response_nonce[-1] ^= 0x1 + return make_packet(request_header.controller_id, response_nonce, response, get_key) diff --git a/deadserver/controller_server.py b/deadserver/controller_server.py new file mode 100644 index 0000000..9da0a60 --- /dev/null +++ b/deadserver/controller_server.py @@ -0,0 +1,33 @@ +"""The UDP server that handles controller messages. + +This is not concerned with the protocol details, it only knows how to receive +requests and send responses, and passes stuff to `controller_api`. +""" + +from . import controller_api, db +import functools +import socketserver + +class MessageHandler(socketserver.BaseRequestHandler): + def __init__(self, api, *args, **kwargs): + self.api = api + super().__init__(*args, **kwargs) + + def handle(self): + """Handles a request from the controller.""" + in_packet, socket = self.request + out_packet = self.api.handle_packet(in_packet) + if out_packet: socket.sendto(out_packet, self.client_address) + +class DeadServer: + def __init__(self, config): + self.config = config + self.db_conn = db.Connection(config.db_url) + self.api = controller_api.API(db_conn=self.db_conn) + + bind_addr = self.config.udp_host, self.config.udp_port + handler = functools.partial(MessageHandler, self.api) + self.server = socketserver.ThreadingUDPServer(bind_addr, handler) + + def serve(self): + self.server.serve_forever() diff --git a/deadserver/db.py b/deadserver/db.py new file mode 100644 index 0000000..1698358 --- /dev/null +++ b/deadserver/db.py @@ -0,0 +1,19 @@ +"""Holds the (global) connection to the DB.""" +# TODO maybe use a connection pool one beautiful day + +import psycopg2 +from psycopg2.extras import RealDictCursor # results as dict instead of tuple + +class Connection: + def __init__(self, db_url): + self.conn = psycopg2.connect(db_url, cursor_factory=RealDictCursor) + self.conn.autocommit = True + + def exec_sql(self, query, args=(), ret=False): + """Executes the query, returning the result as a list if `ret`.""" + with self.conn.cursor() as cur: + cur.execute(query, args) + if ret: return cur.fetchall() + +# thrown when constraints aren't satisfied +IntegrityError = psycopg2.IntegrityError diff --git a/deadserver/utils/__init__.py b/deadserver/utils/__init__.py new file mode 100644 index 0000000..16dd3f8 --- /dev/null +++ b/deadserver/utils/__init__.py @@ -0,0 +1,4 @@ +"""Various utility functions.""" + +def unzip(lst): + return zip(*lst) # yay :D diff --git a/gateserver/controller_api.py b/gateserver/controller_api.py deleted file mode 100644 index e898085..0000000 --- a/gateserver/controller_api.py +++ /dev/null @@ -1,48 +0,0 @@ -from .controller_protocol import (checkmsg, MsgType, ResponseStatus, - parse_packet, parse_request, make_response_packet_for, BadMessageError) -from . import db -from . import utils - -def get_key_for_mac(mac): - """Loads the key for this controller from the DB.""" - rs = db.exec_sql('SELECT key FROM controller WHERE id = %s', - (utils.bytes2mac(mac),), ret=True) - checkmsg(len(rs) == 1, 'unknown controller_id') - return bytes(rs[0]['key']) - -def isic_id_repr(buf): - """See https://github.com/fmfi-svt-gate/server/wiki/Controller-%E2%86%94-Server-Protocol#note-isic-ids-representation""" - length, string = buf[0], buf[1:] - checkmsg(length <= len(string), 'isic_id_repr length > buffer size') - return string[:length] - -# Note: once this function does something useful, it will be defined elsewhere -def handle_open(data): - s = ResponseStatus.OK if isic_id_repr(data) == b'Hello' else ResponseStatus.ERR - return s, None - -process_request = { - MsgType.OPEN: handle_open, -} -assert set(MsgType) == set(process_request), 'Not all message types handled' - -def log_message(controller_id, request, status): - """TODO""" - # print(utils.bytes2mac(controller_id), mtype.name, indata, '->', status.name) - print(utils.bytes2mac(controller_id), request, '->', status.name) - -def log_bad_packet(buf, e): - """TODO""" - raise e - -def handle_packet(buf): - try: - in_packet = parse_packet(buf) - key = get_key_for_mac(in_packet.controller_id) - request = parse_request(in_packet, key) - status, outdata = process_request[MsgType(request.msg_type)](request.data) - out_packet = make_response_packet_for(in_packet, key, request, status, outdata) - log_message(in_packet.controller_id, request, status) - return out_packet.pack() - except BadMessageError as e: - log_bad_packet(buf, e) diff --git a/gateserver/controller_protocol.py b/gateserver/controller_protocol.py deleted file mode 100644 index e2b1801..0000000 --- a/gateserver/controller_protocol.py +++ /dev/null @@ -1,101 +0,0 @@ -"""Implements the Controller <-> Server protocol. - -See https://github.com/fmfi-svt-gate/server/wiki/Controller-%E2%86%94-Server-Protocol . -""" - -from .utils.structparse import * -import nacl.raw as nacl -from enum import Enum - -PROTOCOL_VERSION = bytes([0x00,0x01]) - -class BadMessageError(Exception): pass - -def checkmsg(expression, errmsg): - if not expression: raise BadMessageError(errmsg) - -class MsgType(Enum): - OPEN = 1 - -class ResponseStatus(Enum): - OK = 0x01 - ERR = 0x10 - TRY_AGAIN = 0x11 - -Packet = mystruct('Packet', - (t.bytes(2) , 'protocol_version'), - (t.bytes(6) , 'controller_id' ), - (t.bytes(18), 'nonce' ), - (t.tail , 'payload' )) - -Request = mystruct('Request', - (t.uint8, 'msg_type'), - (t.tail , 'data' )) - -Response = mystruct('Response', - (t.uint8, 'msg_type'), - (t.uint8, 'status' ), - (t.tail , 'data' )) - -def crypto_unwrap_payload(packet, key): - return nacl.crypto_secretbox_open(packet.payload, - packet.controller_id + packet.nonce, key) - -def crypto_wrap_payload(payload, controller_id, nonce, key): - return nacl.crypto_secretbox(payload, - controller_id + nonce, key) - -def parse_packet(buf): - """Parses the packet header, returning that and the rest of the data.""" - try: - p = Packet.unpack(buf) - except ValueError as e: - raise BadMessageError(e.args) - checkmsg(p.protocol_version == PROTOCOL_VERSION, 'Invalid protocol version') - return p - -def parse_payload(struct, packet, key): - """Decrypts the payload and parses the request/response header. - - struct: Request or Response - Returns the parsed payload (as struct). - """ - assert struct in [Request, Response] - try: - payload = crypto_unwrap_payload(packet, key) - except ValueError as e: - raise BadMessageError('Decryption failed') from e - r = struct.unpack(payload) - try: - _ = MsgType(r.msg_type) # check it is valid; TODO change to MsgType during unpack() - except ValueError as e: - raise BadMessageError('Unknown message type') from e - return r - -def parse_request(packet, key): - return parse_payload(Request, packet, key) - -def parse_response(packet, key): - return parse_payload(Response, packet, key) - -def make_packet(controller_id, key, nonce, r): - """Packs and encrypts the packet headers, request/response headers and data. - """ - encrypted = crypto_wrap_payload(r.pack(), controller_id, nonce, key) - return Packet(protocol_version=PROTOCOL_VERSION, - controller_id=controller_id, - nonce=nonce, - payload=encrypted) - -def make_response_packet_for(in_packet, key, request, status, data=None): - """Creates a response packet from the status and data for the given request. - - Packs status and data into a response, encrypting according to `packet_head` - and `key`. Requires `packet_head` and `request_head` to be valid. - """ - if not data: data = b'' - response = Response(msg_type=request.msg_type, - status=status.value, - data=data) - response_nonce = bytearray(in_packet.nonce); response_nonce[-1] ^= 0x1 - return make_packet(in_packet.controller_id, key, response_nonce, response) diff --git a/gateserver/controller_server.py b/gateserver/controller_server.py deleted file mode 100644 index 178fedd..0000000 --- a/gateserver/controller_server.py +++ /dev/null @@ -1,16 +0,0 @@ -"""The UDP server that provides the API for the controllers.""" - -from . import controller_api -import socketserver - -class MessageHandler(socketserver.BaseRequestHandler): - def handle(self): - """Handles a request from the controller.""" - in_packet, socket = self.request - out_packet = controller_api.handle_packet(in_packet) - if out_packet: socket.sendto(out_packet, self.client_address) - -def serve(config): - bind_addr = config.udp_host, config.udp_port - server = socketserver.ThreadingUDPServer(bind_addr, MessageHandler) - server.serve_forever() diff --git a/gateserver/db.py b/gateserver/db.py deleted file mode 100644 index 34ce816..0000000 --- a/gateserver/db.py +++ /dev/null @@ -1,21 +0,0 @@ -"""Holds the (global) connection to the DB.""" -# TODO maybe use a connection pool one beautiful day - -import psycopg2 -from psycopg2.extras import RealDictCursor # results as dict instead of tuple - -conn = None - -def connect(db_url): - global conn - conn = psycopg2.connect(db_url, cursor_factory=RealDictCursor) - conn.autocommit = True - -def exec_sql(query, args=(), ret=False): - """Execute the query, returning the result as a list if `ret` is set.""" - with conn.cursor() as cur: - cur.execute(query, args) - if ret: return cur.fetchall() - -# thrown when constraints aren't satisfied -IntegrityError = psycopg2.IntegrityError diff --git a/gateserver/utils/__init__.py b/gateserver/utils/__init__.py deleted file mode 100644 index 4dfdee7..0000000 --- a/gateserver/utils/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ -"""Various utility functions.""" - -def bytes2mac(mac): - return ':'.join('{:02x}'.format(x) for x in mac) - -def mac2bytes(s): - return bytes.fromhex(s.replace(':', '')) - -def unzip(lst): - return zip(*lst) # yay :D diff --git a/gateserver/utils/structparse.py b/gateserver/utils/structparse.py deleted file mode 100644 index 245375e..0000000 --- a/gateserver/utils/structparse.py +++ /dev/null @@ -1,53 +0,0 @@ -from collections import namedtuple - -def unzip(x): return zip(*x) - -class _Type: - """Defines a type that can be serialized and unserialized.""" - def __init__(self, pack, unpack): - """ - pack: data -> buffer - unpack: buffer -> (parsed data, rest of the buffer) - """ - self.pack = pack - self.unpack = unpack - -def idf(x): return x - -class t: - """Defines a few useful types.""" - # Note: endianity must be handled once multi-byte numbers are needed - tail = _Type(idf, lambda buf: (buf, bytes([]))) - uint8 = _Type(lambda x: bytes([x]), lambda buf: (int(buf[0]), buf[1:])) - bytes = lambda n: _Type(idf, lambda buf: (buf[:n], buf[n:])) - #pstr = lambda n: _Type(lambda s: [n]+bytes(s) if len(s) <= n else raise ValueError('error when encoding TODO'), lambda buf: (buf[1:buf[0]]), buf[n+1:]) - -class MyStructMixin: - """Mixin providing the `pack` and `unpack` methods.""" - - @classmethod - def unpack(cls, data): - """Constructs a new instance by unpacking the given buffer.""" - def _unpack(data): - for t in cls._fieldtypes: - val, data = t.unpack(data) - yield val - if len(data) > 0: raise ValueError('buffer size != struct size') - - return cls(*_unpack(data)) - - def pack(self): - """Returns itself packed as `bytes`.""" - return b''.join([ t.pack(x) for t,x in zip(self._fieldtypes, self) ]) - -def mystruct(name, *fields): - """Creates a namedtuple that can be packed to and unpacked from `bytes`. - - `unpack(buf)` will raise `ValueError` if `len(buf)` doesn't exactly match - the struct's size. - """ - fieldtypes, fieldnames = unzip(fields) - class Cls(namedtuple(name, fieldnames), MyStructMixin): pass - Cls.__name__ = name - Cls._fieldtypes = fieldtypes - return Cls diff --git a/runserver.py b/runserver.py index 1d5b736..f657971 100755 --- a/runserver.py +++ b/runserver.py @@ -1,9 +1,9 @@ #!/usr/bin/env python3 -"""Gate server runner.""" +"""Deadlock server runner.""" -from gateserver import db, controller_server +from deadserver import controller_server import config if __name__ == '__main__': - db.connect(config.db_url) - controller_server.serve(config) + app = controller_server.DeadServer(config) + app.serve() diff --git a/structparse/__init__.py b/structparse/__init__.py new file mode 100644 index 0000000..304ab32 --- /dev/null +++ b/structparse/__init__.py @@ -0,0 +1,63 @@ +"""Simplifies parsing "C structs" -- structured types serialized as byte arrays.""" + +from collections import namedtuple + +def unzip(x): return zip(*x) + +class Type: + """Defines a type that can be serialized and unserialized.""" + + @staticmethod + def unpack(buf): + """Constructs a Python value from the given buffer. + + Signature: buffer -> (parsed data, rest of the buffer) + """ + raise NotImplementedError + + def pack(self): + """Packs itself into `bytes`, returns that buffer. + + Signature: data -> buffer + """ + raise NotImplementedError + +class MyStructMixin(Type): + """Mixin providing the `pack` and `unpack` methods for a struct.""" + def __new__(cls, *args, **kwargs): + if len(args) == 1 and len(kwargs) == 0 and args[0].__class__ is cls: + return args[0] # assumes immutability + if len(args) + len(kwargs) != len(cls._fields): + raise TypeError('Must be initialized with exactly {} arguments'.format(len(cls._fields))) + field_types = dict(zip(cls._fields, cls._fieldtypes)) + to_convert = dict(zip(cls._fields, args), **kwargs) + converted = { n: field_types[n](v) for (n, v) in to_convert.items() } + return super().__new__(cls, **converted) + + @classmethod + def unpack(cls, buf): + """Constructs a new instance by unpacking the given buffer.""" + vals = [] + for t in cls._fieldtypes: + val, buf = t.unpack(buf) + vals.append(val) + return cls(*vals), buf + + def pack(self): + """Returns itself packed as `bytes`.""" + return b''.join([ t.pack(x) for t,x in zip(self._fieldtypes, self) ]) + + @classmethod + def unpack_all(cls, buf): + """raises ValueError if len(buf) doesn't exactly match the struct size.""" + x, rest = cls.unpack(buf) + if len(rest) > 0: raise ValueError('buffer size != struct size') + return x + +def mystruct(name, *fields): + """Creates a "C struct" -- a namedtuple that can be packed and unpacked.""" + fieldtypes, fieldnames = unzip(fields) + class Cls(MyStructMixin, namedtuple(name, fieldnames)): pass + Cls.__name__ = name + Cls._fieldtypes = fieldtypes + return Cls diff --git a/tests/__init__.py b/structparse/tests/__init__.py similarity index 100% rename from tests/__init__.py rename to structparse/tests/__init__.py diff --git a/structparse/tests/test_structparse.py b/structparse/tests/test_structparse.py new file mode 100644 index 0000000..a22b1fb --- /dev/null +++ b/structparse/tests/test_structparse.py @@ -0,0 +1,41 @@ +import pytest + +from .. import mystruct, types + +def test_new(): + TestStruct = mystruct('TestStruct', + (types.Uint8, 'john'), + (types.Uint8, 'paul'), + (types.Uint8, 'george'), + (types.Uint8, 'ringo')) + def check(instance): + assert (instance.john == types.Uint8(12) and + instance.paul == types.Uint8(4) and + instance.george == types.Uint8(6) and + instance.ringo == types.Uint8(0)) + check(TestStruct(12, 4, 6, 0)) + check(TestStruct(paul=4, george=6, john=12, ringo=0)) + check(TestStruct(12, 4, ringo=0, george=6)) + with pytest.raises(TypeError) as err: + TestStruct(1, 2, 47) + assert 'exactly 4' in str(err.value) + with pytest.raises(TypeError) as err: + TestStruct(12, 4, john=42, paul=47) + assert 'arguments' in str(err.value) + +SubStruct = mystruct('SubStruct', + (types.Bytes(4), 'x'), + (types.Uint8, 'y'), + (types.PascalStr(5), 'z')) +Struct = mystruct('Struct', + (SubStruct, 'a'), + (types.Tail, 'b')) + +s = Struct(SubStruct(b'quux', 0x47, 'foo'), b'kaleraby') +packed = b'quux' + b'\x47' + b'\x03foo\x00\x00' + b'kaleraby' + +def test_pack(): + assert s.pack() == packed + +def test_unpack(): + assert Struct.unpack_all(packed) == s diff --git a/structparse/tests/test_types.py b/structparse/tests/test_types.py new file mode 100644 index 0000000..5bc351b --- /dev/null +++ b/structparse/tests/test_types.py @@ -0,0 +1,75 @@ +import pytest + +from .. import types +from enum import Enum + +def test_construction(): + x = types.Uint8(47) + y = types.Uint8(x) + assert x == y + +def test_tobytes(): + assert types._tobytes([97, 98, 99]) == b'abc' + assert types._tobytes('mňau') == b'm\xc5\x88au' + assert len(types._tobytes('mňau')) == 5 + +def test_eq(): + assert types.Uint8(47) == types.Uint8(47) + assert types.Uint8(42) != types.Uint8(47) + +def test_Uint8(): + assert types.Uint8(97).pack() == b'a' + assert types.Uint8.unpack(b'abcd') == (types.Uint8(97), b'bcd') + with pytest.raises(ValueError): types.Uint8(4742) + +def test_Tail(): + assert types.Tail(b'an arbitrarily long whatever').pack() == b'an arbitrarily long whatever' + assert types.Tail([97, 98, 99, 100]) == types.Tail(b'abcd') + assert types.Tail.unpack(b'mrkva') == (types.Tail(b'mrkva'), b'') + +def test_Bytes(): + b4 = types.Bytes(4) + assert repr(b4(b'abcd')) == "Bytes[4](b'abcd')" + + assert b4([97, 98, 99, 100]) == b4(b'abcd') + + with pytest.raises(ValueError): b4(b'abc') + with pytest.raises(ValueError): b4(b'abcde') + + assert b4(b'abcd').pack() == b'abcd' + assert b4.unpack(b'abcdefg') == (b4(b'abcd'), b'efg') + with pytest.raises(ValueError): b4.unpack(b'abc') + +def test_PascalStr(): + p5 = types.PascalStr(5) + assert repr(p5(b'abcd')) == "PascalStr[5](b'abcd')" + + with pytest.raises(ValueError): p5('Hello World') + + assert p5('hello').pack() == b'\x05hello' + assert p5('hell').pack() == b'\x04hell\x00' + + assert p5.unpack(b'\x03hel\x00\x00 world') == (p5(b'hel'), b' world') + with pytest.raises(ValueError): p5.unpack(b'\x03hello world') + with pytest.raises(ValueError): p5.unpack(b'\x47anything') + +def test_hashable(): + assert hash(types.Uint8(47)) == hash(types.Uint8(47)) + assert hash(types.Uint8(47)) != hash(types.Uint8(42)) + assert hash(types.Bytes(2)([42,47])) != hash(types.Tail([42,47])) + +def test_Enum_works(): + class T(types.Uint8, Enum): + A = 1 + B = 2 + Z = 255 + + assert T(1) == T.A + assert T.A.pack() == b'\x01' + assert T.unpack(b'\xff') == (T.Z, b'') + with pytest.raises(ValueError): T(47) + with pytest.raises(ValueError): T.unpack(b'\x47') + + with pytest.raises(ValueError): + class T(types.Uint8, Enum): + X = 4742 # does not fit into 1 byte diff --git a/structparse/types.py b/structparse/types.py new file mode 100644 index 0000000..9a41516 --- /dev/null +++ b/structparse/types.py @@ -0,0 +1,100 @@ +"""Defines default types for structparse.""" + +from . import Type + +class _SimpleType(Type): + def __init__(self, x): + if x.__class__ is self.__class__: self.__val = x.val + else: + self._validate(x) + self.__val = x + + @property + def val(self): + return self.__val + + def pack(self): + return bytes(self._pack()) + + @classmethod + def unpack(cls, buf): + val, rest = cls._unpack(buf) + return cls(val), rest + + def _validate(self, input): + """No-op if the input is valid or raises exception if invalid.""" + pass + + def _pack(self): + return self.val + + def __eq__(self, other): + return self.val == other.val + + def __hash__(self): + return hash(self.__class__.__name__) ^ hash(self.val) + + def __repr__(self): + return self.__class__.__name__+'('+repr(self.val)+')' + +def _tobytes(x): + if isinstance(x, _SimpleType): return bytes(x.val) + if isinstance(x, str): return bytes(x, 'utf8') + return bytes(x) + +class Uint8(_SimpleType): + @staticmethod + def _unpack(buf): + return int(buf[0]), buf[1:] + + @staticmethod + def _validate(x): + if not 0 <= x <= 0xff: raise ValueError('{} is not a 1-byte unsigned int'.format(x)) + + def _pack(self): + return [self.val] + +class _BytesLike(_SimpleType): + def __init__(self, arg): + super().__init__(_tobytes(arg)) + +class Tail(_BytesLike): + @staticmethod + def _unpack(buf): + return buf, b'' + +def Bytes(n): + class Cls(_BytesLike): + @staticmethod + def _validate(arg): + if len(arg) != n: + raise ValueError('{} is not {} bytes'.format(arg, n)) + return bytes(arg) + + @staticmethod + def _unpack(buf): + return buf[:n], buf[n:] + + Cls.__name__ = 'Bytes[{}]'.format(n) + return Cls + +def PascalStr(n): + class Cls(_BytesLike): + @staticmethod + def _validate(arg): + if len(arg) > n: + raise ValueError('string too long (n = {})'.format(n)) + + @staticmethod + def _unpack(buf): + if buf[0] > n: raise ValueError('packed string too long (n = {}, buf[0] = {})'.format(n, buf[0])) + b, e = buf[0]+1, n+1 + if buf[b:e] != b'\0'*(e-b): raise ValueError('packed string not null-padded') + return buf[1:b], buf[e:] + + def _pack(self): + padding = n - len(self.val) + return bytes([len(self.val)]) + self.val + b'\0'*padding + + Cls.__name__ = 'PascalStr[{}]'.format(n) + return Cls diff --git a/tests/config.py.example b/tests/config.py.example deleted file mode 100644 index 3e21c46..0000000 --- a/tests/config.py.example +++ /dev/null @@ -1,9 +0,0 @@ -"""Tests configuration.""" - -http_host = 'localhost' -http_port = 9047 - -udp_host = 'localhost' -udp_port = 9042 - -db_url = 'postgresql://user:password@localhost/gate_test' diff --git a/udpclient.py b/udpclient.py deleted file mode 100644 index 9894282..0000000 --- a/udpclient.py +++ /dev/null @@ -1,43 +0,0 @@ -import config -from gateserver import db -from gateserver.controller_api import * -from gateserver.controller_protocol import * -from gateserver.utils import mac2bytes -import socket -import os -import sys - -def msg(buf): - sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - sock.sendto(buf, (config.udp_host, config.udp_port)) - return sock.recv(1024) - -def request(mac, msgtype, data): - key = get_key_for_mac(mac) - nonce = os.urandom(18) - req = Request(msgtype.value, data) - res = msg(make_packet(mac, key, nonce, req).pack()) - p = parse_packet(res) - r = parse_response(p, key) - return r - -def prettyprint_reply(r): - try: - t = MsgType(r.msg_type) - s = ResponseStatus(r.status) - except ValueError: - raise BadMessageError('Unknown type or status {}'.format(r.status)) - return '{} {}: {}'.format(t.name, s.name, r.data or '(no data)') - -if __name__ == '__main__': - mac, msgtype = sys.argv[1:] - try: - t = MsgType[msgtype.upper()] - except KeyError: - sys.exit('No such message type: '+msgtype) - - indata = sys.stdin.buffer.read() - - db.connect(config.db_url) - reply = request(mac2bytes(mac), t, indata) - print(prettyprint_reply(reply)) From 292ea114e6a72bedca63232fb39b5b2189979de9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamila=20Sou=C4=8Dkov=C3=A1?= Date: Sun, 3 Apr 2016 13:32:13 +0200 Subject: [PATCH 36/46] add {pytest,coverage}-related stuff --- .coveragerc | 13 +++++++++++++ .gitignore | 3 +-- README.md | 7 +++++++ pytest.ini | 3 +++ 4 files changed, 24 insertions(+), 2 deletions(-) create mode 100644 .coveragerc create mode 100644 pytest.ini diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000..95f8a12 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,13 @@ +[report] + +source = + structparse +# deadserver + +exclude_lines = + pragma: no cover + def __repr__ + raise NotImplementedError + if 0: + +show_missing = True diff --git a/.gitignore b/.gitignore index fa2e944..600d8ce 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,4 @@ *.py[co] venv/ config.py -.* -!.git* +.coverage diff --git a/README.md b/README.md index dce13fb..9a413a0 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,5 @@ +**Note: This README is out of date. TODO.** + server: the DB + "manager" ========================== @@ -37,3 +39,8 @@ Next to do: - HTTP: rewrite to use Werkzeug instead of CherryPy - fix DB singleton (who wants a singleton?!) - CI + +Style Guide & such +------------------ + +[PEP-8](https://www.python.org/dev/peps/pep-0008/), `import this`. Also: code and design reviews. diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..d3a5838 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,3 @@ +[pytest] +norecursedirs = venv +testpaths = . From f426b3adb003545a6bf05c00925c9af315fd1cf9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamila=20Sou=C4=8Dkov=C3=A1?= Date: Sun, 3 Apr 2016 13:42:04 +0200 Subject: [PATCH 37/46] add unfrozen requirements.txt --- requirements-fresh.txt | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 requirements-fresh.txt diff --git a/requirements-fresh.txt b/requirements-fresh.txt new file mode 100644 index 0000000..08eab45 --- /dev/null +++ b/requirements-fresh.txt @@ -0,0 +1,4 @@ +psycopg2 +pytest +pytest-cov +https://github.com/warner/python-tweetnacl/tarball/b48a25a33f From 38e5a2447c3ecf9893574e6e6e85dad59c5c0cdd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamila=20Sou=C4=8Dkov=C3=A1?= Date: Sun, 3 Apr 2016 14:56:10 +0200 Subject: [PATCH 38/46] renames: s/controller_// --- controller_client.py | 4 ++-- deadserver/{controller_api.py => api.py} | 4 ++-- deadserver/{controller_protocol.py => protocol.py} | 0 deadserver/{controller_server.py => server.py} | 5 +++-- deadserver/{utils/__init__.py => utils.py} | 0 runhttp.py | 10 ---------- runserver.py | 4 ++-- 7 files changed, 9 insertions(+), 18 deletions(-) rename deadserver/{controller_api.py => api.py} (94%) rename deadserver/{controller_protocol.py => protocol.py} (100%) rename deadserver/{controller_server.py => server.py} (91%) rename deadserver/{utils/__init__.py => utils.py} (100%) delete mode 100755 runhttp.py diff --git a/controller_client.py b/controller_client.py index 9dec868..39b1c8a 100644 --- a/controller_client.py +++ b/controller_client.py @@ -2,8 +2,8 @@ import config from deadserver import db -from deadserver.controller_api import * -from deadserver.controller_protocol import * +from deadserver.api import * +from deadserver.protocol import * import socket import os import sys diff --git a/deadserver/controller_api.py b/deadserver/api.py similarity index 94% rename from deadserver/controller_api.py rename to deadserver/api.py index 45214de..e051f23 100644 --- a/deadserver/controller_api.py +++ b/deadserver/api.py @@ -4,8 +4,8 @@ the message format details. """ -from . import controller_protocol as protocol -from .controller_protocol import MsgType, ResponseStatus as Status +from . import protocol +from .protocol import MsgType, ResponseStatus as Status from structparse.types import Tail # TODO remove diff --git a/deadserver/controller_protocol.py b/deadserver/protocol.py similarity index 100% rename from deadserver/controller_protocol.py rename to deadserver/protocol.py diff --git a/deadserver/controller_server.py b/deadserver/server.py similarity index 91% rename from deadserver/controller_server.py rename to deadserver/server.py index 9da0a60..40cd08f 100644 --- a/deadserver/controller_server.py +++ b/deadserver/server.py @@ -4,7 +4,8 @@ requests and send responses, and passes stuff to `controller_api`. """ -from . import controller_api, db +from .api import API +from . import db import functools import socketserver @@ -23,7 +24,7 @@ class DeadServer: def __init__(self, config): self.config = config self.db_conn = db.Connection(config.db_url) - self.api = controller_api.API(db_conn=self.db_conn) + self.api = API(db_conn=self.db_conn) bind_addr = self.config.udp_host, self.config.udp_port handler = functools.partial(MessageHandler, self.api) diff --git a/deadserver/utils/__init__.py b/deadserver/utils.py similarity index 100% rename from deadserver/utils/__init__.py rename to deadserver/utils.py diff --git a/runhttp.py b/runhttp.py deleted file mode 100755 index 0f97084..0000000 --- a/runhttp.py +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env python3 -"""Gate HTTP API server runner.""" - -#from gateserver import db, http_api -#import config - -#if __name__ == '__main__': - #db.connect(config.db_url) - #http_api.serve(config) -print('Not implemented yet.') diff --git a/runserver.py b/runserver.py index f657971..6b20f7b 100755 --- a/runserver.py +++ b/runserver.py @@ -1,9 +1,9 @@ #!/usr/bin/env python3 """Deadlock server runner.""" -from deadserver import controller_server +from deadserver import server import config if __name__ == '__main__': - app = controller_server.DeadServer(config) + app = server.DeadServer(config) app.serve() From 993366d5d23b93c970e065de4e54ebc0ec3469c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamila=20Sou=C4=8Dkov=C3=A1?= Date: Sun, 3 Apr 2016 17:32:20 +0200 Subject: [PATCH 39/46] clean up structparse and a few imports --- deadserver/protocol.py | 36 +++++++-------- deadserver/server.py | 8 ++-- structparse/__init__.py | 62 +------------------------- structparse/structdef.py | 63 +++++++++++++++++++++++++++ structparse/tests/test_struct.py | 54 +++++++++++++++++++++++ structparse/tests/test_structparse.py | 41 ----------------- structparse/tests/test_types.py | 6 +-- structparse/types.py | 10 ++++- 8 files changed, 151 insertions(+), 129 deletions(-) create mode 100644 structparse/structdef.py create mode 100644 structparse/tests/test_struct.py delete mode 100644 structparse/tests/test_structparse.py diff --git a/deadserver/protocol.py b/deadserver/protocol.py index c8c7522..b27afd0 100644 --- a/deadserver/protocol.py +++ b/deadserver/protocol.py @@ -7,9 +7,9 @@ # TODO: consider [CBOR](http://cbor.io/). # TODO: With the faster processor, we probably can afford assymetric crypto. Switch if possible. -from structparse import mystruct, types +from structparse import struct, types import nacl.raw as nacl -from enum import Enum +import enum class BadMessageError(Exception): pass @@ -18,31 +18,31 @@ def check(expression, errmsg): PROTOCOL_VERSION = types.Bytes(2)([0,1]) -class MsgType(types.Uint8, Enum): +class MsgType(types.Uint8, enum.Enum): OPEN = 1 -class ResponseStatus(types.Uint8, Enum): +class ResponseStatus(types.Uint8, enum.Enum): OK = 0x01 ERR = 0x10 TRY_AGAIN = 0x11 -Request = mystruct('Request', - (MsgType, 'msg_type'), - (types.Tail, 'data' )) +Request = struct('Request', + (MsgType, 'msg_type'), + (types.Tail, 'data' )) -Response = mystruct('Response', - (MsgType, 'msg_type'), - (ResponseStatus, 'status' ), - (types.Tail, 'data' )) +Response = struct('Response', + (MsgType, 'msg_type'), + (ResponseStatus, 'status' ), + (types.Tail, 'data' )) -PacketHeader = mystruct('PacketHeader', - (types.Bytes(2), 'protocol_version'), - (types.Bytes(6), 'controller_id' ), - (types.Bytes(18), 'nonce' )) +PacketHeader = struct('PacketHeader', + (types.Bytes(2), 'protocol_version'), + (types.Bytes(6), 'controller_id' ), + (types.Bytes(18), 'nonce' )) -Packet = mystruct('Packet', - (PacketHeader, 'header'), - (types.Tail, 'payload')) +Packet = struct('Packet', + (PacketHeader, 'header'), + (types.Tail, 'payload')) def id2str(id): return ':'.join('{:02x}'.format(x) for x in id.val) diff --git a/deadserver/server.py b/deadserver/server.py index 40cd08f..5f4cf76 100644 --- a/deadserver/server.py +++ b/deadserver/server.py @@ -4,8 +4,9 @@ requests and send responses, and passes stuff to `controller_api`. """ -from .api import API +from . import api from . import db + import functools import socketserver @@ -24,11 +25,10 @@ class DeadServer: def __init__(self, config): self.config = config self.db_conn = db.Connection(config.db_url) - self.api = API(db_conn=self.db_conn) + self.handler = functools.partial(MessageHandler, api.API(db_conn=self.db_conn)) bind_addr = self.config.udp_host, self.config.udp_port - handler = functools.partial(MessageHandler, self.api) - self.server = socketserver.ThreadingUDPServer(bind_addr, handler) + self.server = socketserver.ThreadingUDPServer(bind_addr, self.handler) def serve(self): self.server.serve_forever() diff --git a/structparse/__init__.py b/structparse/__init__.py index 304ab32..c53097d 100644 --- a/structparse/__init__.py +++ b/structparse/__init__.py @@ -1,63 +1,3 @@ """Simplifies parsing "C structs" -- structured types serialized as byte arrays.""" -from collections import namedtuple - -def unzip(x): return zip(*x) - -class Type: - """Defines a type that can be serialized and unserialized.""" - - @staticmethod - def unpack(buf): - """Constructs a Python value from the given buffer. - - Signature: buffer -> (parsed data, rest of the buffer) - """ - raise NotImplementedError - - def pack(self): - """Packs itself into `bytes`, returns that buffer. - - Signature: data -> buffer - """ - raise NotImplementedError - -class MyStructMixin(Type): - """Mixin providing the `pack` and `unpack` methods for a struct.""" - def __new__(cls, *args, **kwargs): - if len(args) == 1 and len(kwargs) == 0 and args[0].__class__ is cls: - return args[0] # assumes immutability - if len(args) + len(kwargs) != len(cls._fields): - raise TypeError('Must be initialized with exactly {} arguments'.format(len(cls._fields))) - field_types = dict(zip(cls._fields, cls._fieldtypes)) - to_convert = dict(zip(cls._fields, args), **kwargs) - converted = { n: field_types[n](v) for (n, v) in to_convert.items() } - return super().__new__(cls, **converted) - - @classmethod - def unpack(cls, buf): - """Constructs a new instance by unpacking the given buffer.""" - vals = [] - for t in cls._fieldtypes: - val, buf = t.unpack(buf) - vals.append(val) - return cls(*vals), buf - - def pack(self): - """Returns itself packed as `bytes`.""" - return b''.join([ t.pack(x) for t,x in zip(self._fieldtypes, self) ]) - - @classmethod - def unpack_all(cls, buf): - """raises ValueError if len(buf) doesn't exactly match the struct size.""" - x, rest = cls.unpack(buf) - if len(rest) > 0: raise ValueError('buffer size != struct size') - return x - -def mystruct(name, *fields): - """Creates a "C struct" -- a namedtuple that can be packed and unpacked.""" - fieldtypes, fieldnames = unzip(fields) - class Cls(MyStructMixin, namedtuple(name, fieldnames)): pass - Cls.__name__ = name - Cls._fieldtypes = fieldtypes - return Cls +from .structdef import struct # make this easily accessible diff --git a/structparse/structdef.py b/structparse/structdef.py new file mode 100644 index 0000000..4bbd427 --- /dev/null +++ b/structparse/structdef.py @@ -0,0 +1,63 @@ +"""Allows defining serializable simple types and structs.""" + +from collections import namedtuple + +def unzip(x): return zip(*x) + +class Type: + """Defines a type that can be serialized and unserialized.""" + + @staticmethod + def unpack(buf): + """Constructs a Python value from the given buffer. + + Signature: buffer -> (parsed data, rest of the buffer) + """ + raise NotImplementedError + + def pack(self): + """Packs itself into `bytes`, returns that buffer. + + Signature: data -> buffer + """ + raise NotImplementedError + +class StructMixin(Type): + """Mixin providing the `pack` and `unpack` methods for a struct.""" + def __new__(cls, *args, **kwargs): + if len(args) == 1 and len(kwargs) == 0 and args[0].__class__ is cls: + return args[0] # assumes immutability + if len(args) + len(kwargs) != len(cls._fields): + raise TypeError('Must be initialized with exactly {} arguments'.format(len(cls._fields))) + field_types = dict(zip(cls._fields, cls._fieldtypes)) + to_convert = dict(zip(cls._fields, args), **kwargs) + converted = { n: field_types[n](v) for (n, v) in to_convert.items() } + return super().__new__(cls, **converted) + + @classmethod + def unpack(cls, buf): + """Constructs a new instance by unpacking the given buffer.""" + vals = [] + for t in cls._fieldtypes: + val, buf = t.unpack(buf) + vals.append(val) + return cls(*vals), buf + + def pack(self): + """Returns itself packed as `bytes`.""" + return b''.join([ t.pack(x) for t,x in zip(self._fieldtypes, self) ]) + + @classmethod + def unpack_all(cls, buf): + """raises ValueError if len(buf) doesn't exactly match the struct size.""" + x, rest = cls.unpack(buf) + if len(rest) > 0: raise ValueError('buffer size != struct size') + return x + +def struct(name, *fields): + """Creates a "C struct" -- a namedtuple that can be packed and unpacked.""" + fieldtypes, fieldnames = unzip(fields) + class Cls(StructMixin, namedtuple(name, fieldnames)): pass + Cls.__name__ = name + Cls._fieldtypes = fieldtypes + return Cls diff --git a/structparse/tests/test_struct.py b/structparse/tests/test_struct.py new file mode 100644 index 0000000..773e7fe --- /dev/null +++ b/structparse/tests/test_struct.py @@ -0,0 +1,54 @@ +import pytest + +from .. import struct, types + + +def test_new(): + TestStruct = struct('TestStruct', + (types.Uint8, 'john'), + (types.Uint8, 'paul'), + (types.Uint8, 'george'), + (types.Uint8, 'ringo')) + def check(instance): + assert (instance.john == types.Uint8(12) and + instance.paul == types.Uint8(4) and + instance.george == types.Uint8(6) and + instance.ringo == types.Uint8(0)) + check(TestStruct(12, 4, 6, 0)) + check(TestStruct(paul=4, george=6, john=12, ringo=0)) + check(TestStruct(12, 4, ringo=0, george=6)) + with pytest.raises(TypeError) as err: + TestStruct(1, 2, 47) + assert 'exactly 4' in str(err.value) + with pytest.raises(TypeError) as err: + TestStruct(12, 4, john=42, paul=47) + assert 'arguments' in str(err.value) + + +@pytest.fixture +def Sample(): + return struct('Sample', + (types.Bytes(4), 'x'), + (types.Uint8, 'y'), + (types.PascalStr(5), 'z')) + +@pytest.fixture +def Nested(Sample): + return struct('Nested', + (Sample, 'a'), + (types.Tail, 'b')) + +@pytest.fixture +def strct(Sample, Nested): + return Nested(Sample(b'quux', 0x47, 'foo'), b'kaleraby') + +@pytest.fixture +def packed(): + return b'quux' + b'\x47' + b'\x03foo\x00\x00' + b'kaleraby' + + +def test_pack(strct, packed): + assert strct.pack() == packed + +def test_unpack(Nested, packed, strct): + assert Nested.unpack_all(packed) == strct diff --git a/structparse/tests/test_structparse.py b/structparse/tests/test_structparse.py deleted file mode 100644 index a22b1fb..0000000 --- a/structparse/tests/test_structparse.py +++ /dev/null @@ -1,41 +0,0 @@ -import pytest - -from .. import mystruct, types - -def test_new(): - TestStruct = mystruct('TestStruct', - (types.Uint8, 'john'), - (types.Uint8, 'paul'), - (types.Uint8, 'george'), - (types.Uint8, 'ringo')) - def check(instance): - assert (instance.john == types.Uint8(12) and - instance.paul == types.Uint8(4) and - instance.george == types.Uint8(6) and - instance.ringo == types.Uint8(0)) - check(TestStruct(12, 4, 6, 0)) - check(TestStruct(paul=4, george=6, john=12, ringo=0)) - check(TestStruct(12, 4, ringo=0, george=6)) - with pytest.raises(TypeError) as err: - TestStruct(1, 2, 47) - assert 'exactly 4' in str(err.value) - with pytest.raises(TypeError) as err: - TestStruct(12, 4, john=42, paul=47) - assert 'arguments' in str(err.value) - -SubStruct = mystruct('SubStruct', - (types.Bytes(4), 'x'), - (types.Uint8, 'y'), - (types.PascalStr(5), 'z')) -Struct = mystruct('Struct', - (SubStruct, 'a'), - (types.Tail, 'b')) - -s = Struct(SubStruct(b'quux', 0x47, 'foo'), b'kaleraby') -packed = b'quux' + b'\x47' + b'\x03foo\x00\x00' + b'kaleraby' - -def test_pack(): - assert s.pack() == packed - -def test_unpack(): - assert Struct.unpack_all(packed) == s diff --git a/structparse/tests/test_types.py b/structparse/tests/test_types.py index 5bc351b..b7cb419 100644 --- a/structparse/tests/test_types.py +++ b/structparse/tests/test_types.py @@ -1,7 +1,7 @@ import pytest from .. import types -from enum import Enum +import enum def test_construction(): x = types.Uint8(47) @@ -59,7 +59,7 @@ def test_hashable(): assert hash(types.Bytes(2)([42,47])) != hash(types.Tail([42,47])) def test_Enum_works(): - class T(types.Uint8, Enum): + class T(types.Uint8, enum.Enum): A = 1 B = 2 Z = 255 @@ -71,5 +71,5 @@ class T(types.Uint8, Enum): with pytest.raises(ValueError): T.unpack(b'\x47') with pytest.raises(ValueError): - class T(types.Uint8, Enum): + class T(types.Uint8, enum.Enum): X = 4742 # does not fit into 1 byte diff --git a/structparse/types.py b/structparse/types.py index 9a41516..6715392 100644 --- a/structparse/types.py +++ b/structparse/types.py @@ -1,6 +1,6 @@ -"""Defines default types for structparse.""" +"""Defines useful simple types for structparse.""" -from . import Type +from .structdef import Type class _SimpleType(Type): def __init__(self, x): @@ -37,11 +37,13 @@ def __hash__(self): def __repr__(self): return self.__class__.__name__+'('+repr(self.val)+')' + def _tobytes(x): if isinstance(x, _SimpleType): return bytes(x.val) if isinstance(x, str): return bytes(x, 'utf8') return bytes(x) + class Uint8(_SimpleType): @staticmethod def _unpack(buf): @@ -54,15 +56,18 @@ def _validate(x): def _pack(self): return [self.val] + class _BytesLike(_SimpleType): def __init__(self, arg): super().__init__(_tobytes(arg)) + class Tail(_BytesLike): @staticmethod def _unpack(buf): return buf, b'' + def Bytes(n): class Cls(_BytesLike): @staticmethod @@ -78,6 +83,7 @@ def _unpack(buf): Cls.__name__ = 'Bytes[{}]'.format(n) return Cls + def PascalStr(n): class Cls(_BytesLike): @staticmethod From 69e4a69a9d7e1842a4a8ce2d8a92f7cc05068f8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamila=20Sou=C4=8Dkov=C3=A1?= Date: Sun, 3 Apr 2016 17:37:55 +0200 Subject: [PATCH 40/46] update requirements --- requirements.txt | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/requirements.txt b/requirements.txt index b66f131..36655fe 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,6 @@ -cov-core==1.15.0 -coverage==3.7.1 -psycopg2==2.5.4 -py==1.4.26 -pytest==2.6.4 -pytest-cov==1.8.1 +coverage==4.0.3 +psycopg2==2.6.1 +py==1.4.31 +pytest==2.9.1 +pytest-cov==2.2.1 https://github.com/warner/python-tweetnacl/tarball/b48a25a33f From af6b24c3260f7c2e9d81f9a66fc6f73f606a5b90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamila=20Sou=C4=8Dkov=C3=A1?= Date: Sun, 3 Apr 2016 18:04:03 +0200 Subject: [PATCH 41/46] kill deadserver.db in favor of records --- controller_client.py | 12 +++++++----- deadserver/api.py | 14 +++++++------- deadserver/db.py | 19 ------------------- deadserver/server.py | 11 ++++++----- requirements-fresh.txt | 2 +- requirements.txt | 4 ++++ 6 files changed, 25 insertions(+), 37 deletions(-) delete mode 100644 deadserver/db.py diff --git a/controller_client.py b/controller_client.py index 39b1c8a..3898f55 100644 --- a/controller_client.py +++ b/controller_client.py @@ -1,14 +1,16 @@ """Quick & dirty client (i.e. the controller end), used for manual testing of the server.""" -import config -from deadserver import db -from deadserver.api import * -from deadserver.protocol import * import socket import os import sys -api = API(db_conn=db.Connection(config.db_url)) +import records + +import config +from deadserver.api import * +from deadserver.protocol import * + +api = API(db=records.Database(config.db_url)) def msg(buf): sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) diff --git a/deadserver/api.py b/deadserver/api.py index e051f23..d18a6f3 100644 --- a/deadserver/api.py +++ b/deadserver/api.py @@ -10,8 +10,8 @@ from structparse.types import Tail # TODO remove class API: - def __init__(self, db_conn): - self.db = db_conn + def __init__(self, db): + self.db = db def handle_packet(self, in_buf): try: @@ -21,7 +21,7 @@ def handle_packet(self, in_buf): response_packet = protocol.make_response_packet_for(request_header, request.msg_type, status, response_data, get_key=self.get_key) return response_packet.pack() except protocol.BadMessageError as e: - log_bad_message(buf, e) + self.log_bad_message(in_buf, e) # TODO this table will be dynamic via handler registration -- pretend it doesn't exist process_request = { @@ -31,10 +31,10 @@ def handle_packet(self, in_buf): def get_key(self, id): """Loads the key for this controller from the DB.""" - rs = self.db.exec_sql('SELECT key FROM controller WHERE id = %s', - (protocol.id2str(id),), ret=True) - protocol.check(len(rs) == 1, 'unknown controller ID') - return bytes(rs[0]['key']) + rows = self.db.query('SELECT key FROM controller WHERE id = :id', + id=protocol.id2str(id)).all() + protocol.check(len(rows) == 1, 'unknown controller ID') + return bytes(rows[0]['key']) def log_message(self, controller_id, request, status): """TODO""" diff --git a/deadserver/db.py b/deadserver/db.py deleted file mode 100644 index 1698358..0000000 --- a/deadserver/db.py +++ /dev/null @@ -1,19 +0,0 @@ -"""Holds the (global) connection to the DB.""" -# TODO maybe use a connection pool one beautiful day - -import psycopg2 -from psycopg2.extras import RealDictCursor # results as dict instead of tuple - -class Connection: - def __init__(self, db_url): - self.conn = psycopg2.connect(db_url, cursor_factory=RealDictCursor) - self.conn.autocommit = True - - def exec_sql(self, query, args=(), ret=False): - """Executes the query, returning the result as a list if `ret`.""" - with self.conn.cursor() as cur: - cur.execute(query, args) - if ret: return cur.fetchall() - -# thrown when constraints aren't satisfied -IntegrityError = psycopg2.IntegrityError diff --git a/deadserver/server.py b/deadserver/server.py index 5f4cf76..6cb2adb 100644 --- a/deadserver/server.py +++ b/deadserver/server.py @@ -4,12 +4,13 @@ requests and send responses, and passes stuff to `controller_api`. """ -from . import api -from . import db - import functools import socketserver +import records + +from . import api + class MessageHandler(socketserver.BaseRequestHandler): def __init__(self, api, *args, **kwargs): self.api = api @@ -24,8 +25,8 @@ def handle(self): class DeadServer: def __init__(self, config): self.config = config - self.db_conn = db.Connection(config.db_url) - self.handler = functools.partial(MessageHandler, api.API(db_conn=self.db_conn)) + self.db = records.Database(config.db_url) + self.handler = functools.partial(MessageHandler, api.API(db=self.db)) bind_addr = self.config.udp_host, self.config.udp_port self.server = socketserver.ThreadingUDPServer(bind_addr, self.handler) diff --git a/requirements-fresh.txt b/requirements-fresh.txt index 08eab45..fc9a35f 100644 --- a/requirements-fresh.txt +++ b/requirements-fresh.txt @@ -1,4 +1,4 @@ -psycopg2 pytest pytest-cov +records https://github.com/warner/python-tweetnacl/tarball/b48a25a33f diff --git a/requirements.txt b/requirements.txt index 36655fe..f26fde3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,10 @@ +SQLAlchemy==1.0.12 coverage==4.0.3 +docopt==0.6.2 psycopg2==2.6.1 py==1.4.31 pytest==2.9.1 pytest-cov==2.2.1 +records==0.4.3 +tablib==0.11.2 https://github.com/warner/python-tweetnacl/tarball/b48a25a33f From b4420f55a258b00ce8bfdff5781469066581b8fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamila=20Sou=C4=8Dkov=C3=A1?= Date: Sun, 3 Apr 2016 18:59:08 +0200 Subject: [PATCH 42/46] kill that tweetnacl wrapper in favor of pynacl --- deadserver/api.py | 2 ++ deadserver/protocol.py | 13 ++++++++++--- requirements-fresh.txt | 2 +- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/deadserver/api.py b/deadserver/api.py index d18a6f3..764396f 100644 --- a/deadserver/api.py +++ b/deadserver/api.py @@ -29,6 +29,8 @@ def handle_packet(self, in_buf): MsgType.OPEN: (lambda id, data: ((Status.OK if data == Tail(b'Hello') else Status.ERR), None)) } + # TODO if protocol crypto and insides were better separated, this could just create a + # {de,en}cryption black box and thereby avoid telling the key to anyone else. def get_key(self, id): """Loads the key for this controller from the DB.""" rows = self.db.query('SELECT key FROM controller WHERE id = :id', diff --git a/deadserver/protocol.py b/deadserver/protocol.py index b27afd0..2a080a6 100644 --- a/deadserver/protocol.py +++ b/deadserver/protocol.py @@ -6,16 +6,20 @@ # TODO: consider [CBOR](http://cbor.io/). # TODO: With the faster processor, we probably can afford assymetric crypto. Switch if possible. +# TODO: separate the 2 layers of the protocol +# TODO: ... and then blackboxes instead of secret keys from structparse import struct, types -import nacl.raw as nacl +import nacl.secret import enum + class BadMessageError(Exception): pass def check(expression, errmsg): if not expression: raise BadMessageError(errmsg) + PROTOCOL_VERSION = types.Bytes(2)([0,1]) class MsgType(types.Uint8, enum.Enum): @@ -44,17 +48,20 @@ class ResponseStatus(types.Uint8, enum.Enum): (PacketHeader, 'header'), (types.Tail, 'payload')) + def id2str(id): return ':'.join('{:02x}'.format(x) for x in id.val) def str2id(s): return types.Bytes(6)(bytes.fromhex(s.replace(':', ''))) + def crypto_unwrap_payload(nonce, payload, key): - return nacl.crypto_secretbox_open(payload, nonce, key) + return nacl.secret.SecretBox(key).decrypt(payload, nonce) def crypto_wrap_payload(nonce, payload, key): - return nacl.crypto_secretbox(payload, nonce, key) + # Note: encrypt returns the ciphertext prepended by nonce. We don't want this, so strip it. + return nacl.secret.SecretBox(key).encrypt(payload, nonce)[nacl.secret.SecretBox.NONCE_SIZE:] def parse_packet(struct, buf, get_key): """Parses the buffer into PacketHeader and `struct`, which must be Request or Response. diff --git a/requirements-fresh.txt b/requirements-fresh.txt index fc9a35f..92479cb 100644 --- a/requirements-fresh.txt +++ b/requirements-fresh.txt @@ -1,4 +1,4 @@ +pynacl pytest pytest-cov records -https://github.com/warner/python-tweetnacl/tarball/b48a25a33f From 764d6cd3386d5568b9659ce23b11741381e596ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamila=20Sou=C4=8Dkov=C3=A1?= Date: Sun, 3 Apr 2016 19:01:14 +0200 Subject: [PATCH 43/46] add notes about fun problems --- fun_stuff.txt | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 fun_stuff.txt diff --git a/fun_stuff.txt b/fun_stuff.txt new file mode 100644 index 0000000..7aa0c85 --- /dev/null +++ b/fun_stuff.txt @@ -0,0 +1,5 @@ +Fun problems that I've overcome +=============================== + +- stateless is awesome +- avoid running around with secret keys by passing just a crypto black box (TODO but do it :D), and actually storing / loading the keys encrypted and decrypting only when really needed (TODO also do it :D) From 0db210a5cb7a4e28ade8614a9a26100afc81d6b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamila=20Sou=C4=8Dkov=C3=A1?= Date: Sun, 3 Apr 2016 22:50:51 +0200 Subject: [PATCH 44/46] structparse fixes --- deadserver/protocol.py | 4 ++-- structparse/structdef.py | 29 ++++++++++++---------- structparse/tests/test_struct.py | 4 ++-- structparse/tests/test_types.py | 29 +++++++++++----------- structparse/types.py | 41 ++++++++++++++++++++++---------- 5 files changed, 65 insertions(+), 42 deletions(-) diff --git a/deadserver/protocol.py b/deadserver/protocol.py index 2a080a6..ca7ce1a 100644 --- a/deadserver/protocol.py +++ b/deadserver/protocol.py @@ -71,11 +71,11 @@ def parse_packet(struct, buf, get_key): assert struct in [Request, Response] try: - hdr, tail = PacketHeader.unpack(buf) + hdr, tail = PacketHeader.unpack_from(buf) check(hdr.protocol_version == PROTOCOL_VERSION, 'Invalid protocol version') payload_buf = crypto_unwrap_payload(hdr.controller_id.val + hdr.nonce.val, tail, get_key(hdr.controller_id)) - payload = struct.unpack_all(payload_buf) + payload = struct.unpack(payload_buf) except ValueError as e: raise BadMessageError('parse_packet failed') from e diff --git a/structparse/structdef.py b/structparse/structdef.py index 4bbd427..19cc564 100644 --- a/structparse/structdef.py +++ b/structparse/structdef.py @@ -8,7 +8,7 @@ class Type: """Defines a type that can be serialized and unserialized.""" @staticmethod - def unpack(buf): + def unpack_from(buf): """Constructs a Python value from the given buffer. Signature: buffer -> (parsed data, rest of the buffer) @@ -22,8 +22,19 @@ def pack(self): """ raise NotImplementedError -class StructMixin(Type): - """Mixin providing the `pack` and `unpack` methods for a struct.""" + @classmethod + def unpack(cls, buf): + """Unpacks the whole buffer into this Type. + + Raises ValueError if len(buf) doesn't exactly match the type size. + """ + x, rest = cls.unpack_from(buf) + if len(rest) > 0: raise ValueError('buffer size != struct size') + return x + + +class _StructMixin(Type): + """Mixin providing the `pack`, `unpack` and `unpack_from` methods for a struct.""" def __new__(cls, *args, **kwargs): if len(args) == 1 and len(kwargs) == 0 and args[0].__class__ is cls: return args[0] # assumes immutability @@ -35,11 +46,11 @@ def __new__(cls, *args, **kwargs): return super().__new__(cls, **converted) @classmethod - def unpack(cls, buf): + def unpack_from(cls, buf): """Constructs a new instance by unpacking the given buffer.""" vals = [] for t in cls._fieldtypes: - val, buf = t.unpack(buf) + val, buf = t.unpack_from(buf) vals.append(val) return cls(*vals), buf @@ -47,17 +58,11 @@ def pack(self): """Returns itself packed as `bytes`.""" return b''.join([ t.pack(x) for t,x in zip(self._fieldtypes, self) ]) - @classmethod - def unpack_all(cls, buf): - """raises ValueError if len(buf) doesn't exactly match the struct size.""" - x, rest = cls.unpack(buf) - if len(rest) > 0: raise ValueError('buffer size != struct size') - return x def struct(name, *fields): """Creates a "C struct" -- a namedtuple that can be packed and unpacked.""" fieldtypes, fieldnames = unzip(fields) - class Cls(StructMixin, namedtuple(name, fieldnames)): pass + class Cls(_StructMixin, namedtuple(name, fieldnames)): pass Cls.__name__ = name Cls._fieldtypes = fieldtypes return Cls diff --git a/structparse/tests/test_struct.py b/structparse/tests/test_struct.py index 773e7fe..e4ca7f6 100644 --- a/structparse/tests/test_struct.py +++ b/structparse/tests/test_struct.py @@ -44,11 +44,11 @@ def strct(Sample, Nested): @pytest.fixture def packed(): - return b'quux' + b'\x47' + b'\x03foo\x00\x00' + b'kaleraby' + return b'quux' + b'\x47' + b'\x03foo\x00' + b'kaleraby' def test_pack(strct, packed): assert strct.pack() == packed def test_unpack(Nested, packed, strct): - assert Nested.unpack_all(packed) == strct + assert Nested.unpack(packed) == strct diff --git a/structparse/tests/test_types.py b/structparse/tests/test_types.py index b7cb419..c534a9a 100644 --- a/structparse/tests/test_types.py +++ b/structparse/tests/test_types.py @@ -19,13 +19,13 @@ def test_eq(): def test_Uint8(): assert types.Uint8(97).pack() == b'a' - assert types.Uint8.unpack(b'abcd') == (types.Uint8(97), b'bcd') + assert types.Uint8.unpack_from(b'abcd') == (types.Uint8(97), b'bcd') with pytest.raises(ValueError): types.Uint8(4742) def test_Tail(): assert types.Tail(b'an arbitrarily long whatever').pack() == b'an arbitrarily long whatever' assert types.Tail([97, 98, 99, 100]) == types.Tail(b'abcd') - assert types.Tail.unpack(b'mrkva') == (types.Tail(b'mrkva'), b'') + assert types.Tail.unpack(b'mrkva') == types.Tail(b'mrkva') def test_Bytes(): b4 = types.Bytes(4) @@ -37,21 +37,22 @@ def test_Bytes(): with pytest.raises(ValueError): b4(b'abcde') assert b4(b'abcd').pack() == b'abcd' - assert b4.unpack(b'abcdefg') == (b4(b'abcd'), b'efg') - with pytest.raises(ValueError): b4.unpack(b'abc') + assert b4.unpack_from(b'abcdefg') == (b4(b'abcd'), b'efg') + with pytest.raises(ValueError): b4.unpack_from(b'abc') def test_PascalStr(): - p5 = types.PascalStr(5) - assert repr(p5(b'abcd')) == "PascalStr[5](b'abcd')" + p6 = types.PascalStr(6) + assert repr(p6(b'abcd')) == "PascalStr[6](b'abcd')" - with pytest.raises(ValueError): p5('Hello World') + p6('Hello') # n-1 bytes fit + with pytest.raises(ValueError): p6('Hello!') # n and more don't - assert p5('hello').pack() == b'\x05hello' - assert p5('hell').pack() == b'\x04hell\x00' + assert p6('hello').pack() == b'\x05hello' + assert p6('hell').pack() == b'\x04hell\x00' - assert p5.unpack(b'\x03hel\x00\x00 world') == (p5(b'hel'), b' world') - with pytest.raises(ValueError): p5.unpack(b'\x03hello world') - with pytest.raises(ValueError): p5.unpack(b'\x47anything') + assert p6.unpack_from(b'\x03hel\x00\x00 world') == (p6(b'hel'), b' world') + with pytest.raises(ValueError): p6.unpack_from(b'\x03hello world') + with pytest.raises(ValueError): p6.unpack_from(b'\x47anything') def test_hashable(): assert hash(types.Uint8(47)) == hash(types.Uint8(47)) @@ -66,9 +67,9 @@ class T(types.Uint8, enum.Enum): assert T(1) == T.A assert T.A.pack() == b'\x01' - assert T.unpack(b'\xff') == (T.Z, b'') + assert T.unpack(b'\xff') == T.Z with pytest.raises(ValueError): T(47) - with pytest.raises(ValueError): T.unpack(b'\x47') + with pytest.raises(ValueError): T.unpack_from(b'\x47') with pytest.raises(ValueError): class T(types.Uint8, enum.Enum): diff --git a/structparse/types.py b/structparse/types.py index 6715392..4bb546f 100644 --- a/structparse/types.py +++ b/structparse/types.py @@ -17,8 +17,8 @@ def pack(self): return bytes(self._pack()) @classmethod - def unpack(cls, buf): - val, rest = cls._unpack(buf) + def unpack_from(cls, buf): + val, rest = cls._unpack_from(buf) return cls(val), rest def _validate(self, input): @@ -46,7 +46,7 @@ def _tobytes(x): class Uint8(_SimpleType): @staticmethod - def _unpack(buf): + def _unpack_from(buf): return int(buf[0]), buf[1:] @staticmethod @@ -64,7 +64,7 @@ def __init__(self, arg): class Tail(_BytesLike): @staticmethod - def _unpack(buf): + def _unpack_from(buf): return buf, b'' @@ -77,7 +77,7 @@ def _validate(arg): return bytes(arg) @staticmethod - def _unpack(buf): + def _unpack_from(buf): return buf[:n], buf[n:] Cls.__name__ = 'Bytes[{}]'.format(n) @@ -85,22 +85,39 @@ def _unpack(buf): def PascalStr(n): + """A fixed-length "Pascal string" -- byte 0 is length, the rest is null-padded string content. + + Note that n is the number of bytes in the resulting binary representation, so at most n-1 bytes + fit inside. + """ + assert n >= 1, 'Cannot create PascalStr that can fit -1 bytes' + assert n-1 <= 0xff, 'Max length of PascalStr must fit into 1 byte' class Cls(_BytesLike): @staticmethod def _validate(arg): - if len(arg) > n: + if len(arg) > n-1: raise ValueError('string too long (n = {})'.format(n)) @staticmethod - def _unpack(buf): - if buf[0] > n: raise ValueError('packed string too long (n = {}, buf[0] = {})'.format(n, buf[0])) - b, e = buf[0]+1, n+1 - if buf[b:e] != b'\0'*(e-b): raise ValueError('packed string not null-padded') - return buf[1:b], buf[e:] + def _unpack_from(buf): + buf, tail = buf[:n], buf[n:] + s, data = buf[0], buf[1:] + if s > n-1: raise ValueError('packed string too long (n-1 = {}, s = {})'.format(n-1, s)) + result, padding = data[:s], data[s:] + if not all([b == 0 for b in padding]): raise ValueError('packed string not null-padded') + return result, tail def _pack(self): - padding = n - len(self.val) + padding = n-1 - len(self.val) return bytes([len(self.val)]) + self.val + b'\0'*padding Cls.__name__ = 'PascalStr[{}]'.format(n) return Cls + + +# Note: if you are looking for Enum, this Just Works with Python's enum.Enum: +# +# class T(types.Uint8, enum.Enum): +# A = 1 +# B = 2 +# Z = 255 From 42346dd626f77849f13d1299dd92932e6c7de606 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamila=20Sou=C4=8Dkov=C3=A1?= Date: Mon, 4 Apr 2016 11:09:59 +0200 Subject: [PATCH 45/46] add infrastructure for pluggable packet handlers --- deadserver/__init__.py | 1 + deadserver/api.py | 20 ++++++------------- deadserver/handlers/__init__.py | 35 +++++++++++++++++++++++++++++++++ deadserver/handlers/defs.py | 13 ++++++++++++ deadserver/handlers/echotest.py | 9 +++++++++ deadserver/handlers/open.py | 17 ++++++++++++++++ deadserver/handlers/utils.py | 26 ++++++++++++++++++++++++ deadserver/protocol.py | 5 +++-- deadserver/server.py | 4 ++-- fun_stuff.txt | 4 +++- 10 files changed, 115 insertions(+), 19 deletions(-) create mode 100644 deadserver/handlers/__init__.py create mode 100644 deadserver/handlers/defs.py create mode 100644 deadserver/handlers/echotest.py create mode 100644 deadserver/handlers/open.py create mode 100644 deadserver/handlers/utils.py diff --git a/deadserver/__init__.py b/deadserver/__init__.py index e69de29..8ca01a4 100644 --- a/deadserver/__init__.py +++ b/deadserver/__init__.py @@ -0,0 +1 @@ +"""The Deadlock server -- communicates with controllers.""" diff --git a/deadserver/api.py b/deadserver/api.py index 764396f..265bc81 100644 --- a/deadserver/api.py +++ b/deadserver/api.py @@ -4,10 +4,8 @@ the message format details. """ +from . import handlers from . import protocol -from .protocol import MsgType, ResponseStatus as Status - -from structparse.types import Tail # TODO remove class API: def __init__(self, db): @@ -15,20 +13,16 @@ def __init__(self, db): def handle_packet(self, in_buf): try: - request_header, request = protocol.parse_packet(protocol.Request, in_buf, get_key=self.get_key) - status, response_data = self.process_request[request.msg_type](request_header.controller_id, request.data) # TODO this will be rewritten + request_header, request = protocol.parse_packet(protocol.Request, in_buf, self.get_key) + handler = handlers.get_handler_for(request.msg_type) + status, response = handler(request_header.controller_id, request.data.val, api=self) self.log_message(request_header.controller_id, request, status) - response_packet = protocol.make_response_packet_for(request_header, request.msg_type, status, response_data, get_key=self.get_key) + response_packet = protocol.make_response_packet_for(request_header, request.msg_type, + status, response, get_key=self.get_key) return response_packet.pack() except protocol.BadMessageError as e: self.log_bad_message(in_buf, e) - # TODO this table will be dynamic via handler registration -- pretend it doesn't exist - process_request = { - # TODO this should get data.isic_id, except that's not implemented yet and structparse needs unions and stuff - MsgType.OPEN: (lambda id, data: ((Status.OK if data == Tail(b'Hello') else Status.ERR), None)) - } - # TODO if protocol crypto and insides were better separated, this could just create a # {de,en}cryption black box and thereby avoid telling the key to anyone else. def get_key(self, id): @@ -46,5 +40,3 @@ def log_message(self, controller_id, request, status): def log_bad_message(self, buf, e): """TODO""" raise e - -assert set(protocol.MsgType) == set(API.process_request), 'Not all message types handled' diff --git a/deadserver/handlers/__init__.py b/deadserver/handlers/__init__.py new file mode 100644 index 0000000..49aa513 --- /dev/null +++ b/deadserver/handlers/__init__.py @@ -0,0 +1,35 @@ +"""Collects request handlers for the various message types. + +How to write a request handler: + +Your module must define `function(controller_id, request_data) -> status, response_data`. This must +be registered as a handler for a request type using the `@handles(deadserver.protocol.MsgType)` +decorator. See below for note on importing. + +Hello world example: + +```python +from deadserver.protocol import MsgType, ResponseStatus + +@handles(deadserver.protocol.MsgType.HELLO) # actually, this doesn't exist, but if it did... +def handle_hello(controller_id, data): + return ResponseStatus.OK, b'Hello ' + controller_id + b'! You sent: ' + data +``` + +See `./open.py` for a real-world example. + +---------------------------------------------------------------------------------------------------- + +In order to be executed (and therefore registered), your handler module must be imported somewhere. +This file is a good place for that, as it is imported by `deadserver.api`. Unless you have a reason +to do this differently, add your handlers below. +""" + +### LIST OF ALL STANDARD HANDLER IMPORTS ########################################################### + +from . import echotest +from . import open + +#################################################################################################### + +from .defs import get_handler_for # for more convenient access diff --git a/deadserver/handlers/defs.py b/deadserver/handlers/defs.py new file mode 100644 index 0000000..4360224 --- /dev/null +++ b/deadserver/handlers/defs.py @@ -0,0 +1,13 @@ +"""Provides functions for defining request handlers, such as the `handles(msg_type)` decorator.""" + +_all_handlers = {} + +def handles(msg_type): + def decorator(fn): + _all_handlers[msg_type] = fn + fn.handles = msg_type + return fn + return decorator + +def get_handler_for(msg_type): + return _all_handlers[msg_type] diff --git a/deadserver/handlers/echotest.py b/deadserver/handlers/echotest.py new file mode 100644 index 0000000..3555229 --- /dev/null +++ b/deadserver/handlers/echotest.py @@ -0,0 +1,9 @@ +"""Handler for ECHOTEST requests.""" + +from ..protocol import MsgType, ResponseStatus + +from .defs import handles + +@handles(MsgType.ECHOTEST) +def handle_hello(controller_id, data, api): + return ResponseStatus.OK, data diff --git a/deadserver/handlers/open.py b/deadserver/handlers/open.py new file mode 100644 index 0000000..a2f6a86 --- /dev/null +++ b/deadserver/handlers/open.py @@ -0,0 +1,17 @@ +"""Handler for OPEN requests.""" + +from .defs import handles +from . import utils + +from structparse import struct, types +from deadserver.protocol import MsgType, ResponseStatus + +ISICid = types.PascalStr(12) + +OpenRequest = struct('OpenRequest', (ISICid, 'isic_id')) + +@handles(MsgType.OPEN) +@utils.unpack_indata_as(OpenRequest) +def handle(controller_id, data, api): + status = ResponseStatus.OK if data.isic_id == ISICid('hello') else ResponseStatus.ERR + return status, None diff --git a/deadserver/handlers/utils.py b/deadserver/handlers/utils.py new file mode 100644 index 0000000..0cf024f --- /dev/null +++ b/deadserver/handlers/utils.py @@ -0,0 +1,26 @@ +"""Provides helpers for conveniently defining request handlers.""" + +import functools + +from .. import protocol + +def unpack_indata_as(struct): + """Decorator to first unpack the request data buffer into the given struct.""" + def decorator(fn): + @functools.wraps(fn) + def decorated(controller_id, indata, api): + try: + unpacked = struct.unpack(indata) + except ValueError as e: + raise protocol.BadMessageError('parsing data failed') from e + return fn(controller_id, unpacked, api) + return decorated + return decorator + +def pack_outdata(fn): + """Decorator to pack the structured response data into bytes.""" + @functools.wraps(fn) + def decorated(controller_id, indata, api): + status, outdata = fn(controller_id, indata, api) + return status, outdata.pack() + return decorated diff --git a/deadserver/protocol.py b/deadserver/protocol.py index ca7ce1a..3ded85e 100644 --- a/deadserver/protocol.py +++ b/deadserver/protocol.py @@ -23,7 +23,8 @@ def check(expression, errmsg): PROTOCOL_VERSION = types.Bytes(2)([0,1]) class MsgType(types.Uint8, enum.Enum): - OPEN = 1 + OPEN = 1 + ECHOTEST = 254 class ResponseStatus(types.Uint8, enum.Enum): OK = 0x01 @@ -77,7 +78,7 @@ def parse_packet(struct, buf, get_key): tail, get_key(hdr.controller_id)) payload = struct.unpack(payload_buf) except ValueError as e: - raise BadMessageError('parse_packet failed') from e + raise BadMessageError('parsing packet failed') from e return hdr, payload diff --git a/deadserver/server.py b/deadserver/server.py index 6cb2adb..fe5f722 100644 --- a/deadserver/server.py +++ b/deadserver/server.py @@ -24,8 +24,8 @@ def handle(self): class DeadServer: def __init__(self, config): - self.config = config - self.db = records.Database(config.db_url) + self.config = config + self.db = records.Database(self.config.db_url) self.handler = functools.partial(MessageHandler, api.API(db=self.db)) bind_addr = self.config.udp_host, self.config.udp_port diff --git a/fun_stuff.txt b/fun_stuff.txt index 7aa0c85..a8d7204 100644 --- a/fun_stuff.txt +++ b/fun_stuff.txt @@ -2,4 +2,6 @@ Fun problems that I've overcome =============================== - stateless is awesome -- avoid running around with secret keys by passing just a crypto black box (TODO but do it :D), and actually storing / loading the keys encrypted and decrypting only when really needed (TODO also do it :D) +- avoid running around with secret keys by passing just a crypto black box (TODO but do it :D), and actually storing / loading the keys encrypted and decrypting only when really needed in order to avoid e.g. accidentally logging it (TODO also do it :D) +- extensibility + runtime configuration for packet types +- extensibility for "batch jobs" (e.g. for local dbs creation) From 8166d30745d5c0e39728b8a06670d21de3c39ecb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamila=20Sou=C4=8Dkov=C3=A1?= Date: Tue, 5 Apr 2016 12:13:58 +0200 Subject: [PATCH 46/46] review-induced fixes --- controller_client.py | 2 +- deadserver/api.py | 5 +++-- deadserver/handlers/open.py | 6 +++--- deadserver/server.py | 35 +++++++++++++---------------------- deadserver/utils.py | 4 ---- requirements-fresh.txt | 1 + requirements.txt | 5 ++++- runserver.py | 3 +-- structparse/structdef.py | 2 +- 9 files changed, 27 insertions(+), 36 deletions(-) delete mode 100644 deadserver/utils.py diff --git a/controller_client.py b/controller_client.py index 3898f55..c042671 100644 --- a/controller_client.py +++ b/controller_client.py @@ -10,7 +10,7 @@ from deadserver.api import * from deadserver.protocol import * -api = API(db=records.Database(config.db_url)) +api = API(config=config, db=records.Database(config.db_url)) def msg(buf): sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) diff --git a/deadserver/api.py b/deadserver/api.py index 265bc81..296edef 100644 --- a/deadserver/api.py +++ b/deadserver/api.py @@ -8,8 +8,9 @@ from . import protocol class API: - def __init__(self, db): - self.db = db + def __init__(self, config, db): + self.config = config + self.db = db def handle_packet(self, in_buf): try: diff --git a/deadserver/handlers/open.py b/deadserver/handlers/open.py index a2f6a86..a97666c 100644 --- a/deadserver/handlers/open.py +++ b/deadserver/handlers/open.py @@ -6,12 +6,12 @@ from structparse import struct, types from deadserver.protocol import MsgType, ResponseStatus -ISICid = types.PascalStr(12) +CardId = types.PascalStr(12) -OpenRequest = struct('OpenRequest', (ISICid, 'isic_id')) +OpenRequest = struct('OpenRequest', (CardId, 'card_id')) @handles(MsgType.OPEN) @utils.unpack_indata_as(OpenRequest) def handle(controller_id, data, api): - status = ResponseStatus.OK if data.isic_id == ISICid('hello') else ResponseStatus.ERR + status = ResponseStatus.OK if data.card_id == CardId('hello') else ResponseStatus.ERR return status, None diff --git a/deadserver/server.py b/deadserver/server.py index fe5f722..9caec51 100644 --- a/deadserver/server.py +++ b/deadserver/server.py @@ -1,7 +1,8 @@ """The UDP server that handles controller messages. -This is not concerned with the protocol details, it only knows how to receive -requests and send responses, and passes stuff to `controller_api`. +This only knows how to receive requests and send responses -- it is just a thin wrapper around +`api.API` and it is not concerned with the protocol details. + """ import functools @@ -11,25 +12,15 @@ from . import api -class MessageHandler(socketserver.BaseRequestHandler): - def __init__(self, api, *args, **kwargs): - self.api = api - super().__init__(*args, **kwargs) - - def handle(self): - """Handles a request from the controller.""" - in_packet, socket = self.request - out_packet = self.api.handle_packet(in_packet) - if out_packet: socket.sendto(out_packet, self.client_address) - -class DeadServer: - def __init__(self, config): - self.config = config - self.db = records.Database(self.config.db_url) - self.handler = functools.partial(MessageHandler, api.API(db=self.db)) +def serve(config): + app = api.API(config=config, db=records.Database(config.db_url)) - bind_addr = self.config.udp_host, self.config.udp_port - self.server = socketserver.ThreadingUDPServer(bind_addr, self.handler) + class MessageHandler(socketserver.BaseRequestHandler): + def handle(self): + """Handles a request from the controller.""" + in_packet, socket = self.request + out_packet = app.handle_packet(in_packet) + if out_packet: socket.sendto(out_packet, self.client_address) - def serve(self): - self.server.serve_forever() + server = socketserver.ThreadingUDPServer((config.udp_host, config.udp_port), MessageHandler) + server.serve_forever() diff --git a/deadserver/utils.py b/deadserver/utils.py deleted file mode 100644 index 16dd3f8..0000000 --- a/deadserver/utils.py +++ /dev/null @@ -1,4 +0,0 @@ -"""Various utility functions.""" - -def unzip(lst): - return zip(*lst) # yay :D diff --git a/requirements-fresh.txt b/requirements-fresh.txt index 92479cb..6207840 100644 --- a/requirements-fresh.txt +++ b/requirements-fresh.txt @@ -1,3 +1,4 @@ +psycopg2 pynacl pytest pytest-cov diff --git a/requirements.txt b/requirements.txt index f26fde3..90c35ed 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,10 +1,13 @@ +PyNaCl==1.0.1 SQLAlchemy==1.0.12 +cffi==1.5.2 coverage==4.0.3 docopt==0.6.2 psycopg2==2.6.1 py==1.4.31 +pycparser==2.14 pytest==2.9.1 pytest-cov==2.2.1 records==0.4.3 +six==1.10.0 tablib==0.11.2 -https://github.com/warner/python-tweetnacl/tarball/b48a25a33f diff --git a/runserver.py b/runserver.py index 6b20f7b..6aa028f 100755 --- a/runserver.py +++ b/runserver.py @@ -5,5 +5,4 @@ import config if __name__ == '__main__': - app = server.DeadServer(config) - app.serve() + server.serve(config) diff --git a/structparse/structdef.py b/structparse/structdef.py index 19cc564..08a47b7 100644 --- a/structparse/structdef.py +++ b/structparse/structdef.py @@ -29,7 +29,7 @@ def unpack(cls, buf): Raises ValueError if len(buf) doesn't exactly match the type size. """ x, rest = cls.unpack_from(buf) - if len(rest) > 0: raise ValueError('buffer size != struct size') + if rest: raise ValueError('buffer size != struct size') return x