From 61affaff50340bd71396c09a085648a373a9937b Mon Sep 17 00:00:00 2001 From: JohnDamilola Date: Wed, 7 Aug 2024 11:07:28 -0400 Subject: [PATCH 01/20] Fix the text reconstruction --- learning_observer/learning_observer/google.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/learning_observer/learning_observer/google.py b/learning_observer/learning_observer/google.py index 35df5dd99..d51a6808a 100644 --- a/learning_observer/learning_observer/google.py +++ b/learning_observer/learning_observer/google.py @@ -432,7 +432,7 @@ def extract_text_from_google_doc_json( length = j['body']['content'][-1]['endIndex'] elements = [a.get('paragraph', {}).get('elements', []) for a in j['body']['content']] flat = sum(elements, []) - text_chunks = [f['textRun']['content'] for f in flat] + text_chunks = [f.get('textRun', {}).get('content', '') for f in flat] if align: lengths = [f['endIndex'] - f['startIndex'] for f in flat] text_chunks = [_force_text_length(chunk, length) for chunk, length in zip(text_chunks, lengths)] From 91164d15da6ff5ca68abfe564a17f9abf5019a49 Mon Sep 17 00:00:00 2001 From: JohnDamilola Date: Wed, 7 Aug 2024 11:09:44 -0400 Subject: [PATCH 02/20] Add missing import --- modules/writing_observer/writing_observer/aggregator.py | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/writing_observer/writing_observer/aggregator.py b/modules/writing_observer/writing_observer/aggregator.py index 6a81b422a..dfa727df3 100644 --- a/modules/writing_observer/writing_observer/aggregator.py +++ b/modules/writing_observer/writing_observer/aggregator.py @@ -285,6 +285,7 @@ async def fetch_doc_from_google(student, doc_id): if student is None or doc_id is None or len(doc_id) == 0: return None import learning_observer.google + from learning_observer.stream_analytics.fields import KeyField, KeyStateType, EventField kvs = learning_observer.kvs.KVS() From 7ee9dd1d4b9c4fff5c4dc34b755d5b3d47fef17c Mon Sep 17 00:00:00 2001 From: JohnDamilola Date: Wed, 7 Aug 2024 11:17:11 -0400 Subject: [PATCH 03/20] Implement canvas roster integration --- .gitignore | 4 +- learning_observer/learning_observer/canvas.py | 364 ++++++++++++++++++ .../learning_observer/rosters.py | 11 +- learning_observer/learning_observer/routes.py | 2 + 4 files changed, 379 insertions(+), 2 deletions(-) create mode 100755 learning_observer/learning_observer/canvas.py diff --git a/.gitignore b/.gitignore index d951ebd57..c212cac96 100644 --- a/.gitignore +++ b/.gitignore @@ -37,4 +37,6 @@ LanguageTool-5.4 package-lock.json learning_observer/learning_observer/static_data/google/ learning_observer/learning_observer/static_data/admins.yaml -.ipynb_checkpoints/ \ No newline at end of file +.ipynb_checkpoints/ +learning_observer/learning_observer/config.ini + diff --git a/learning_observer/learning_observer/canvas.py b/learning_observer/learning_observer/canvas.py new file mode 100755 index 000000000..f8b7a20c2 --- /dev/null +++ b/learning_observer/learning_observer/canvas.py @@ -0,0 +1,364 @@ +import os +import json +import string +import requests +import recordclass +import configparser +import aiohttp +import aiohttp.web +import pmss +import yaml + +import learning_observer.settings as settings +import learning_observer.log_event +import learning_observer.util +import learning_observer.auth +import learning_observer.runtime + +cache = None + +pmss.register_field( + name="default_server", + type=pmss.pmsstypes.TYPES.string, + description="The default server for Canva", + required=True +) +pmss.register_field( + name="client_id", + type=pmss.pmsstypes.TYPES.string, + description="The client ID for Canva", + required=True +) +pmss.register_field( + name="client_secret", + type=pmss.pmsstypes.TYPES.string, + description="The client secret for Canva", + required=True +) +pmss.register_field( + name="access_token", + type=pmss.pmsstypes.TYPES.string, + description="The access token for Canva", + required=True +) +pmss.register_field( + name="refresh_token", + type=pmss.pmsstypes.TYPES.string, + description="The refresh token for Canva", + required=True +) + +class Endpoint(recordclass.make_dataclass("Endpoint", ["name", "remote_url", "doc", "cleaners"], defaults=["", None])): + def arguments(self): + return extract_parameters_from_format_string(self.remote_url) + + def _local_url(self): + parameters = "}/{".join(self.arguments()) + base_url = f"/canvas/{self.name}" + if len(parameters) == 0: + return base_url + else: + return base_url + "/{" + parameters + "}" + + def _add_cleaner(self, name, cleaner): + if self.cleaners is None: + self.cleaners = dict() + self.cleaners[name] = cleaner + if 'local_url' not in cleaner: + cleaner['local_url'] = self._local_url + "/" + name + + def _cleaners(self): + if self.cleaners is None: + return [] + else: + return self.cleaners + +ENDPOINTS = list(map(lambda x: Endpoint(*x), [ + ("course_list", "/courses"), + ("course_roster", "/courses/{courseId}/students"), + ("course_work", "/courses/{courseId}/assignments"), + ("coursework_submissions", "/courses/{courseId}/assignments/{assignmentId}/submissions"), +])) + +def extract_parameters_from_format_string(format_string): + ''' + Extracts parameters from a format string. E.g. + + >>> ("hello {hi} my {bye}")] + ['hi', 'bye'] + ''' + # The parse returns a lot of context, which we discard. In particular, the + # last item is often about the suffix after the last parameter and may be + # `None` + return [f[1] for f in string.Formatter().parse(format_string) if f[1] is not None] + +class Canvas: + def __init__(self, config_path='./config.ini'): + script_dir = os.path.dirname(os.path.abspath(__file__)) + config_path = os.path.join(script_dir, config_path) + + self.config = configparser.ConfigParser() + self.config.read(config_path) + + # Check if 'SCHOOLOGY_CONFIG' section is present + if 'CANVAS_CONFIG' not in self.config: + raise KeyError("The configuration file does not contain 'CANVAS_CONFIG' section") + + try: + self.defaultServer = self.config['CANVAS_CONFIG']['DEFAULT_SERVER'] + self.access_token = self.config['CANVAS_CONFIG']['ACCESS_TOKEN'] + self.refresh_token = self.config['CANVAS_CONFIG']['REFRESH_TOKEN'] + self.client_id = self.config['CANVAS_CONFIG']['CLIENT_ID'] + self.client_secret = self.config['CANVAS_CONFIG']['CLIENT_SECRET'] + except KeyError as e: + raise KeyError(f"Missing required configuration key: {e}") + + """ + self.defaultServer = settings.pmss_settings.default_server(types=['canvas']) + self.access_token = settings.pmss_settings.access_token(types=['canvas']) + self.refresh_token = settings.pmss_settings.refresh_token(types=['canvas']) + self.client_id = settings.pmss_settings.client_id(types=['canvas']) + self.client_secret = settings.pmss_settings.client_secret(types=['canvas']) + """ + self.default_version = 'v1' + self.defaultPerPage = 10000 + self.base_url = f'https://{self.defaultServer}/api/{self.default_version}' + + def update_access_tokens(self, access_token): + self.config['CANVAS_CONFIG']['ACCESS_TOKEN'] = access_token + self.access_token = access_token + script_dir = os.path.dirname(os.path.abspath(__file__)) + """ + config_path = os.path.join(script_dir, '../creds.yaml') + with open(config_path, 'r') as configfile: + config = yaml.safe_load(configfile) + + config['canvas']['access_token'] = access_token + + with open(config_path, 'w') as configfile: + yaml.safe_dump(config, configfile, sort_keys=False) + """ + config_path = os.path.join(script_dir, './config.ini') + with open(config_path, 'w') as configfile: + self.config.write(configfile) + + async def api_call(self, method, endpoint, params=None, data=None, absolute_url=False, retry=True, **kwargs): + if absolute_url: + url = endpoint + else: + url = self.base_url + endpoint + #if params: + #url += '?' + '&'.join(f"{k}={v}" for k, v in params.items()) + + url = url.format(**kwargs) + + headers = { + 'Authorization': f'Bearer {self.access_token}', + 'Content-Type': 'application/json' + } + + async with aiohttp.ClientSession() as client: + response_func = getattr(client, method.lower()) + async with response_func(url, headers=headers, params=params, json=data) as response: + if response.status == 401 and retry: + new_tokens = await self.refresh_tokens() + if 'access_token' in new_tokens: + self.update_access_tokens(new_tokens['access_token']) + return await self.api_call(method, endpoint, params, data, absolute_url, retry=False, **kwargs) + + if response.status != 200: + response.raise_for_status() + + return await response.json() + + async def refresh_tokens(self): + url = f'https://{self.defaultServer}/login/oauth2/token' + params = { + "grant_type": "refresh_token", + "client_id": self.client_id, + "client_secret": self.client_secret, + "refresh_token": self.refresh_token + } + return await self.api_call('POST', url, params=params, absolute_url=True) + +async def raw_canvas_ajax(runtime, target_url, retry=False, **kwargs): + ''' + Make an AJAX call to Canvas, managing auth + auth. + + * runtime is a Runtime class containing request information. + * target_url is typically grabbed from ENDPOINTS + * ... and we pass the named parameters + ''' + canvas = Canvas() + + params = {k: v for k, v in kwargs.items() if v is not None} + try: + response = await canvas.api_call('GET', target_url, params=params, **kwargs) + #response["kwargs"] = kwargs + except aiohttp.ClientResponseError as e: + if e.status == 401 and retry: + new_tokens = await canvas.refresh_tokens() + if 'access_token' in new_tokens: + canvas.update_access_tokens(new_tokens['access_token']) + return await raw_canvas_ajax(runtime, target_url, retry=False, **kwargs) + raise + + print(kwargs) + return response + +def raw_access_partial(remote_url, name=None): + ''' + This is a helper which allows us to create a function which calls specific + Canvas APIs. + ''' + async def caller(request, **kwargs): + ''' + Make an AJAX request to Canvas + ''' + return await raw_canvas_ajax(request, remote_url, **kwargs) + setattr(caller, "__qualname__", name) + + return caller + +def initialize_and_register_routes(app): + ''' + This is a big 'ol function which might be broken into smaller ones at some + point. We: + + - Created debug routes to pass through AJAX requests to Google + - Created production APIs to have access to cleaned versions of said data + - Create local function calls to call from other pieces of code + within process + + We probably don't need all of this in production, but a lot of this is + very important for debugging. Having APIs is more useful than it looks, since + making use of Google APIs requires a lot of infrastructure (registering + apps, auth/auth, etc.) which we already have in place on dev / debug servers. + ''' + app.add_routes([ + aiohttp.web.get("/canvas", api_docs_handler) + ]) + + def make_ajax_raw_handler(remote_url): + async def ajax_passthrough(request): + runtime = learning_observer.runtime.Runtime(request) + response = await raw_canvas_ajax(runtime, remote_url, retry=True, **request.match_info) + return aiohttp.web.json_response(response) + return ajax_passthrough + + def make_cleaner_handler(raw_function, cleaner_function, name=None): + async def cleaner_handler(request): + response = cleaner_function(await raw_function(request, **request.match_info)) + if isinstance(response, dict) or isinstance(response, list): + return aiohttp.web.json_response(response) + elif isinstance(response, str): + return aiohttp.web.Response(text=response) + else: + raise AttributeError(f"Invalid response type: {type(response)}") + if name is not None: + setattr(cleaner_handler, "__qualname__", name + "_handler") + + return cleaner_handler + + def make_cleaner_function(raw_function, cleaner_function, name=None): + async def cleaner_local(request, **kwargs): + canvas_response = await raw_function(request, **kwargs) + clean = cleaner_function(canvas_response) + return clean + if name is not None: + setattr(cleaner_local, "__qualname__", name) + return cleaner_local + + for e in ENDPOINTS: + function_name = f"raw_{e.name}" + raw_function = raw_access_partial(remote_url=e.remote_url, name=e.name) + globals()[function_name] = raw_function + cleaners = e._cleaners() + for c in cleaners: + app.add_routes([ + aiohttp.web.get( + cleaners[c]['local_url'], + make_cleaner_handler( + raw_function, + cleaners[c]['function'], + name=cleaners[c]['name'] + ) + ) + ]) + globals()[cleaners[c]['name']] = make_cleaner_function( + raw_function, + cleaners[c]['function'], + name=cleaners[c]['name'] + ) + app.add_routes([ + aiohttp.web.get(e._local_url(), make_ajax_raw_handler(e.remote_url)) + ]) + +def api_docs_handler(request): + response = "URL Endpoints:\n\n" + for endpoint in ENDPOINTS: + response += f"{endpoint._local_url()}\n" + cleaners = endpoint._cleaners() + for c in cleaners: + response += f" {cleaners[c]['local_url']}\n" + response += "\n\n Globals:" + return aiohttp.web.Response(text=response) + +def register_cleaner(data_source, cleaner_name): + def decorator(f): + found = False + for endpoint in ENDPOINTS: + if endpoint.name == data_source: + found = True + endpoint._add_cleaner( + cleaner_name, + { + 'function': f, + 'local_url': f'{endpoint._local_url()}/{cleaner_name}', + 'name': cleaner_name + } + ) + if not found: + raise AttributeError(f"Data source {data_source} invalid; not found in endpoints.") + return f + return decorator + +@register_cleaner("course_roster", "roster") +def clean_course_roster(canvas_json): + students = canvas_json + students_updated = [] + #students.sort(key=lambda x: x.get('name', {}).get('fullName', 'ZZ')) + for student_json in students: + canvas_id = student_json['id'] + integration_id = student_json['integration_id'] + #canvas_id = "118337587800688588675" + local_id = learning_observer.auth.google_id_to_user_id(integration_id) + student = { + "course_id": "1", + "user_id": local_id, + "profile": { + "id": canvas_id, + "name": { + "given_name": student_json['name'], + "family_name": student_json['name'], + "full_name": student_json['name'] + } + } + } + #local_id = learning_observer.auth.canvas_id_to_user_id(canvas_id) + #student_json['user_id'] = local_id + if 'external_ids' not in student_json: + student_json['external_ids'] = [] + student_json['external_ids'].append({"source": "canvas", "id": integration_id}) + students_updated.append(student) + return students_updated + +@register_cleaner("course_list", "courses") +def clean_course_list(canvas_json): + courses = canvas_json + courses.sort(key=lambda x: x.get('name', 'ZZ')) + return courses + +if __name__ == '__main__': + output = clean_course_roster({}) + print(json.dumps(output, indent=2)) diff --git a/learning_observer/learning_observer/rosters.py b/learning_observer/learning_observer/rosters.py index 0f233f92a..868e0027e 100644 --- a/learning_observer/learning_observer/rosters.py +++ b/learning_observer/learning_observer/rosters.py @@ -73,6 +73,7 @@ import learning_observer.cache import learning_observer.constants as constants import learning_observer.google +import learning_observer.canvas import learning_observer.kvs import learning_observer.log_event as log_event from learning_observer.log_event import debug_log @@ -86,7 +87,7 @@ COURSE_URL = 'https://classroom.googleapis.com/v1/courses' ROSTER_URL = 'https://classroom.googleapis.com/v1/courses/{courseid}/students' -pmss.parser('roster_source', parent='string', choices=['google_api', 'all', 'test', 'filesystem'], transform=None) +pmss.parser('roster_source', parent='string', choices=['google_api', 'all', 'test', 'canvas', 'filesystem'], transform=None) pmss.register_field( name='source', type='roster_source', @@ -363,6 +364,8 @@ def init(): ajax = google_ajax elif roster_source in ["all"]: ajax = all_ajax + elif roster_source in ["canvas"]: + ajax = google_ajax else: raise learning_observer.prestartup.StartupCheck( "Settings file `roster_data` element should have `source` field\n" @@ -413,6 +416,9 @@ async def courselist(request): if settings.pmss_settings.source(types=['roster_data']) in ["google_api"]: runtime = learning_observer.runtime.Runtime(request) return await learning_observer.google.courses(runtime) + elif settings.pmss_settings.source(types=['roster_data']) in ["canvas"]: + runtime = learning_observer.runtime.Runtime(request) + return await learning_observer.canvas.courses(runtime) # Legacy code course_list = await ajax( @@ -457,6 +463,9 @@ async def courseroster(request, course_id): if settings.pmss_settings.source(types=['roster_data']) in ["google_api"]: runtime = learning_observer.runtime.Runtime(request) return await learning_observer.google.roster(runtime, courseId=course_id) + elif settings.pmss_settings.source(types=['roster_data']) in ["canvas"]: + runtime = learning_observer.runtime.Runtime(request) + return await learning_observer.canvas.roster(runtime, courseId=course_id) roster = await ajax( request, diff --git a/learning_observer/learning_observer/routes.py b/learning_observer/learning_observer/routes.py index b7adf3812..90ea5bdad 100644 --- a/learning_observer/learning_observer/routes.py +++ b/learning_observer/learning_observer/routes.py @@ -23,6 +23,7 @@ import learning_observer.incoming_student_event as incoming_student_event import learning_observer.dashboard import learning_observer.google +import learning_observer.canvas import learning_observer.rosters as rosters import learning_observer.module_loader @@ -67,6 +68,7 @@ def tracemalloc_handler(request): register_incoming_event_views(app) register_debug_routes(app) learning_observer.google.initialize_and_register_routes(app) + learning_observer.canvas.initialize_and_register_routes(app) app.add_routes([ aiohttp.web.get( From 877d330f59152529df4db4761e47a35cb557c54a Mon Sep 17 00:00:00 2001 From: JohnDamilola Date: Wed, 7 Aug 2024 11:19:40 -0400 Subject: [PATCH 04/20] Implement canvas roster integration --- learning_observer/learning_observer/canvas.py | 23 ------------------- 1 file changed, 23 deletions(-) diff --git a/learning_observer/learning_observer/canvas.py b/learning_observer/learning_observer/canvas.py index f8b7a20c2..254d37c5b 100755 --- a/learning_observer/learning_observer/canvas.py +++ b/learning_observer/learning_observer/canvas.py @@ -112,14 +112,6 @@ def __init__(self, config_path='./config.ini'): self.client_secret = self.config['CANVAS_CONFIG']['CLIENT_SECRET'] except KeyError as e: raise KeyError(f"Missing required configuration key: {e}") - - """ - self.defaultServer = settings.pmss_settings.default_server(types=['canvas']) - self.access_token = settings.pmss_settings.access_token(types=['canvas']) - self.refresh_token = settings.pmss_settings.refresh_token(types=['canvas']) - self.client_id = settings.pmss_settings.client_id(types=['canvas']) - self.client_secret = settings.pmss_settings.client_secret(types=['canvas']) - """ self.default_version = 'v1' self.defaultPerPage = 10000 self.base_url = f'https://{self.defaultServer}/api/{self.default_version}' @@ -128,16 +120,6 @@ def update_access_tokens(self, access_token): self.config['CANVAS_CONFIG']['ACCESS_TOKEN'] = access_token self.access_token = access_token script_dir = os.path.dirname(os.path.abspath(__file__)) - """ - config_path = os.path.join(script_dir, '../creds.yaml') - with open(config_path, 'r') as configfile: - config = yaml.safe_load(configfile) - - config['canvas']['access_token'] = access_token - - with open(config_path, 'w') as configfile: - yaml.safe_dump(config, configfile, sort_keys=False) - """ config_path = os.path.join(script_dir, './config.ini') with open(config_path, 'w') as configfile: self.config.write(configfile) @@ -194,7 +176,6 @@ async def raw_canvas_ajax(runtime, target_url, retry=False, **kwargs): params = {k: v for k, v in kwargs.items() if v is not None} try: response = await canvas.api_call('GET', target_url, params=params, **kwargs) - #response["kwargs"] = kwargs except aiohttp.ClientResponseError as e: if e.status == 401 and retry: new_tokens = await canvas.refresh_tokens() @@ -327,11 +308,9 @@ def decorator(f): def clean_course_roster(canvas_json): students = canvas_json students_updated = [] - #students.sort(key=lambda x: x.get('name', {}).get('fullName', 'ZZ')) for student_json in students: canvas_id = student_json['id'] integration_id = student_json['integration_id'] - #canvas_id = "118337587800688588675" local_id = learning_observer.auth.google_id_to_user_id(integration_id) student = { "course_id": "1", @@ -345,8 +324,6 @@ def clean_course_roster(canvas_json): } } } - #local_id = learning_observer.auth.canvas_id_to_user_id(canvas_id) - #student_json['user_id'] = local_id if 'external_ids' not in student_json: student_json['external_ids'] = [] student_json['external_ids'].append({"source": "canvas", "id": integration_id}) From a6fe147e17cbb7897d06588885589964a7d4e73e Mon Sep 17 00:00:00 2001 From: JohnDamilola Date: Wed, 7 Aug 2024 13:38:08 -0400 Subject: [PATCH 05/20] Update comments to accommodate for Canvas --- .../learning_observer/rosters.py | 29 ++++++++++--------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/learning_observer/learning_observer/rosters.py b/learning_observer/learning_observer/rosters.py index 868e0027e..8d24fb971 100644 --- a/learning_observer/learning_observer/rosters.py +++ b/learning_observer/learning_observer/rosters.py @@ -10,6 +10,7 @@ We can either retrieve class rosters from: - Google Classroom (config setting: 'google') +- Canvas (config setting: 'canvas') - Text files on the disk for testing. (config setting: 'test') We have two files: - courses.json @@ -27,14 +28,14 @@ As well as the option for several sources in the same system, perhaps. This file could be cleaned up a lot. Right now, we do a lot of this by -mock calls to Google AJAX. It also contains a large number of hacks which +mock calls to Google or Canvas AJAX. It also contains a large number of hacks which we use to manage the data and to address variations in the roster sources -whether we are taking them from google or from our own backup data. +whether we are taking them from google or canvas or from our own backup data. As of now this partially implements a separation between the internal ID which shows up in our rosters as id or `user_id` and the id used for the external sources of data. We store external ids on student data under -external_ids and keep space for ids from google etc. However as of now +external_ids and keep space for ids from google, canvas etc. However as of now we do not make use of it. Ultimately it would be ideal to move so that remote data retreival and raw document storage are done under an internal id with this translation taking place at event storage time *or* that the @@ -44,7 +45,7 @@ the potential to create some extra, though probably manageable, queries. In either case we get around it now by also adding in a cheap hack that -makes the internal ID for google-sourced users match the google ID. This +makes the internal ID for google/canvas-sourced users match the google ID. This will need to change in a stable way for future use. Note that these APIs and file locations aren't finished. In the future, @@ -95,14 +96,15 @@ '`all`: aggregate all available students into a single class\n'\ '`test`: use sample course and student files\n'\ '`filesystem`: read rosters defined on filesystem\n'\ - '`google_api`: fetch from Google API', + '`google_api`: fetch from Google API\n'\ + '`canvas`: fetch from Canvas API', required=True ) -def clean_google_ajax_data(resp_json, key, sort_key, default=None, source=None): +def clean_combined_ajax_data(resp_json, key, sort_key, default=None, source=None): ''' - This cleans up / standardizes Google AJAX data. In particular: + This cleans up / standardizes Google/Canvas AJAX data. In particular: - We want to handle errors and empty lists better - We often don't want the whole response, but just one field (`key`) @@ -257,7 +259,7 @@ async def synthetic_ajax( request, url, parameters=None, key=None, sort_key=None, default=None): ''' - Stub similar to google_ajax, but grabbing data from local files. + Stub similar to combined_ajax, but grabbing data from local files. This is helpful for testing, but it's even more helpful since Google is an amazingly unreliable B2B company, and this lets us @@ -302,11 +304,11 @@ async def synthetic_ajax( return data -async def google_ajax( +async def combined_ajax( request, url, parameters=None, key=None, sort_key=None, default=None): ''' - Request information through Google's API + Request information through the specified API Most requests return a dictionary with one key. If we just want that element, set `key` to be the element of the dictionary we want @@ -330,7 +332,7 @@ async def google_ajax( async with client.get(url.format(**parameters), headers=request[constants.AUTH_HEADERS]) as resp: resp_json = await resp.json() log_event.log_ajax(url, resp_json, request) - return clean_google_ajax_data( + return clean_combined_ajax_data( resp_json, key, sort_key, default=default ) @@ -361,17 +363,18 @@ def init(): elif roster_source in ['test', 'filesystem']: ajax = synthetic_ajax elif roster_source in ["google_api"]: - ajax = google_ajax + ajax = combined_ajax elif roster_source in ["all"]: ajax = all_ajax elif roster_source in ["canvas"]: - ajax = google_ajax + ajax = combined_ajax else: raise learning_observer.prestartup.StartupCheck( "Settings file `roster_data` element should have `source` field\n" "set to either:\n" " test (retrieve from files courses.json and students.json)\n" " google_api (retrieve roster data from Google)\n" + " canvas (retrieve roster data from Canvas)\n" " filesystem (retrieve roster data from file system hierarchy\n" " all (retrieve roster data as all students)" ) From 0eb0f62bd9f5a3cc9681bf8cc55a154272675530 Mon Sep 17 00:00:00 2001 From: JohnDamilola Date: Fri, 9 Aug 2024 00:04:29 -0400 Subject: [PATCH 06/20] Abstract LMS shared code --- learning_observer/learning_observer/canvas.py | 188 ++-------------- learning_observer/learning_observer/google.py | 204 +----------------- .../learning_observer/lms_integration.py | 164 ++++++++++++++ learning_observer/learning_observer/routes.py | 11 +- 4 files changed, 200 insertions(+), 367 deletions(-) create mode 100644 learning_observer/learning_observer/lms_integration.py diff --git a/learning_observer/learning_observer/canvas.py b/learning_observer/learning_observer/canvas.py index 254d37c5b..2c8979ef0 100755 --- a/learning_observer/learning_observer/canvas.py +++ b/learning_observer/learning_observer/canvas.py @@ -1,97 +1,25 @@ import os import json -import string -import requests -import recordclass import configparser import aiohttp import aiohttp.web -import pmss -import yaml -import learning_observer.settings as settings import learning_observer.log_event import learning_observer.util import learning_observer.auth import learning_observer.runtime -cache = None +from learning_observer.lms_integration import Endpoint, register_cleaner, api_docs_handler, raw_access_partial, make_ajax_raw_handler, make_cleaner_handler, make_cleaner_function -pmss.register_field( - name="default_server", - type=pmss.pmsstypes.TYPES.string, - description="The default server for Canva", - required=True -) -pmss.register_field( - name="client_id", - type=pmss.pmsstypes.TYPES.string, - description="The client ID for Canva", - required=True -) -pmss.register_field( - name="client_secret", - type=pmss.pmsstypes.TYPES.string, - description="The client secret for Canva", - required=True -) -pmss.register_field( - name="access_token", - type=pmss.pmsstypes.TYPES.string, - description="The access token for Canva", - required=True -) -pmss.register_field( - name="refresh_token", - type=pmss.pmsstypes.TYPES.string, - description="The refresh token for Canva", - required=True -) +LMS = "canvas" -class Endpoint(recordclass.make_dataclass("Endpoint", ["name", "remote_url", "doc", "cleaners"], defaults=["", None])): - def arguments(self): - return extract_parameters_from_format_string(self.remote_url) - - def _local_url(self): - parameters = "}/{".join(self.arguments()) - base_url = f"/canvas/{self.name}" - if len(parameters) == 0: - return base_url - else: - return base_url + "/{" + parameters + "}" - - def _add_cleaner(self, name, cleaner): - if self.cleaners is None: - self.cleaners = dict() - self.cleaners[name] = cleaner - if 'local_url' not in cleaner: - cleaner['local_url'] = self._local_url + "/" + name - - def _cleaners(self): - if self.cleaners is None: - return [] - else: - return self.cleaners - -ENDPOINTS = list(map(lambda x: Endpoint(*x), [ +ENDPOINTS = list(map(lambda x: Endpoint(*x, "", None, LMS), [ ("course_list", "/courses"), ("course_roster", "/courses/{courseId}/students"), ("course_work", "/courses/{courseId}/assignments"), ("coursework_submissions", "/courses/{courseId}/assignments/{assignmentId}/submissions"), ])) -def extract_parameters_from_format_string(format_string): - ''' - Extracts parameters from a format string. E.g. - - >>> ("hello {hi} my {bye}")] - ['hi', 'bye'] - ''' - # The parse returns a lot of context, which we discard. In particular, the - # last item is often about the suffix after the last parameter and may be - # `None` - return [f[1] for f in string.Formatter().parse(format_string) if f[1] is not None] - class Canvas: def __init__(self, config_path='./config.ini'): script_dir = os.path.dirname(os.path.abspath(__file__)) @@ -186,84 +114,29 @@ async def raw_canvas_ajax(runtime, target_url, retry=False, **kwargs): print(kwargs) return response - -def raw_access_partial(remote_url, name=None): - ''' - This is a helper which allows us to create a function which calls specific - Canvas APIs. - ''' - async def caller(request, **kwargs): - ''' - Make an AJAX request to Canvas - ''' - return await raw_canvas_ajax(request, remote_url, **kwargs) - setattr(caller, "__qualname__", name) - return caller - -def initialize_and_register_routes(app): + +def initialize_canvas_routes(app): ''' - This is a big 'ol function which might be broken into smaller ones at some - point. We: - - - Created debug routes to pass through AJAX requests to Google + - Created debug routes to pass through AJAX requests to Canvas - Created production APIs to have access to cleaned versions of said data - - Create local function calls to call from other pieces of code - within process - - We probably don't need all of this in production, but a lot of this is - very important for debugging. Having APIs is more useful than it looks, since - making use of Google APIs requires a lot of infrastructure (registering - apps, auth/auth, etc.) which we already have in place on dev / debug servers. - ''' + - Create local function calls to call from other pieces of code within process + ''' + # Provide documentation on what we're doing app.add_routes([ aiohttp.web.get("/canvas", api_docs_handler) ]) - def make_ajax_raw_handler(remote_url): - async def ajax_passthrough(request): - runtime = learning_observer.runtime.Runtime(request) - response = await raw_canvas_ajax(runtime, remote_url, retry=True, **request.match_info) - return aiohttp.web.json_response(response) - return ajax_passthrough - - def make_cleaner_handler(raw_function, cleaner_function, name=None): - async def cleaner_handler(request): - response = cleaner_function(await raw_function(request, **request.match_info)) - if isinstance(response, dict) or isinstance(response, list): - return aiohttp.web.json_response(response) - elif isinstance(response, str): - return aiohttp.web.Response(text=response) - else: - raise AttributeError(f"Invalid response type: {type(response)}") - if name is not None: - setattr(cleaner_handler, "__qualname__", name + "_handler") - - return cleaner_handler - - def make_cleaner_function(raw_function, cleaner_function, name=None): - async def cleaner_local(request, **kwargs): - canvas_response = await raw_function(request, **kwargs) - clean = cleaner_function(canvas_response) - return clean - if name is not None: - setattr(cleaner_local, "__qualname__", name) - return cleaner_local - for e in ENDPOINTS: function_name = f"raw_{e.name}" - raw_function = raw_access_partial(remote_url=e.remote_url, name=e.name) + raw_function = raw_access_partial(raw_canvas_ajax, remote_url=e.remote_url, name=e.name) globals()[function_name] = raw_function cleaners = e._cleaners() for c in cleaners: app.add_routes([ aiohttp.web.get( cleaners[c]['local_url'], - make_cleaner_handler( - raw_function, - cleaners[c]['function'], - name=cleaners[c]['name'] - ) + make_cleaner_handler(raw_function, cleaners[c]['function'], name=cleaners[c]['name']) ) ]) globals()[cleaners[c]['name']] = make_cleaner_function( @@ -272,39 +145,10 @@ async def cleaner_local(request, **kwargs): name=cleaners[c]['name'] ) app.add_routes([ - aiohttp.web.get(e._local_url(), make_ajax_raw_handler(e.remote_url)) + aiohttp.web.get(e._local_url(), make_ajax_raw_handler(raw_canvas_ajax, e.remote_url)) ]) - -def api_docs_handler(request): - response = "URL Endpoints:\n\n" - for endpoint in ENDPOINTS: - response += f"{endpoint._local_url()}\n" - cleaners = endpoint._cleaners() - for c in cleaners: - response += f" {cleaners[c]['local_url']}\n" - response += "\n\n Globals:" - return aiohttp.web.Response(text=response) - -def register_cleaner(data_source, cleaner_name): - def decorator(f): - found = False - for endpoint in ENDPOINTS: - if endpoint.name == data_source: - found = True - endpoint._add_cleaner( - cleaner_name, - { - 'function': f, - 'local_url': f'{endpoint._local_url()}/{cleaner_name}', - 'name': cleaner_name - } - ) - if not found: - raise AttributeError(f"Data source {data_source} invalid; not found in endpoints.") - return f - return decorator - -@register_cleaner("course_roster", "roster") + +@register_cleaner("course_roster", "roster", ENDPOINTS) def clean_course_roster(canvas_json): students = canvas_json students_updated = [] @@ -330,7 +174,7 @@ def clean_course_roster(canvas_json): students_updated.append(student) return students_updated -@register_cleaner("course_list", "courses") +@register_cleaner("course_list", "courses", ENDPOINTS) def clean_course_list(canvas_json): courses = canvas_json courses.sort(key=lambda x: x.get('name', 'ZZ')) @@ -338,4 +182,4 @@ def clean_course_list(canvas_json): if __name__ == '__main__': output = clean_course_roster({}) - print(json.dumps(output, indent=2)) + print(json.dumps(output, indent=2)) \ No newline at end of file diff --git a/learning_observer/learning_observer/google.py b/learning_observer/learning_observer/google.py index d51a6808a..b923a94df 100644 --- a/learning_observer/learning_observer/google.py +++ b/learning_observer/learning_observer/google.py @@ -21,16 +21,12 @@ analysis. ''' -import collections import itertools import json -import recordclass -import string import re import aiohttp import aiohttp.web -import aiohttp_session import learning_observer.constants as constants import learning_observer.settings as settings @@ -40,8 +36,11 @@ import learning_observer.runtime import learning_observer.prestartup +from learning_observer.lms_integration import Endpoint, register_cleaner, api_docs_handler, raw_access_partial, make_ajax_raw_handler, make_cleaner_handler, make_cleaner_function + cache = None +LMS = "google" GOOGLE_FIELDS = [ @@ -59,38 +58,7 @@ GOOGLE_TO_SNAKE = {field: camel_to_snake.sub('_', field).lower() for field in GOOGLE_FIELDS} -# These took a while to find, but many are documented here: -# https://developers.google.com/drive/api/v3/reference/ -# This list might change. Many of these contain additional (optional) parameters -# which we might add later. This is here for debugging, mostly. We'll stabilize -# APIs later. -class Endpoint(recordclass.make_dataclass("Endpoint", ["name", "remote_url", "doc", "cleaners"], defaults=["", None])): - def arguments(self): - return extract_parameters_from_format_string(self.remote_url) - - def _local_url(self): - parameters = "}/{".join(self.arguments()) - base_url = f"/google/{self.name}" - if len(parameters) == 0: - return base_url - else: - return base_url + "/{" + parameters + "}" - - def _add_cleaner(self, name, cleaner): - if self.cleaners is None: - self.cleaners = dict() - self.cleaners[name] = cleaner - if 'local_url' not in cleaner: - cleaner['local_url'] = self._local_url + "/" + name - - def _cleaners(self): - if self.cleaners is None: - return [] - else: - return self.cleaners - - -ENDPOINTS = list(map(lambda x: Endpoint(*x), [ +ENDPOINTS = list(map(lambda x: Endpoint(*x, "", None, LMS), [ ("document", "https://docs.googleapis.com/v1/documents/{documentId}"), ("course_list", "https://classroom.googleapis.com/v1/courses"), ("course_roster", "https://classroom.googleapis.com/v1/courses/{courseId}/students"), @@ -105,19 +73,6 @@ def _cleaners(self): ])) -def extract_parameters_from_format_string(format_string): - ''' - Extracts parameters from a format string. E.g. - - >>> ("hello {hi} my {bye}")] - ['hi', 'bye'] - ''' - # The parse returns a lot of context, which we discard. In particular, the - # last item is often about the suffix after the last parameter and may be - # `None` - return [f[1] for f in string.Formatter().parse(format_string) if f[1] is not None] - - async def raw_google_ajax(runtime, target_url, **kwargs): ''' Make an AJAX call to Google, managing auth + auth. @@ -152,25 +107,6 @@ async def raw_google_ajax(runtime, target_url, **kwargs): ) -def raw_access_partial(remote_url, name=None): - ''' - This is a helper which allows us to create a function which calls specific - Google APIs. - - To test this, try: - - print(await raw_document(request, documentId="some_google_doc_id")) - ''' - async def caller(request, **kwargs): - ''' - Make an AJAX request to Google - ''' - return await raw_google_ajax(request, remote_url, **kwargs) - setattr(caller, "__qualname__", name) - - return caller - - @learning_observer.prestartup.register_startup_check def connect_to_google_cache(): '''Setup cache for requests to the Google API. @@ -193,84 +129,18 @@ def connect_to_google_cache(): ' subdirs: true\n```\nOR\n'\ '```\ngoogle_cache:\n type: redis_ephemeral\n expiry: 600\n```' raise learning_observer.prestartup.StartupCheck("Google KVS: " + error_text) - - -def initialize_and_register_routes(app): + +def initialize_google_routes(app): ''' - This is a big 'ol function which might be broken into smaller ones at some - point. We: - - Created debug routes to pass through AJAX requests to Google - Created production APIs to have access to cleaned versions of said data - - Create local function calls to call from other pieces of code - within process - - We probably don't need all of this in production, but a lot of this is - very important for debugging. Having APIs is more useful than it looks, since - making use of Google APIs requires a lot of infrastructure (registering - apps, auth/auth, etc.) which we already have in place on dev / debug servers. + - Create local function calls to call from other pieces of code within process ''' - # # For now, all of this is behind one big feature flag. In the future, - # # we'll want seperate ones for the debugging tools and the production - # # staff - # if 'google_routes' not in settings.settings['feature_flags']: - # return - # Provide documentation on what we're doing app.add_routes([ aiohttp.web.get("/google", api_docs_handler) ]) - def make_ajax_raw_handler(remote_url): - ''' - This creates a handler to forward Google requests to the client. It's used - for debugging right now. We should think through APIs before relying on this. - ''' - async def ajax_passthrough(request): - ''' - And the actual handler.... - ''' - runtime = learning_observer.runtime.Runtime(request) - response = await raw_google_ajax( - runtime, - remote_url, - **request.match_info - ) - - return aiohttp.web.json_response(response) - return ajax_passthrough - - def make_cleaner_handler(raw_function, cleaner_function, name=None): - async def cleaner_handler(request): - ''' - ''' - response = cleaner_function( - await raw_function(request, **request.match_info) - ) - if isinstance(response, dict) or isinstance(response, list): - return aiohttp.web.json_response( - response - ) - elif isinstance(response, str): - return aiohttp.web.Response( - text=response - ) - else: - raise AttributeError(f"Invalid response type: {type(response)}") - if name is not None: - setattr(cleaner_handler, "__qualname__", name + "_handler") - - return cleaner_handler - - def make_cleaner_function(raw_function, cleaner_function, name=None): - async def cleaner_local(request, **kwargs): - google_response = await raw_function(request, **kwargs) - clean = cleaner_function(google_response) - return clean - if name is not None: - setattr(cleaner_local, "__qualname__", name) - return cleaner_local - for e in ENDPOINTS: function_name = f"raw_{e.name}" raw_function = raw_access_partial(remote_url=e.remote_url, name=e.name) @@ -280,11 +150,7 @@ async def cleaner_local(request, **kwargs): app.add_routes([ aiohttp.web.get( cleaners[c]['local_url'], - make_cleaner_handler( - raw_function, - cleaners[c]['function'], - name=cleaners[c]['name'] - ) + make_cleaner_handler(raw_function, cleaners[c]['function'], name=cleaners[c]['name']) ) ]) globals()[cleaners[c]['name']] = make_cleaner_function( @@ -293,59 +159,11 @@ async def cleaner_local(request, **kwargs): name=cleaners[c]['name'] ) app.add_routes([ - aiohttp.web.get( - e._local_url(), - make_ajax_raw_handler(e.remote_url) - ) + aiohttp.web.get(raw_google_ajax, e._local_url(), make_ajax_raw_handler(e.remote_url)) ]) - -def api_docs_handler(request): - ''' - Return a list of available endpoints. - - Eventually, we should also document available function calls - ''' - response = "URL Endpoints:\n\n" - for endpoint in ENDPOINTS: - response += f"{endpoint._local_url()}\n" - cleaners = endpoint._cleaners() - for c in cleaners: - response += f" {cleaners[c]['local_url']}\n" - response += "\n\n Globals:" - if False: - response += str(globals()) - return aiohttp.web.Response(text=response) - - -def register_cleaner(data_source, cleaner_name): - ''' - This will register a cleaner function, for export both as a web service - and as a local function call. - ''' - def decorator(f): - found = False - for endpoint in ENDPOINTS: - if endpoint.name == data_source: - found = True - endpoint._add_cleaner( - cleaner_name, - { - 'function': f, - 'local_url': f'{endpoint._local_url()}/{cleaner_name}', - 'name': cleaner_name - } - ) - - if not found: - raise AttributeError(f"Data source {data_source} invalid; not found in endpoints.") - return f - - return decorator - - # Rosters -@register_cleaner("course_roster", "roster") +@register_cleaner("course_roster", "roster", ENDPOINTS) def clean_course_roster(google_json): ''' Retrieve the roster for a course, alphabetically @@ -368,7 +186,7 @@ def clean_course_roster(google_json): return students -@register_cleaner("course_list", "courses") +@register_cleaner("course_list", "courses", ENDPOINTS) def clean_course_list(google_json): ''' Google's course list is one object deeper than we'd like, and alphabetic diff --git a/learning_observer/learning_observer/lms_integration.py b/learning_observer/learning_observer/lms_integration.py new file mode 100644 index 000000000..f633938da --- /dev/null +++ b/learning_observer/learning_observer/lms_integration.py @@ -0,0 +1,164 @@ +import recordclass +import string +import aiohttp +import aiohttp.web +import learning_observer.runtime + + +# These took a while to find, but many are documented here: +# https://developers.google.com/drive/api/v3/reference/ +# This list might change. Many of these contain additional (optional) parameters +# which we might add later. This is here for debugging, mostly. We'll stabilize +# APIs later. +class Endpoint(recordclass.make_dataclass("Endpoint", ["name", "remote_url", "doc", "cleaners", "lms"], defaults=["", None])): + def arguments(self): + return extract_parameters_from_format_string(self.remote_url) + + def _local_url(self): + parameters = "}/{".join(self.arguments()) + base_url = f"/{self.lms}/{self.name}" + if len(parameters) == 0: + return base_url + else: + return base_url + "/{" + parameters + "}" + + def _add_cleaner(self, name, cleaner): + if self.cleaners is None: + self.cleaners = dict() + self.cleaners[name] = cleaner + if 'local_url' not in cleaner: + cleaner['local_url'] = self._local_url + "/" + name + + def _cleaners(self): + if self.cleaners is None: + return [] + else: + return self.cleaners + +def extract_parameters_from_format_string(format_string): + ''' + Extracts parameters from a format string. E.g. + >>> ("hello {hi} my {bye}")] + ['hi', 'bye'] + ''' + return [f[1] for f in string.Formatter().parse(format_string) if f[1] is not None] + +def raw_access_partial(raw_ajax, remote_url, name=None): + ''' + This is a helper which allows us to create a function which calls specific + Google APIs. + + To test this, try: + + print(await raw_document(request, documentId="some_google_doc_id")) + ''' + async def caller(request, **kwargs): + ''' + Make an AJAX request to Google + ''' + return await raw_ajax(request, remote_url, **kwargs) + setattr(caller, "__qualname__", name) + + return caller + +def api_docs_handler(request, ENDPOINTS): + ''' + Return a list of available endpoints. + + Eventually, we should also document available function calls + ''' + response = "URL Endpoints:\n\n" + for endpoint in ENDPOINTS: + response += f"{endpoint._local_url()}\n" + cleaners = endpoint._cleaners() + for c in cleaners: + response += f" {cleaners[c]['local_url']}\n" + response += "\n\n Globals:" + return aiohttp.web.Response(text=response) + +def register_cleaner(data_source, cleaner_name, ENDPOINTS): + ''' + This will register a cleaner function, for export both as a web service + and as a local function call. + ''' + def decorator(f): + found = False + for endpoint in ENDPOINTS: + if endpoint.name == data_source: + found = True + endpoint._add_cleaner( + cleaner_name, + { + 'function': f, + 'local_url': f'{endpoint._local_url()}/{cleaner_name}', + 'name': cleaner_name + } + ) + + if not found: + raise AttributeError(f"Data source {data_source} invalid; not found in endpoints.") + return f + + return decorator + +def make_ajax_raw_handler(raw_canvas_ajax, remote_url): + async def ajax_passthrough(request): + runtime = learning_observer.runtime.Runtime(request) + response = await raw_canvas_ajax(runtime, remote_url, retry=True, **request.match_info) + return aiohttp.web.json_response(response) + return ajax_passthrough + +def make_cleaner_handler(raw_function, cleaner_function, name=None): + async def cleaner_handler(request): + response = cleaner_function(await raw_function(request, **request.match_info)) + if isinstance(response, dict) or isinstance(response, list): + return aiohttp.web.json_response(response) + elif isinstance(response, str): + return aiohttp.web.Response(text=response) + else: + raise AttributeError(f"Invalid response type: {type(response)}") + if name is not None: + setattr(cleaner_handler, "__qualname__", name + "_handler") + + return cleaner_handler + +def make_cleaner_function(raw_function, cleaner_function, name=None): + async def cleaner_local(request, **kwargs): + canvas_response = await raw_function(request, **kwargs) + clean = cleaner_function(canvas_response) + return clean + if name is not None: + setattr(cleaner_local, "__qualname__", name) + return cleaner_local + +class BaseLMS: + def __init__(self, lms_name, endpoints, raw_ajax_function): + self.lms_name = lms_name + self.endpoints = endpoints + self.raw_ajax_function = raw_ajax_function + + def initialize_routes(self, app): + app.add_routes([ + aiohttp.web.get(f"/{self.lms_name}", lambda request: api_docs_handler(request, self.endpoints)) + ]) + + for e in self.endpoints: + function_name = f"raw_{e.name}" + raw_function = raw_access_partial(self.raw_ajax_function, remote_url=e.remote_url, name=e.name) + globals()[function_name] = raw_function + cleaners = e._cleaners() + for c in cleaners: + app.add_routes([ + aiohttp.web.get( + cleaners[c]['local_url'], + make_cleaner_handler(raw_function, cleaners[c]['function'], name=cleaners[c]['name']) + ) + ]) + globals()[cleaners[c]['name']] = make_cleaner_function( + raw_function, + cleaners[c]['function'], + name=cleaners[c]['name'] + ) + app.add_routes([ + aiohttp.web.get(e._local_url(), make_ajax_raw_handler(self.raw_ajax_function, e.remote_url)) + ]) \ No newline at end of file diff --git a/learning_observer/learning_observer/routes.py b/learning_observer/learning_observer/routes.py index 90ea5bdad..1124eb4d6 100644 --- a/learning_observer/learning_observer/routes.py +++ b/learning_observer/learning_observer/routes.py @@ -67,8 +67,7 @@ def tracemalloc_handler(request): register_static_routes(app) register_incoming_event_views(app) register_debug_routes(app) - learning_observer.google.initialize_and_register_routes(app) - learning_observer.canvas.initialize_and_register_routes(app) + register_lms_routes(app) app.add_routes([ aiohttp.web.get( @@ -167,6 +166,14 @@ def tracemalloc_handler(request): register_wsgi_routes(app) +def register_lms_routes(app): + ''' + Register routes for the various lms + ''' + learning_observer.google.initialize_google_routes(app) + learning_observer.canvas.initialize_canvas_routes(app) + + def register_debug_routes(app): ''' Handy-dandy information views, useful for debugging and development. From eb206d057329bb2c623103e095a68825fad109e7 Mon Sep 17 00:00:00 2001 From: JohnDamilola Date: Fri, 9 Aug 2024 09:22:03 -0400 Subject: [PATCH 07/20] Improve LMS Abstractions --- learning_observer/learning_observer/canvas.py | 153 +++----- learning_observer/learning_observer/google.py | 349 ++++++++---------- .../learning_observer/lms_integration.py | 6 +- 3 files changed, 221 insertions(+), 287 deletions(-) diff --git a/learning_observer/learning_observer/canvas.py b/learning_observer/learning_observer/canvas.py index 2c8979ef0..93297185e 100755 --- a/learning_observer/learning_observer/canvas.py +++ b/learning_observer/learning_observer/canvas.py @@ -1,5 +1,4 @@ import os -import json import configparser import aiohttp import aiohttp.web @@ -9,11 +8,9 @@ import learning_observer.auth import learning_observer.runtime -from learning_observer.lms_integration import Endpoint, register_cleaner, api_docs_handler, raw_access_partial, make_ajax_raw_handler, make_cleaner_handler, make_cleaner_function +from learning_observer.lms_integration import BaseLMS, Endpoint, register_cleaner -LMS = "canvas" - -ENDPOINTS = list(map(lambda x: Endpoint(*x, "", None, LMS), [ +CANVAS_ENDPOINTS = list(map(lambda x: Endpoint(*x, "", None, "canvas"), [ ("course_list", "/courses"), ("course_roster", "/courses/{courseId}/students"), ("course_work", "/courses/{courseId}/assignments"), @@ -28,10 +25,6 @@ def __init__(self, config_path='./config.ini'): self.config = configparser.ConfigParser() self.config.read(config_path) - # Check if 'SCHOOLOGY_CONFIG' section is present - if 'CANVAS_CONFIG' not in self.config: - raise KeyError("The configuration file does not contain 'CANVAS_CONFIG' section") - try: self.defaultServer = self.config['CANVAS_CONFIG']['DEFAULT_SERVER'] self.access_token = self.config['CANVAS_CONFIG']['ACCESS_TOKEN'] @@ -91,95 +84,65 @@ async def refresh_tokens(self): } return await self.api_call('POST', url, params=params, absolute_url=True) -async def raw_canvas_ajax(runtime, target_url, retry=False, **kwargs): - ''' - Make an AJAX call to Canvas, managing auth + auth. + +class CanvasLMS(BaseLMS): + def __init__(self): + super().__init__(lms_name="canvas", endpoints=CANVAS_ENDPOINTS, raw_ajax_function=self.raw_canvas_ajax) + self.canvas = Canvas() - * runtime is a Runtime class containing request information. - * target_url is typically grabbed from ENDPOINTS - * ... and we pass the named parameters - ''' - canvas = Canvas() - - params = {k: v for k, v in kwargs.items() if v is not None} - try: - response = await canvas.api_call('GET', target_url, params=params, **kwargs) - except aiohttp.ClientResponseError as e: - if e.status == 401 and retry: - new_tokens = await canvas.refresh_tokens() - if 'access_token' in new_tokens: - canvas.update_access_tokens(new_tokens['access_token']) - return await raw_canvas_ajax(runtime, target_url, retry=False, **kwargs) - raise - - print(kwargs) - return response - - -def initialize_canvas_routes(app): - ''' - - Created debug routes to pass through AJAX requests to Canvas - - Created production APIs to have access to cleaned versions of said data - - Create local function calls to call from other pieces of code within process - ''' - # Provide documentation on what we're doing - app.add_routes([ - aiohttp.web.get("/canvas", api_docs_handler) - ]) + async def raw_canvas_ajax(self, runtime, target_url, retry=False, **kwargs): + ''' + Make an AJAX call to Canvas, managing auth + auth. - for e in ENDPOINTS: - function_name = f"raw_{e.name}" - raw_function = raw_access_partial(raw_canvas_ajax, remote_url=e.remote_url, name=e.name) - globals()[function_name] = raw_function - cleaners = e._cleaners() - for c in cleaners: - app.add_routes([ - aiohttp.web.get( - cleaners[c]['local_url'], - make_cleaner_handler(raw_function, cleaners[c]['function'], name=cleaners[c]['name']) - ) - ]) - globals()[cleaners[c]['name']] = make_cleaner_function( - raw_function, - cleaners[c]['function'], - name=cleaners[c]['name'] - ) - app.add_routes([ - aiohttp.web.get(e._local_url(), make_ajax_raw_handler(raw_canvas_ajax, e.remote_url)) - ]) + * runtime is a Runtime class containing request information. + * target_url is typically grabbed from ENDPOINTS + * ... and we pass the named parameters + ''' + params = {k: v for k, v in kwargs.items() if v is not None} + try: + response = await self.canvas.api_call('GET', target_url, params=params, **kwargs) + except aiohttp.ClientResponseError as e: + if e.status == 401 and retry: + new_tokens = await self.canvas.refresh_tokens() + if 'access_token' in new_tokens: + self.canvas.update_access_tokens(new_tokens['access_token']) + return await self.raw_canvas_ajax(runtime, target_url, retry=False, **kwargs) + raise + return response -@register_cleaner("course_roster", "roster", ENDPOINTS) -def clean_course_roster(canvas_json): - students = canvas_json - students_updated = [] - for student_json in students: - canvas_id = student_json['id'] - integration_id = student_json['integration_id'] - local_id = learning_observer.auth.google_id_to_user_id(integration_id) - student = { - "course_id": "1", - "user_id": local_id, - "profile": { - "id": canvas_id, - "name": { - "given_name": student_json['name'], - "family_name": student_json['name'], - "full_name": student_json['name'] + @register_cleaner("course_roster", "roster", CANVAS_ENDPOINTS) + def clean_course_roster(canvas_json): + students = canvas_json + students_updated = [] + for student_json in students: + canvas_id = student_json['id'] + integration_id = student_json['integration_id'] + local_id = learning_observer.auth.google_id_to_user_id(integration_id) + student = { + "course_id": "1", + "user_id": local_id, + "profile": { + "id": canvas_id, + "name": { + "given_name": student_json['name'], + "family_name": student_json['name'], + "full_name": student_json['name'] + } } } - } - if 'external_ids' not in student_json: - student_json['external_ids'] = [] - student_json['external_ids'].append({"source": "canvas", "id": integration_id}) - students_updated.append(student) - return students_updated + if 'external_ids' not in student_json: + student_json['external_ids'] = [] + student_json['external_ids'].append({"source": "canvas", "id": integration_id}) + students_updated.append(student) + return students_updated -@register_cleaner("course_list", "courses", ENDPOINTS) -def clean_course_list(canvas_json): - courses = canvas_json - courses.sort(key=lambda x: x.get('name', 'ZZ')) - return courses - -if __name__ == '__main__': - output = clean_course_roster({}) - print(json.dumps(output, indent=2)) \ No newline at end of file + @register_cleaner("course_list", "courses", CANVAS_ENDPOINTS) + def clean_course_list(canvas_json): + courses = canvas_json + courses.sort(key=lambda x: x.get('name', 'ZZ')) + return courses + +canvas_lms = CanvasLMS() + +def initialize_canvas_routes(app): + canvas_lms.initialize_routes(app) \ No newline at end of file diff --git a/learning_observer/learning_observer/google.py b/learning_observer/learning_observer/google.py index b923a94df..408354806 100644 --- a/learning_observer/learning_observer/google.py +++ b/learning_observer/learning_observer/google.py @@ -36,7 +36,7 @@ import learning_observer.runtime import learning_observer.prestartup -from learning_observer.lms_integration import Endpoint, register_cleaner, api_docs_handler, raw_access_partial, make_ajax_raw_handler, make_cleaner_handler, make_cleaner_function +from learning_observer.lms_integration import BaseLMS, Endpoint, register_cleaner cache = None @@ -58,7 +58,7 @@ GOOGLE_TO_SNAKE = {field: camel_to_snake.sub('_', field).lower() for field in GOOGLE_FIELDS} -ENDPOINTS = list(map(lambda x: Endpoint(*x, "", None, LMS), [ +GOOGLE_ENDPOINTS = list(map(lambda x: Endpoint(*x, "", None, LMS), [ ("document", "https://docs.googleapis.com/v1/documents/{documentId}"), ("course_list", "https://classroom.googleapis.com/v1/courses"), ("course_roster", "https://classroom.googleapis.com/v1/courses/{courseId}/students"), @@ -72,133 +72,6 @@ ("drive_revisions", "https://www.googleapis.com/drive/v3/files/{documentId}/revisions") ])) - -async def raw_google_ajax(runtime, target_url, **kwargs): - ''' - Make an AJAX call to Google, managing auth + auth. - - * runtime is a Runtime class containing request information. - * default_url is typically grabbed from ENDPOINTS - * ... and we pass the named parameters - ''' - request = runtime.get_request() - url = target_url.format(**kwargs) - user = await learning_observer.auth.get_active_user(request) - if constants.AUTH_HEADERS not in request: - raise aiohttp.web.HTTPUnauthorized(text="Please log in") # TODO: Consistent way to flag this - - cache_key = "raw_google/" + learning_observer.auth.encode_id('session', user[constants.USER_ID]) + '/' + learning_observer.util.url_pathname(url) - if settings.feature_flag('use_google_ajax') is not None: - value = await cache[cache_key] - if value is not None: - return learning_observer.util.translate_json_keys( - json.loads(value), - GOOGLE_TO_SNAKE - ) - async with aiohttp.ClientSession(loop=request.app.loop) as client: - async with client.get(url, headers=request[constants.AUTH_HEADERS]) as resp: - response = await resp.json() - learning_observer.log_event.log_ajax(target_url, response, request) - if settings.feature_flag('use_google_ajax') is not None: - await cache.set(cache_key, json.dumps(response, indent=2)) - return learning_observer.util.translate_json_keys( - response, - GOOGLE_TO_SNAKE - ) - - -@learning_observer.prestartup.register_startup_check -def connect_to_google_cache(): - '''Setup cache for requests to the Google API. - The cache is currently only used with the `use_google_ajax` - feature flag. - ''' - if 'google_routes' not in settings.settings['feature_flags']: - return - - for key in ['save_google_ajax', 'use_google_ajax', 'save_clean_ajax', 'use_clean_ajax']: - if key in settings.settings['feature_flags']: - global cache - try: - cache = learning_observer.kvs.KVS.google_cache() - except AttributeError: - error_text = 'The google_cache KVS is not configured.\n'\ - 'Please add a `google_cache` kvs item to the `kvs` '\ - 'key in `creds.yaml`.\n'\ - '```\ngoogle_cache:\n type: filesystem\n path: ./learning_observer/static_data/google\n'\ - ' subdirs: true\n```\nOR\n'\ - '```\ngoogle_cache:\n type: redis_ephemeral\n expiry: 600\n```' - raise learning_observer.prestartup.StartupCheck("Google KVS: " + error_text) - -def initialize_google_routes(app): - ''' - - Created debug routes to pass through AJAX requests to Google - - Created production APIs to have access to cleaned versions of said data - - Create local function calls to call from other pieces of code within process - ''' - # Provide documentation on what we're doing - app.add_routes([ - aiohttp.web.get("/google", api_docs_handler) - ]) - - for e in ENDPOINTS: - function_name = f"raw_{e.name}" - raw_function = raw_access_partial(remote_url=e.remote_url, name=e.name) - globals()[function_name] = raw_function - cleaners = e._cleaners() - for c in cleaners: - app.add_routes([ - aiohttp.web.get( - cleaners[c]['local_url'], - make_cleaner_handler(raw_function, cleaners[c]['function'], name=cleaners[c]['name']) - ) - ]) - globals()[cleaners[c]['name']] = make_cleaner_function( - raw_function, - cleaners[c]['function'], - name=cleaners[c]['name'] - ) - app.add_routes([ - aiohttp.web.get(raw_google_ajax, e._local_url(), make_ajax_raw_handler(e.remote_url)) - ]) - -# Rosters -@register_cleaner("course_roster", "roster", ENDPOINTS) -def clean_course_roster(google_json): - ''' - Retrieve the roster for a course, alphabetically - ''' - students = google_json.get('students', []) - students.sort( - key=lambda x: x.get('name', {}).get('fullName', 'ZZ'), - ) - # Convert Google IDs to internal ideas (which are the same, but with a gc- prefix) - for student_json in students: - google_id = student_json['profile']['id'] - local_id = learning_observer.auth.google_id_to_user_id(google_id) - student_json[constants.USER_ID] = local_id - del student_json['profile']['id'] - - # For the present there is only one external id so we will add that directly. - if 'external_ids' not in student_json['profile']: - student_json['profile']['external_ids'] = [] - student_json['profile']['external_ids'].append({"source": "google", "id": google_id}) - return students - - -@register_cleaner("course_list", "courses", ENDPOINTS) -def clean_course_list(google_json): - ''' - Google's course list is one object deeper than we'd like, and alphabetic - sort order is nicer. This will clean it up a bit - ''' - courses = google_json.get('courses', []) - courses.sort( - key=lambda x: x.get('name', 'ZZ'), - ) - return courses - - # Google Docs def _force_text_length(text, length): ''' @@ -212,7 +85,6 @@ def _force_text_length(text, length): ''' return text[:length] + " " * (length - len(text)) - def get_error_details(error): messages = { 403: 'Student working on private document.', @@ -222,73 +94,168 @@ def get_error_details(error): message = messages.get(code, 'Unknown error.') return {'error': {'code': code, 'message': message}} +class GoogleLMS(BaseLMS): + def __init__(self): + super().__init__(lms_name="google", endpoints=GOOGLE_ENDPOINTS, raw_ajax_function=self.raw_google_ajax) + + async def raw_google_ajax(self, runtime, target_url, **kwargs): + ''' + Make an AJAX call to Google, managing auth + auth. + + * runtime is a Runtime class containing request information. + * default_url is typically grabbed from ENDPOINTS + * ... and we pass the named parameters + ''' + request = runtime.get_request() + url = target_url.format(**kwargs) + user = await learning_observer.auth.get_active_user(request) + if constants.AUTH_HEADERS not in request: + raise aiohttp.web.HTTPUnauthorized(text="Please log in") # TODO: Consistent way to flag this + + cache_key = "raw_google/" + learning_observer.auth.encode_id('session', user[constants.USER_ID]) + '/' + learning_observer.util.url_pathname(url) + if settings.feature_flag('use_google_ajax') is not None: + value = await cache[cache_key] + if value is not None: + return learning_observer.util.translate_json_keys( + json.loads(value), + GOOGLE_TO_SNAKE + ) + async with aiohttp.ClientSession(loop=request.app.loop) as client: + async with client.get(url, headers=request[constants.AUTH_HEADERS]) as resp: + response = await resp.json() + learning_observer.log_event.log_ajax(target_url, response, request) + if settings.feature_flag('use_google_ajax') is not None: + await cache.set(cache_key, json.dumps(response, indent=2)) + return learning_observer.util.translate_json_keys( + response, + GOOGLE_TO_SNAKE + ) -@register_cleaner("document", "doctext") -def extract_text_from_google_doc_json( - j, align=True, - EXTRACT_DEBUG_CHECKS=False): - ''' - Extract text from a Google Docs JSON object, ignoring formatting. - - There is an alignment issue between Google's and Python's handling - of Unicode. We can either: - * extract text faithfully (align=False) - * extract text with aligned indexes by cutting text / adding - spaces (align=True) - - This issue came up in text with a Russian flag unicode symbol - (referencing the current conflict). I tried various encodings, - and none quite matched Google 100%. - - Note that align=True doesn't necessarily give perfect local alignment - within text chunks, since we do have different lengths for something like - this flag. It does work okay globally. - ''' - # return error message for text - if 'error' in j: - return get_error_details(j['error']) - length = j['body']['content'][-1]['endIndex'] - elements = [a.get('paragraph', {}).get('elements', []) for a in j['body']['content']] - flat = sum(elements, []) - text_chunks = [f.get('textRun', {}).get('content', '') for f in flat] - if align: - lengths = [f['endIndex'] - f['startIndex'] for f in flat] - text_chunks = [_force_text_length(chunk, length) for chunk, length in zip(text_chunks, lengths)] - text = ''.join(text_chunks) - - if EXTRACT_DEBUG_CHECKS: - print("Text length versus Google length:") - print(len(text), length) - print("We expect these to be off by one, since Google seems to starts at 1 (and Python at 0)") + # Rosters + @register_cleaner("course_roster", "roster", GOOGLE_ENDPOINTS) + def clean_course_roster(google_json): + ''' + Retrieve the roster for a course, alphabetically + ''' + students = google_json.get('students', []) + students.sort( + key=lambda x: x.get('name', {}).get('fullName', 'ZZ'), + ) + # Convert Google IDs to internal ideas (which are the same, but with a gc- prefix) + for student_json in students: + google_id = student_json['profile']['id'] + local_id = learning_observer.auth.google_id_to_user_id(google_id) + student_json[constants.USER_ID] = local_id + del student_json['profile']['id'] + + # For the present there is only one external id so we will add that directly. + if 'external_ids' not in student_json['profile']: + student_json['profile']['external_ids'] = [] + student_json['profile']['external_ids'].append({"source": "google", "id": google_id}) + return students + + @register_cleaner("course_list", "courses", GOOGLE_ENDPOINTS) + def clean_course_list(google_json): + ''' + Google's course list is one object deeper than we'd like, and alphabetic + sort order is nicer. This will clean it up a bit + ''' + courses = google_json.get('courses', []) + courses.sort( + key=lambda x: x.get('name', 'ZZ'), + ) + return courses + + @register_cleaner("document", "doctext", GOOGLE_ENDPOINTS) + def extract_text_from_google_doc_json( + j, align=True, + EXTRACT_DEBUG_CHECKS=False): + ''' + Extract text from a Google Docs JSON object, ignoring formatting. + + There is an alignment issue between Google's and Python's handling + of Unicode. We can either: + * extract text faithfully (align=False) + * extract text with aligned indexes by cutting text / adding + spaces (align=True) + + This issue came up in text with a Russian flag unicode symbol + (referencing the current conflict). I tried various encodings, + and none quite matched Google 100%. + + Note that align=True doesn't necessarily give perfect local alignment + within text chunks, since we do have different lengths for something like + this flag. It does work okay globally. + ''' + # return error message for text + if 'error' in j: + return get_error_details(j['error']) + length = j['body']['content'][-1]['endIndex'] + elements = [a.get('paragraph', {}).get('elements', []) for a in j['body']['content']] + flat = sum(elements, []) + text_chunks = [f.get('textRun', {}).get('content', '') for f in flat] if align: - print - print("Offsets (these should match):") - print(list(zip(itertools.accumulate(map(len, text_chunks)), itertools.accumulate(lengths)))) - - return {'text': text} - - -@register_cleaner("coursework_submissions", "assigned_docs") -def clean_assignment_docs(google_json): - ''' - Retrieve set of documents per student associated with an assignment - ''' - student_submissions = google_json.get('studentSubmissions', []) - for student_json in student_submissions: - google_id = student_json[constants.USER_ID] - local_id = learning_observer.auth.google_id_to_user_id(google_id) - student_json[constants.USER_ID] = local_id - docs = [d['driveFile'] for d in learning_observer.util.get_nested_dict_value(student_json, 'assignmentSubmission.attachments', []) if 'driveFile' in d] - student_json['documents'] = docs - # TODO we should probably remove some of the keys provided - return student_submissions + lengths = [f['endIndex'] - f['startIndex'] for f in flat] + text_chunks = [_force_text_length(chunk, length) for chunk, length in zip(text_chunks, lengths)] + text = ''.join(text_chunks) + + if EXTRACT_DEBUG_CHECKS: + print("Text length versus Google length:") + print(len(text), length) + print("We expect these to be off by one, since Google seems to starts at 1 (and Python at 0)") + if align: + print + print("Offsets (these should match):") + print(list(zip(itertools.accumulate(map(len, text_chunks)), itertools.accumulate(lengths)))) + + return {'text': text} + + @register_cleaner("coursework_submissions", "assigned_docs", GOOGLE_ENDPOINTS) + def clean_assignment_docs(google_json): + ''' + Retrieve set of documents per student associated with an assignment + ''' + student_submissions = google_json.get('studentSubmissions', []) + for student_json in student_submissions: + google_id = student_json[constants.USER_ID] + local_id = learning_observer.auth.google_id_to_user_id(google_id) + student_json[constants.USER_ID] = local_id + docs = [d['driveFile'] for d in learning_observer.util.get_nested_dict_value(student_json, 'assignmentSubmission.attachments', []) if 'driveFile' in d] + student_json['documents'] = docs + # TODO we should probably remove some of the keys provided + return student_submissions + + @learning_observer.prestartup.register_startup_check + def connect_to_google_cache(): + '''Setup cache for requests to the Google API. + The cache is currently only used with the `use_google_ajax` + feature flag. + ''' + if 'google_routes' not in settings.settings['feature_flags']: + return + + for key in ['save_google_ajax', 'use_google_ajax', 'save_clean_ajax', 'use_clean_ajax']: + if key in settings.settings['feature_flags']: + global cache + try: + cache = learning_observer.kvs.KVS.google_cache() + except AttributeError: + error_text = 'The google_cache KVS is not configured.\n'\ + 'Please add a `google_cache` kvs item to the `kvs` '\ + 'key in `creds.yaml`.\n'\ + '```\ngoogle_cache:\n type: filesystem\n path: ./learning_observer/static_data/google\n'\ + ' subdirs: true\n```\nOR\n'\ + '```\ngoogle_cache:\n type: redis_ephemeral\n expiry: 600\n```' + raise learning_observer.prestartup.StartupCheck("Google KVS: " + error_text) + +google_lms = GoogleLMS() +def initialize_google_routes(app): + google_lms.initialize_routes(app) if __name__ == '__main__': import json import sys j = json.load(open(sys.argv[1])) - # extract_text_from_google_doc_json(j, align=False, EXTRACT_DEBUG_CHECKS=True) - # extract_text_from_google_doc_json(j, align=True, EXTRACT_DEBUG_CHECKS=True) - output = clean_assignment_docs(j) + output = google_lms.clean_assignment_docs(j) print(json.dumps(output, indent=2)) diff --git a/learning_observer/learning_observer/lms_integration.py b/learning_observer/learning_observer/lms_integration.py index f633938da..b70d415b9 100644 --- a/learning_observer/learning_observer/lms_integration.py +++ b/learning_observer/learning_observer/lms_integration.py @@ -2,6 +2,7 @@ import string import aiohttp import aiohttp.web +import learning_observer import learning_observer.runtime @@ -154,11 +155,14 @@ def initialize_routes(self, app): make_cleaner_handler(raw_function, cleaners[c]['function'], name=cleaners[c]['name']) ) ]) - globals()[cleaners[c]['name']] = make_cleaner_function( + lms_module = getattr(learning_observer, self.lms_name) + + cleaner_function = make_cleaner_function( raw_function, cleaners[c]['function'], name=cleaners[c]['name'] ) + setattr(lms_module, cleaners[c]['name'], cleaner_function) app.add_routes([ aiohttp.web.get(e._local_url(), make_ajax_raw_handler(self.raw_ajax_function, e.remote_url)) ]) \ No newline at end of file From 75d10938b94b399477413ca40f3378dce3ab7243 Mon Sep 17 00:00:00 2001 From: JohnDamilola Date: Fri, 9 Aug 2024 09:39:03 -0400 Subject: [PATCH 08/20] Fix import position --- modules/writing_observer/writing_observer/aggregator.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/modules/writing_observer/writing_observer/aggregator.py b/modules/writing_observer/writing_observer/aggregator.py index dfa727df3..6dee29e1d 100644 --- a/modules/writing_observer/writing_observer/aggregator.py +++ b/modules/writing_observer/writing_observer/aggregator.py @@ -275,6 +275,9 @@ async def update_reconstruct_reducer_with_google_api(runtime, doc_ids): We use a closure here to make use of memoization so we do not update the KVS every time we call this method. """ + + import learning_observer.google + from learning_observer.stream_analytics.fields import KeyField, KeyStateType, EventField @learning_observer.cache.async_memoization() async def fetch_doc_from_google(student, doc_id): @@ -284,8 +287,6 @@ async def fetch_doc_from_google(student, doc_id): """ if student is None or doc_id is None or len(doc_id) == 0: return None - import learning_observer.google - from learning_observer.stream_analytics.fields import KeyField, KeyStateType, EventField kvs = learning_observer.kvs.KVS() From ca4725bfeddbd1d1d699f320f18440c63fd4506c Mon Sep 17 00:00:00 2001 From: JohnDamilola Date: Thu, 22 Aug 2024 18:49:33 -0400 Subject: [PATCH 09/20] Refactor lms abstraction --- .gitignore | 1 - .../learning_observer/auth/social_sso.py | 48 +++++- learning_observer/learning_observer/canvas.py | 127 +++----------- .../learning_observer/constants.py | 1 + .../learning_observer/creds.yaml.example | 6 + learning_observer/learning_observer/google.py | 58 ++----- .../learning_observer/lms_integration.py | 162 +++++++++++++++--- .../learning_observer/settings.py | 2 +- 8 files changed, 232 insertions(+), 173 deletions(-) diff --git a/.gitignore b/.gitignore index c212cac96..1ad3eaf7a 100644 --- a/.gitignore +++ b/.gitignore @@ -38,5 +38,4 @@ package-lock.json learning_observer/learning_observer/static_data/google/ learning_observer/learning_observer/static_data/admins.yaml .ipynb_checkpoints/ -learning_observer/learning_observer/config.ini diff --git a/learning_observer/learning_observer/auth/social_sso.py b/learning_observer/learning_observer/auth/social_sso.py index 6d47d7b5a..8fb27168f 100644 --- a/learning_observer/learning_observer/auth/social_sso.py +++ b/learning_observer/learning_observer/auth/social_sso.py @@ -67,13 +67,13 @@ pmss.register_field( name="client_id", type=pmss.pmsstypes.TYPES.string, - description="The Google OAuth client ID", + description="The Google/Canvas OAuth client ID", required=True ) pmss.register_field( name="client_secret", type=pmss.pmsstypes.TYPES.string, - description="The Google OAuth client secret", + description="The Google/Canvas OAuth client secret", required=True ) pmss.register_field( @@ -83,6 +83,18 @@ 'fetch all text from current rosters.', default=False ) +pmss.register_field( + name="default_server", + type=pmss.pmsstypes.TYPES.string, + description="The Canvas OAuth default server", + required=True +) +pmss.register_field( + name="refresh_token", + type=pmss.pmsstypes.TYPES.string, + description="The Canvas OAuth refresh token", + required=True +) DEFAULT_GOOGLE_SCOPES = [ @@ -129,6 +141,8 @@ async def social_handler(request): ) user = await _google(request) + if "canvas is activated": + await _canvas(request) if constants.USER_ID in user: await learning_observer.auth.utils.update_session_user_info(request, user) @@ -210,6 +224,36 @@ async def _process_student_documents(student): await _process_student_documents(student) # TODO saved skipped doc ids somewhere? +async def _canvas(request): + ''' + Handle Canvas authorization + ''' + if 'error' in request.query: + return {} + + default_server = settings.pmss_settings.default_server(types=['lms', 'canvas_oauth']) + + url = f'https://{default_server}/login/oauth2/token' + common_params = { + "grant_type": "refresh_token", + 'client_id': settings.pmss_settings.client_id(types=['lms', 'canvas_oauth']), + 'client_secret': settings.pmss_settings.client_secret(types=['lms', 'canvas_oauth']), + "refresh_token": settings.pmss_settings.refresh_token(types=['lms', 'canvas_oauth']) + } + params = common_params.copy() + async with aiohttp.ClientSession(loop=request.app.loop) as client: + async with client.post(url, data=params) as resp: + data = await resp.json() + assert 'access_token' in data, data + + # get user profile + canvas_headers = {'Authorization': 'Bearer ' + data['access_token']} + session = await aiohttp_session.get_session(request) + session[constants.CANVAS_AUTH_HEADERS] = canvas_headers + request[constants.CANVAS_AUTH_HEADERS] = canvas_headers + session.save() + + return data async def _google(request): ''' diff --git a/learning_observer/learning_observer/canvas.py b/learning_observer/learning_observer/canvas.py index 93297185e..cc587ef21 100755 --- a/learning_observer/learning_observer/canvas.py +++ b/learning_observer/learning_observer/canvas.py @@ -1,116 +1,37 @@ -import os -import configparser -import aiohttp -import aiohttp.web +import functools +import learning_observer.auth.social_sso import learning_observer.log_event import learning_observer.util import learning_observer.auth import learning_observer.runtime +import learning_observer.lms_integration +import pmss -from learning_observer.lms_integration import BaseLMS, Endpoint, register_cleaner +pmss.register_field( + name="default_server", + type=pmss.pmsstypes.TYPES.string, + description="The Canvas OAuth default server", + required=True +) -CANVAS_ENDPOINTS = list(map(lambda x: Endpoint(*x, "", None, "canvas"), [ +LMS_NAME = "canvas" + +CANVAS_ENDPOINTS = list(map(lambda x: learning_observer.lms_integration.Endpoint(*x, "", None, LMS_NAME), [ ("course_list", "/courses"), ("course_roster", "/courses/{courseId}/students"), - ("course_work", "/courses/{courseId}/assignments"), - ("coursework_submissions", "/courses/{courseId}/assignments/{assignmentId}/submissions"), + ("course_assignments", "/courses/{courseId}/assignments"), + ("course_assignments_submissions", "/courses/{courseId}/assignments/{assignmentId}/submissions"), ])) -class Canvas: - def __init__(self, config_path='./config.ini'): - script_dir = os.path.dirname(os.path.abspath(__file__)) - config_path = os.path.join(script_dir, config_path) - - self.config = configparser.ConfigParser() - self.config.read(config_path) - - try: - self.defaultServer = self.config['CANVAS_CONFIG']['DEFAULT_SERVER'] - self.access_token = self.config['CANVAS_CONFIG']['ACCESS_TOKEN'] - self.refresh_token = self.config['CANVAS_CONFIG']['REFRESH_TOKEN'] - self.client_id = self.config['CANVAS_CONFIG']['CLIENT_ID'] - self.client_secret = self.config['CANVAS_CONFIG']['CLIENT_SECRET'] - except KeyError as e: - raise KeyError(f"Missing required configuration key: {e}") - self.default_version = 'v1' - self.defaultPerPage = 10000 - self.base_url = f'https://{self.defaultServer}/api/{self.default_version}' - - def update_access_tokens(self, access_token): - self.config['CANVAS_CONFIG']['ACCESS_TOKEN'] = access_token - self.access_token = access_token - script_dir = os.path.dirname(os.path.abspath(__file__)) - config_path = os.path.join(script_dir, './config.ini') - with open(config_path, 'w') as configfile: - self.config.write(configfile) - - async def api_call(self, method, endpoint, params=None, data=None, absolute_url=False, retry=True, **kwargs): - if absolute_url: - url = endpoint - else: - url = self.base_url + endpoint - #if params: - #url += '?' + '&'.join(f"{k}={v}" for k, v in params.items()) - - url = url.format(**kwargs) - - headers = { - 'Authorization': f'Bearer {self.access_token}', - 'Content-Type': 'application/json' - } - - async with aiohttp.ClientSession() as client: - response_func = getattr(client, method.lower()) - async with response_func(url, headers=headers, params=params, json=data) as response: - if response.status == 401 and retry: - new_tokens = await self.refresh_tokens() - if 'access_token' in new_tokens: - self.update_access_tokens(new_tokens['access_token']) - return await self.api_call(method, endpoint, params, data, absolute_url, retry=False, **kwargs) - - if response.status != 200: - response.raise_for_status() - - return await response.json() - - async def refresh_tokens(self): - url = f'https://{self.defaultServer}/login/oauth2/token' - params = { - "grant_type": "refresh_token", - "client_id": self.client_id, - "client_secret": self.client_secret, - "refresh_token": self.refresh_token - } - return await self.api_call('POST', url, params=params, absolute_url=True) +register_cleaner_with_endpoints = functools.partial(learning_observer.lms_integration.register_cleaner, endpoints=CANVAS_ENDPOINTS) -class CanvasLMS(BaseLMS): +class CanvasLMS(learning_observer.lms_integration.LMS): def __init__(self): - super().__init__(lms_name="canvas", endpoints=CANVAS_ENDPOINTS, raw_ajax_function=self.raw_canvas_ajax) - self.canvas = Canvas() - - async def raw_canvas_ajax(self, runtime, target_url, retry=False, **kwargs): - ''' - Make an AJAX call to Canvas, managing auth + auth. - - * runtime is a Runtime class containing request information. - * target_url is typically grabbed from ENDPOINTS - * ... and we pass the named parameters - ''' - params = {k: v for k, v in kwargs.items() if v is not None} - try: - response = await self.canvas.api_call('GET', target_url, params=params, **kwargs) - except aiohttp.ClientResponseError as e: - if e.status == 401 and retry: - new_tokens = await self.canvas.refresh_tokens() - if 'access_token' in new_tokens: - self.canvas.update_access_tokens(new_tokens['access_token']) - return await self.raw_canvas_ajax(runtime, target_url, retry=False, **kwargs) - raise - return response - - @register_cleaner("course_roster", "roster", CANVAS_ENDPOINTS) + super().__init__(lms_name=LMS_NAME, endpoints=CANVAS_ENDPOINTS) + + @register_cleaner_with_endpoints("course_roster", "roster") def clean_course_roster(canvas_json): students = canvas_json students_updated = [] @@ -136,12 +57,18 @@ def clean_course_roster(canvas_json): students_updated.append(student) return students_updated - @register_cleaner("course_list", "courses", CANVAS_ENDPOINTS) + @register_cleaner_with_endpoints("course_list", "courses") def clean_course_list(canvas_json): courses = canvas_json courses.sort(key=lambda x: x.get('name', 'ZZ')) return courses + @register_cleaner_with_endpoints("course_assignments", "assignments") + def clean_course_assignment_list(canvas_json): + assignments = canvas_json + assignments.sort(key=lambda x: x.get('name', 'ZZ')) + return assignments + canvas_lms = CanvasLMS() def initialize_canvas_routes(app): diff --git a/learning_observer/learning_observer/constants.py b/learning_observer/learning_observer/constants.py index 6a923793c..e0d879033 100644 --- a/learning_observer/learning_observer/constants.py +++ b/learning_observer/learning_observer/constants.py @@ -10,6 +10,7 @@ ''' # used in request headers to hold auth information AUTH_HEADERS = 'auth_headers' +CANVAS_AUTH_HEADERS = 'canvas_auth_headers' # used for storing impersonation information in session IMPERSONATING_AS = 'impersonating_as' diff --git a/learning_observer/learning_observer/creds.yaml.example b/learning_observer/learning_observer/creds.yaml.example index 33d06c722..816c9b861 100644 --- a/learning_observer/learning_observer/creds.yaml.example +++ b/learning_observer/learning_observer/creds.yaml.example @@ -101,3 +101,9 @@ modules: writing_observer: use_nlp: false openai_api_key: '' # can also be set with OPENAI_API_KEY environment variable +lms: + canvas_oauth: + default_server: {canvas-default-server} + client_id: {canvas-client-id} + client_secret: {canvas-client-secret} + refresh_token: {canvas-refresh-token} \ No newline at end of file diff --git a/learning_observer/learning_observer/google.py b/learning_observer/learning_observer/google.py index 408354806..4c65f10fb 100644 --- a/learning_observer/learning_observer/google.py +++ b/learning_observer/learning_observer/google.py @@ -24,9 +24,7 @@ import itertools import json import re - -import aiohttp -import aiohttp.web +import functools import learning_observer.constants as constants import learning_observer.settings as settings @@ -35,12 +33,11 @@ import learning_observer.auth import learning_observer.runtime import learning_observer.prestartup - -from learning_observer.lms_integration import BaseLMS, Endpoint, register_cleaner +import learning_observer.lms_integration cache = None -LMS = "google" +LMS_NAME = "google" GOOGLE_FIELDS = [ @@ -58,7 +55,7 @@ GOOGLE_TO_SNAKE = {field: camel_to_snake.sub('_', field).lower() for field in GOOGLE_FIELDS} -GOOGLE_ENDPOINTS = list(map(lambda x: Endpoint(*x, "", None, LMS), [ +GOOGLE_ENDPOINTS = list(map(lambda x: learning_observer.lms_integration.Endpoint(*x, "", None, LMS_NAME), [ ("document", "https://docs.googleapis.com/v1/documents/{documentId}"), ("course_list", "https://classroom.googleapis.com/v1/courses"), ("course_roster", "https://classroom.googleapis.com/v1/courses/{courseId}/students"), @@ -72,6 +69,8 @@ ("drive_revisions", "https://www.googleapis.com/drive/v3/files/{documentId}/revisions") ])) +register_cleaner_with_endpoints = functools.partial(learning_observer.lms_integration.register_cleaner, endpoints=GOOGLE_ENDPOINTS) + # Google Docs def _force_text_length(text, length): ''' @@ -94,45 +93,12 @@ def get_error_details(error): message = messages.get(code, 'Unknown error.') return {'error': {'code': code, 'message': message}} -class GoogleLMS(BaseLMS): +class GoogleLMS(learning_observer.lms_integration.LMS): def __init__(self): - super().__init__(lms_name="google", endpoints=GOOGLE_ENDPOINTS, raw_ajax_function=self.raw_google_ajax) - - async def raw_google_ajax(self, runtime, target_url, **kwargs): - ''' - Make an AJAX call to Google, managing auth + auth. - - * runtime is a Runtime class containing request information. - * default_url is typically grabbed from ENDPOINTS - * ... and we pass the named parameters - ''' - request = runtime.get_request() - url = target_url.format(**kwargs) - user = await learning_observer.auth.get_active_user(request) - if constants.AUTH_HEADERS not in request: - raise aiohttp.web.HTTPUnauthorized(text="Please log in") # TODO: Consistent way to flag this - - cache_key = "raw_google/" + learning_observer.auth.encode_id('session', user[constants.USER_ID]) + '/' + learning_observer.util.url_pathname(url) - if settings.feature_flag('use_google_ajax') is not None: - value = await cache[cache_key] - if value is not None: - return learning_observer.util.translate_json_keys( - json.loads(value), - GOOGLE_TO_SNAKE - ) - async with aiohttp.ClientSession(loop=request.app.loop) as client: - async with client.get(url, headers=request[constants.AUTH_HEADERS]) as resp: - response = await resp.json() - learning_observer.log_event.log_ajax(target_url, response, request) - if settings.feature_flag('use_google_ajax') is not None: - await cache.set(cache_key, json.dumps(response, indent=2)) - return learning_observer.util.translate_json_keys( - response, - GOOGLE_TO_SNAKE - ) + super().__init__(lms_name=LMS_NAME, endpoints=GOOGLE_ENDPOINTS) # Rosters - @register_cleaner("course_roster", "roster", GOOGLE_ENDPOINTS) + @register_cleaner_with_endpoints("course_roster", "roster") def clean_course_roster(google_json): ''' Retrieve the roster for a course, alphabetically @@ -154,7 +120,7 @@ def clean_course_roster(google_json): student_json['profile']['external_ids'].append({"source": "google", "id": google_id}) return students - @register_cleaner("course_list", "courses", GOOGLE_ENDPOINTS) + @register_cleaner_with_endpoints("course_list", "courses") def clean_course_list(google_json): ''' Google's course list is one object deeper than we'd like, and alphabetic @@ -166,7 +132,7 @@ def clean_course_list(google_json): ) return courses - @register_cleaner("document", "doctext", GOOGLE_ENDPOINTS) + @register_cleaner_with_endpoints("document", "doctext") def extract_text_from_google_doc_json( j, align=True, EXTRACT_DEBUG_CHECKS=False): @@ -210,7 +176,7 @@ def extract_text_from_google_doc_json( return {'text': text} - @register_cleaner("coursework_submissions", "assigned_docs", GOOGLE_ENDPOINTS) + @register_cleaner_with_endpoints("coursework_submissions", "assigned_docs") def clean_assignment_docs(google_json): ''' Retrieve set of documents per student associated with an assignment diff --git a/learning_observer/learning_observer/lms_integration.py b/learning_observer/learning_observer/lms_integration.py index b70d415b9..53ec14ede 100644 --- a/learning_observer/learning_observer/lms_integration.py +++ b/learning_observer/learning_observer/lms_integration.py @@ -1,16 +1,26 @@ +import json import recordclass import string import aiohttp import aiohttp.web +import aiohttp_session + import learning_observer import learning_observer.runtime +import learning_observer.google +import learning_observer.constants as constants +import learning_observer.settings as settings +import pmss + +pmss.register_field( + name="default_server", + type=pmss.pmsstypes.TYPES.string, + description="The Canvas OAuth default server", + required=True +) +cache = None -# These took a while to find, but many are documented here: -# https://developers.google.com/drive/api/v3/reference/ -# This list might change. Many of these contain additional (optional) parameters -# which we might add later. This is here for debugging, mostly. We'll stabilize -# APIs later. class Endpoint(recordclass.make_dataclass("Endpoint", ["name", "remote_url", "doc", "cleaners", "lms"], defaults=["", None])): def arguments(self): return extract_parameters_from_format_string(self.remote_url) @@ -44,7 +54,7 @@ def extract_parameters_from_format_string(format_string): ''' return [f[1] for f in string.Formatter().parse(format_string) if f[1] is not None] -def raw_access_partial(raw_ajax, remote_url, name=None): +def raw_access_partial(raw_ajax_function, target_url, name=None): ''' This is a helper which allows us to create a function which calls specific Google APIs. @@ -57,19 +67,19 @@ async def caller(request, **kwargs): ''' Make an AJAX request to Google ''' - return await raw_ajax(request, remote_url, **kwargs) + return await raw_ajax_function(request, target_url, **kwargs) setattr(caller, "__qualname__", name) return caller -def api_docs_handler(request, ENDPOINTS): +def api_docs_handler(endpoints): ''' Return a list of available endpoints. Eventually, we should also document available function calls ''' response = "URL Endpoints:\n\n" - for endpoint in ENDPOINTS: + for endpoint in endpoints: response += f"{endpoint._local_url()}\n" cleaners = endpoint._cleaners() for c in cleaners: @@ -77,14 +87,14 @@ def api_docs_handler(request, ENDPOINTS): response += "\n\n Globals:" return aiohttp.web.Response(text=response) -def register_cleaner(data_source, cleaner_name, ENDPOINTS): +def register_cleaner(data_source, cleaner_name, endpoints): ''' This will register a cleaner function, for export both as a web service and as a local function call. ''' def decorator(f): found = False - for endpoint in ENDPOINTS: + for endpoint in endpoints: if endpoint.name == data_source: found = True endpoint._add_cleaner( @@ -102,12 +112,12 @@ def decorator(f): return decorator -def make_ajax_raw_handler(raw_canvas_ajax, remote_url): - async def ajax_passthrough(request): - runtime = learning_observer.runtime.Runtime(request) - response = await raw_canvas_ajax(runtime, remote_url, retry=True, **request.match_info) - return aiohttp.web.json_response(response) - return ajax_passthrough +def make_ajax_raw_handler(raw_ajax_function, remote_url): + async def ajax_passthrough(request): + runtime = learning_observer.runtime.Runtime(request) + response = await raw_ajax_function(runtime, remote_url, retry=True, **request.match_info) + return aiohttp.web.json_response(response) + return ajax_passthrough def make_cleaner_handler(raw_function, cleaner_function, name=None): async def cleaner_handler(request): @@ -132,20 +142,123 @@ async def cleaner_local(request, **kwargs): setattr(cleaner_local, "__qualname__", name) return cleaner_local -class BaseLMS: - def __init__(self, lms_name, endpoints, raw_ajax_function): +async def raw_ajax(runtime, target_url, lms_name, base_url=None, **kwargs): + """ + Make an authenticated AJAX call to a specified service (e.g., Google, Canvas), handling + authorization, caching, and retries. + + Parameters: + - runtime: An instance of the Runtime class containing request information. + - lms_name: A string indicating the name of the service ('google' or 'canvas'). + - target_url: The URL endpoint to be called, with optional formatting using kwargs. + - base_url: An optional base URL for the service. If provided, it will be prefixed + to target_url. + - kwargs: Additional keyword arguments to format the target_url or control behavior + (e.g., retry). + + Returns: + - A JSON response from the requested service, with key translation if necessary. + + Raises: + - aiohttp.web.HTTPUnauthorized: If the request lacks necessary authorization. + - aiohttp.ClientResponseError: If the request fails, with special handling for 401 errors on Canvas. + """ + # Retrieve the incoming request and active user + request = runtime.get_request() + user = await learning_observer.auth.get_active_user(request) + + # Extract 'retry' flag from kwargs (defaults to True) + retry = kwargs.pop('retry', True) + + # Retrieve session and determine the appropriate headers based on the service + session = await aiohttp_session.get_session(request) + headers = { + 'google': request[constants.AUTH_HEADERS], + 'canvas': session.get(constants.CANVAS_AUTH_HEADERS) + } + + # Ensure Google requests are authenticated + if lms_name == 'google' and constants.AUTH_HEADERS not in request: + raise aiohttp.web.HTTPUnauthorized(text="Please log in") + + # Construct the full URL using the base URL if provided, otherwise use the target URL directly + url = f"{base_url}{target_url.format(**kwargs)}" if base_url else target_url.format(**kwargs) + + if base_url: + url = base_url + target_url.format(**kwargs) + else: + url = target_url.format(**kwargs) + + # Generate a unique cache key based on the service, user, and request URL + cache_key = f"raw_{lms_name}/" + learning_observer.auth.encode_id('session', user[constants.USER_ID]) + '/' + learning_observer.util.url_pathname(url) + + # Check cache and return cached response if available + if settings.feature_flag(f"use_{lms_name}_ajax") is not None: + value = await cache[cache_key] + if value is not None: + # Translate keys if the service is Google, otherwise return raw JSON + if lms_name == 'google': + return learning_observer.util.translate_json_keys( + json.loads(value), + learning_observer.google.GOOGLE_TO_SNAKE + ) + else: json.loads(value) + + # Make the actual AJAX call to the service + async with aiohttp.ClientSession(loop=request.app.loop) as client: + try: + async with client.get(url, headers=headers[lms_name]) as resp: + response = await resp.json() + # Log the AJAX request and response + learning_observer.log_event.log_ajax(target_url, response, request) + # Cache the response if the feature flag is enabled + if settings.feature_flag(f"use_{lms_name}_ajax") is not None: + await cache.set(cache_key, json.dumps(response, indent=2)) + # Translate keys if the service is Google, otherwise return raw JSON + if lms_name == 'google': + learning_observer.util.translate_json_keys( + response, + learning_observer.google.GOOGLE_TO_SNAKE + ) + else: return response + # Handle 401 errors for Canvas with an optional retry + except aiohttp.ClientResponseError as e: + if lms_name == 'canvas' and e.status == 401 and retry: + new_tokens = learning_observer.auth.social_sso._canvas() + if 'access_token' in new_tokens: + return await raw_ajax(runtime, target_url, lms_name, base_url, **kwargs) + raise + +async def raw_google_ajax(runtime, target_url, **kwargs): + return await raw_ajax(runtime, target_url, 'google', **kwargs) + +async def raw_canvas_ajax(runtime, target_url, **kwargs): + default_server = settings.pmss_settings.default_server(types=['lms', 'canvas_oauth']) + base_url = f'https://{default_server}/api/v1/' + return await raw_ajax(runtime, target_url, 'canvas', base_url, **kwargs) + + +class LMS: + def __init__(self, lms_name, endpoints): self.lms_name = lms_name self.endpoints = endpoints - self.raw_ajax_function = raw_ajax_function + self.raw_ajax_function = { + 'google': raw_google_ajax, + 'canvas': raw_canvas_ajax + } def initialize_routes(self, app): app.add_routes([ - aiohttp.web.get(f"/{self.lms_name}", lambda request: api_docs_handler(request, self.endpoints)) + aiohttp.web.get(f"/{self.lms_name}", lambda _: api_docs_handler(self.endpoints)) ]) for e in self.endpoints: function_name = f"raw_{e.name}" - raw_function = raw_access_partial(self.raw_ajax_function, remote_url=e.remote_url, name=e.name) + raw_function = raw_access_partial( + raw_ajax_function = self.raw_ajax_function[self.lms_name], + target_url = e.remote_url, + name = e.name + ) globals()[function_name] = raw_function cleaners = e._cleaners() for c in cleaners: @@ -164,5 +277,8 @@ def initialize_routes(self, app): ) setattr(lms_module, cleaners[c]['name'], cleaner_function) app.add_routes([ - aiohttp.web.get(e._local_url(), make_ajax_raw_handler(self.raw_ajax_function, e.remote_url)) + aiohttp.web.get(e._local_url(), make_ajax_raw_handler( + self.raw_ajax_function[self.lms_name], + e.remote_url + )) ]) \ No newline at end of file diff --git a/learning_observer/learning_observer/settings.py b/learning_observer/learning_observer/settings.py index f9140b3b5..1c1de6aa6 100644 --- a/learning_observer/learning_observer/settings.py +++ b/learning_observer/learning_observer/settings.py @@ -223,7 +223,7 @@ def initialized(): # Not all of these are guaranteed to work on every branch of the codebase. -AVAILABLE_FEATURE_FLAGS = ['uvloop', 'watchdog', 'auth_headers_page', 'merkle', 'save_google_ajax', 'use_google_ajax'] +AVAILABLE_FEATURE_FLAGS = ['uvloop', 'watchdog', 'auth_headers_page', 'merkle', 'save_google_ajax', 'use_google_ajax', 'use_canvas_ajax'] def feature_flag(flag): From 5a1c9758a0410f6f9eeb850f20c4b5c38294f25f Mon Sep 17 00:00:00 2001 From: JohnDamilola Date: Fri, 23 Aug 2024 08:47:43 -0400 Subject: [PATCH 10/20] Fix authorization error handling --- .../learning_observer/auth/social_sso.py | 18 +++++++++++-- learning_observer/learning_observer/canvas.py | 2 -- .../learning_observer/lms_integration.py | 25 +++++++++++-------- 3 files changed, 30 insertions(+), 15 deletions(-) diff --git a/learning_observer/learning_observer/auth/social_sso.py b/learning_observer/learning_observer/auth/social_sso.py index 8fb27168f..0d54c07e1 100644 --- a/learning_observer/learning_observer/auth/social_sso.py +++ b/learning_observer/learning_observer/auth/social_sso.py @@ -96,6 +96,18 @@ required=True ) +pmss.register_field( + name='source', + type='roster_source', + description='Source to use for student class rosters. This can be\n'\ + '`all`: aggregate all available students into a single class\n'\ + '`test`: use sample course and student files\n'\ + '`filesystem`: read rosters defined on filesystem\n'\ + '`google_api`: fetch from Google API\n'\ + '`canvas`: fetch from Canvas API', + required=True +) + DEFAULT_GOOGLE_SCOPES = [ 'https://www.googleapis.com/auth/userinfo.profile', @@ -141,7 +153,10 @@ async def social_handler(request): ) user = await _google(request) - if "canvas is activated": + + # Check roster data source to obtain relevant authorization + roster_source = settings.pmss_settings.source(types=['roster_data']) + if roster_source == 'canvas': await _canvas(request) if constants.USER_ID in user: @@ -251,7 +266,6 @@ async def _canvas(request): session = await aiohttp_session.get_session(request) session[constants.CANVAS_AUTH_HEADERS] = canvas_headers request[constants.CANVAS_AUTH_HEADERS] = canvas_headers - session.save() return data diff --git a/learning_observer/learning_observer/canvas.py b/learning_observer/learning_observer/canvas.py index cc587ef21..26fdb83fc 100755 --- a/learning_observer/learning_observer/canvas.py +++ b/learning_observer/learning_observer/canvas.py @@ -1,10 +1,8 @@ import functools -import learning_observer.auth.social_sso import learning_observer.log_event import learning_observer.util import learning_observer.auth -import learning_observer.runtime import learning_observer.lms_integration import pmss diff --git a/learning_observer/learning_observer/lms_integration.py b/learning_observer/learning_observer/lms_integration.py index 53ec14ede..64140fb09 100644 --- a/learning_observer/learning_observer/lms_integration.py +++ b/learning_observer/learning_observer/lms_integration.py @@ -65,7 +65,7 @@ def raw_access_partial(raw_ajax_function, target_url, name=None): ''' async def caller(request, **kwargs): ''' - Make an AJAX request to Google + Make an AJAX request to LMS ''' return await raw_ajax_function(request, target_url, **kwargs) setattr(caller, "__qualname__", name) @@ -135,8 +135,8 @@ async def cleaner_handler(request): def make_cleaner_function(raw_function, cleaner_function, name=None): async def cleaner_local(request, **kwargs): - canvas_response = await raw_function(request, **kwargs) - clean = cleaner_function(canvas_response) + lms_response = await raw_function(request, **kwargs) + clean = cleaner_function(lms_response) return clean if name is not None: setattr(cleaner_local, "__qualname__", name) @@ -167,8 +167,8 @@ async def raw_ajax(runtime, target_url, lms_name, base_url=None, **kwargs): request = runtime.get_request() user = await learning_observer.auth.get_active_user(request) - # Extract 'retry' flag from kwargs (defaults to True) - retry = kwargs.pop('retry', True) + # Extract 'retry' flag from kwargs (defaults to False) + retry = kwargs.pop('retry', False) # Retrieve session and determine the appropriate headers based on the service session = await aiohttp_session.get_session(request) @@ -182,8 +182,6 @@ async def raw_ajax(runtime, target_url, lms_name, base_url=None, **kwargs): raise aiohttp.web.HTTPUnauthorized(text="Please log in") # Construct the full URL using the base URL if provided, otherwise use the target URL directly - url = f"{base_url}{target_url.format(**kwargs)}" if base_url else target_url.format(**kwargs) - if base_url: url = base_url + target_url.format(**kwargs) else: @@ -202,13 +200,15 @@ async def raw_ajax(runtime, target_url, lms_name, base_url=None, **kwargs): json.loads(value), learning_observer.google.GOOGLE_TO_SNAKE ) - else: json.loads(value) + else: + return json.loads(value) # Make the actual AJAX call to the service async with aiohttp.ClientSession(loop=request.app.loop) as client: try: async with client.get(url, headers=headers[lms_name]) as resp: response = await resp.json() + # Log the AJAX request and response learning_observer.log_event.log_ajax(target_url, response, request) # Cache the response if the feature flag is enabled @@ -216,15 +216,17 @@ async def raw_ajax(runtime, target_url, lms_name, base_url=None, **kwargs): await cache.set(cache_key, json.dumps(response, indent=2)) # Translate keys if the service is Google, otherwise return raw JSON if lms_name == 'google': - learning_observer.util.translate_json_keys( + return learning_observer.util.translate_json_keys( response, learning_observer.google.GOOGLE_TO_SNAKE ) - else: return response + else: + resp.raise_for_status() + return response # Handle 401 errors for Canvas with an optional retry except aiohttp.ClientResponseError as e: if lms_name == 'canvas' and e.status == 401 and retry: - new_tokens = learning_observer.auth.social_sso._canvas() + new_tokens = await learning_observer.auth.social_sso._canvas(request) if 'access_token' in new_tokens: return await raw_ajax(runtime, target_url, lms_name, base_url, **kwargs) raise @@ -235,6 +237,7 @@ async def raw_google_ajax(runtime, target_url, **kwargs): async def raw_canvas_ajax(runtime, target_url, **kwargs): default_server = settings.pmss_settings.default_server(types=['lms', 'canvas_oauth']) base_url = f'https://{default_server}/api/v1/' + kwargs.setdefault('retry', True) return await raw_ajax(runtime, target_url, 'canvas', base_url, **kwargs) From 3bd8f5f7b933f56f24ae4576896b2c9fff8a28cd Mon Sep 17 00:00:00 2001 From: JohnDamilola Date: Fri, 23 Aug 2024 08:50:07 -0400 Subject: [PATCH 11/20] Streamline LMS setup based on the settings --- learning_observer/learning_observer/routes.py | 43 ++++++++++++++++--- 1 file changed, 38 insertions(+), 5 deletions(-) diff --git a/learning_observer/learning_observer/routes.py b/learning_observer/learning_observer/routes.py index 1124eb4d6..9837f6c1f 100644 --- a/learning_observer/learning_observer/routes.py +++ b/learning_observer/learning_observer/routes.py @@ -33,6 +33,20 @@ from learning_observer.log_event import debug_log, startup_state from learning_observer.utility_handlers import * +import pmss + + +pmss.register_field( + name='source', + type='roster_source', + description='Source to use for student class rosters. This can be\n'\ + '`all`: aggregate all available students into a single class\n'\ + '`test`: use sample course and student files\n'\ + '`filesystem`: read rosters defined on filesystem\n'\ + '`google_api`: fetch from Google API\n'\ + '`canvas`: fetch from Canvas API', + required=True +) def add_routes(app): @@ -165,13 +179,32 @@ def tracemalloc_handler(request): # and figuring stuff out, this feels safest to put last. register_wsgi_routes(app) - + def register_lms_routes(app): - ''' - Register routes for the various lms - ''' + """ + Register routes for the various Learning Management Systems (LMS). + + This function maps each LMS to its corresponding route initialization function + and registers the routes based on the active roster data source. + + Parameters: + - app: An instance of aiohttp.web.Application where the routes will be registered. + """ + + # Define a mapping of LMS names to their respective route initialization functions + LMS_ROUTES_MAP = { + 'google': learning_observer.google.initialize_google_routes, + 'canvas': learning_observer.canvas.initialize_canvas_routes + } + + # Retrieve the active roster data source from the settings (e.g., 'google', 'canvas'). + roster_source = settings.pmss_settings.source(types=['roster_data']) + + # Call the corresponding route initialization function for the active LMS. + if roster_source in LMS_ROUTES_MAP: + LMS_ROUTES_MAP[roster_source](app) + learning_observer.google.initialize_google_routes(app) - learning_observer.canvas.initialize_canvas_routes(app) def register_debug_routes(app): From 449e834c8571d6c9ea90ed605e097063c11591ae Mon Sep 17 00:00:00 2001 From: JohnDamilola Date: Fri, 23 Aug 2024 08:56:21 -0400 Subject: [PATCH 12/20] Fix text alignment --- learning_observer/learning_observer/google.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/learning_observer/learning_observer/google.py b/learning_observer/learning_observer/google.py index 4c65f10fb..b40d57faa 100644 --- a/learning_observer/learning_observer/google.py +++ b/learning_observer/learning_observer/google.py @@ -161,8 +161,16 @@ def extract_text_from_google_doc_json( flat = sum(elements, []) text_chunks = [f.get('textRun', {}).get('content', '') for f in flat] if align: - lengths = [f['endIndex'] - f['startIndex'] for f in flat] - text_chunks = [_force_text_length(chunk, length) for chunk, length in zip(text_chunks, lengths)] + for f in flat: + text = f.get('textRun', {}).get('content', None) + if text != None: + length = f['endIndex'] - f['startIndex'] + text_chunks.append(_force_text_length(text, length)) + else: + for f in flat: + text = f.get('textRun', {}).get('content', None) + if text != None: + text_chunks.append(text) text = ''.join(text_chunks) if EXTRACT_DEBUG_CHECKS: From 06a75b39f34be3fb14651d4d504e7f7a64ab8106 Mon Sep 17 00:00:00 2001 From: JohnDamilola Date: Fri, 23 Aug 2024 08:56:35 -0400 Subject: [PATCH 13/20] Add missing import --- modules/writing_observer/writing_observer/aggregator.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/writing_observer/writing_observer/aggregator.py b/modules/writing_observer/writing_observer/aggregator.py index 6dee29e1d..8d378d07d 100644 --- a/modules/writing_observer/writing_observer/aggregator.py +++ b/modules/writing_observer/writing_observer/aggregator.py @@ -339,6 +339,8 @@ async def fetch_doc_from_google(student): :return: The text of the latest document """ import learning_observer.google + from learning_observer.stream_analytics.fields import KeyField, KeyStateType, EventField + kvs = learning_observer.kvs.KVS() From 70ab5a2d1ff54dd9e183b8e78aaf7b79807cad8e Mon Sep 17 00:00:00 2001 From: JohnDamilola Date: Wed, 28 Aug 2024 08:54:25 -0400 Subject: [PATCH 14/20] Refactor code abstractions --- .../learning_observer/auth/social_sso.py | 30 +++++++++----- learning_observer/learning_observer/canvas.py | 7 ---- .../learning_observer/creds.yaml.example | 3 +- .../learning_observer/lms_integration.py | 12 +++--- .../learning_observer/rosters.py | 41 +++++++++++-------- learning_observer/learning_observer/routes.py | 14 ------- 6 files changed, 53 insertions(+), 54 deletions(-) diff --git a/learning_observer/learning_observer/auth/social_sso.py b/learning_observer/learning_observer/auth/social_sso.py index 0d54c07e1..5b8a046d8 100644 --- a/learning_observer/learning_observer/auth/social_sso.py +++ b/learning_observer/learning_observer/auth/social_sso.py @@ -84,9 +84,9 @@ default=False ) pmss.register_field( - name="default_server", + name="token_uri", type=pmss.pmsstypes.TYPES.string, - description="The Canvas OAuth default server", + description="The Canvas OAuth token uri", required=True ) pmss.register_field( @@ -154,10 +154,7 @@ async def social_handler(request): user = await _google(request) - # Check roster data source to obtain relevant authorization - roster_source = settings.pmss_settings.source(types=['roster_data']) - if roster_source == 'canvas': - await _canvas(request) + await handle_lms_request_authorization(request) if constants.USER_ID in user: await learning_observer.auth.utils.update_session_user_info(request, user) @@ -172,6 +169,20 @@ async def social_handler(request): return aiohttp.web.HTTPFound(url) +async def handle_lms_request_authorization(request): + """ + Handles the authorization of the specified Learning Management System (LMS) + by checking the roster data source and delegating the request to the appropriate handler + based on the data source type. + """ + # Retrieve the data source type for roster data from the PMSS settings + roster_source = settings.pmss_settings.source(types=['roster_data']) + + # Handle the request depending on the roster source + if roster_source == 'canvas': + await _canvas(request) + + async def _store_teacher_info_for_background_process(id, request): '''HACK this code stores 2 pieces of information when teacher logs in with a social handlers. @@ -246,16 +257,15 @@ async def _canvas(request): if 'error' in request.query: return {} - default_server = settings.pmss_settings.default_server(types=['lms', 'canvas_oauth']) + token_uri = settings.pmss_settings.token_uri(types=['lms', 'canvas_oauth']) + url = token_uri - url = f'https://{default_server}/login/oauth2/token' - common_params = { + params = { "grant_type": "refresh_token", 'client_id': settings.pmss_settings.client_id(types=['lms', 'canvas_oauth']), 'client_secret': settings.pmss_settings.client_secret(types=['lms', 'canvas_oauth']), "refresh_token": settings.pmss_settings.refresh_token(types=['lms', 'canvas_oauth']) } - params = common_params.copy() async with aiohttp.ClientSession(loop=request.app.loop) as client: async with client.post(url, data=params) as resp: data = await resp.json() diff --git a/learning_observer/learning_observer/canvas.py b/learning_observer/learning_observer/canvas.py index 26fdb83fc..356674f55 100755 --- a/learning_observer/learning_observer/canvas.py +++ b/learning_observer/learning_observer/canvas.py @@ -4,14 +4,7 @@ import learning_observer.util import learning_observer.auth import learning_observer.lms_integration -import pmss -pmss.register_field( - name="default_server", - type=pmss.pmsstypes.TYPES.string, - description="The Canvas OAuth default server", - required=True -) LMS_NAME = "canvas" diff --git a/learning_observer/learning_observer/creds.yaml.example b/learning_observer/learning_observer/creds.yaml.example index 816c9b861..0bc3e4070 100644 --- a/learning_observer/learning_observer/creds.yaml.example +++ b/learning_observer/learning_observer/creds.yaml.example @@ -103,7 +103,8 @@ modules: openai_api_key: '' # can also be set with OPENAI_API_KEY environment variable lms: canvas_oauth: - default_server: {canvas-default-server} + lms_api: {canvas-lms-api} + token_uri: {canvas-token-uri} client_id: {canvas-client-id} client_secret: {canvas-client-secret} refresh_token: {canvas-refresh-token} \ No newline at end of file diff --git a/learning_observer/learning_observer/lms_integration.py b/learning_observer/learning_observer/lms_integration.py index 64140fb09..babbb4dc0 100644 --- a/learning_observer/learning_observer/lms_integration.py +++ b/learning_observer/learning_observer/lms_integration.py @@ -13,9 +13,9 @@ import pmss pmss.register_field( - name="default_server", + name="lms_api", type=pmss.pmsstypes.TYPES.string, - description="The Canvas OAuth default server", + description="The Canvas Base API URL", required=True ) @@ -190,8 +190,9 @@ async def raw_ajax(runtime, target_url, lms_name, base_url=None, **kwargs): # Generate a unique cache key based on the service, user, and request URL cache_key = f"raw_{lms_name}/" + learning_observer.auth.encode_id('session', user[constants.USER_ID]) + '/' + learning_observer.util.url_pathname(url) + cache_flag = f"use_{lms_name}_ajax" # Check cache and return cached response if available - if settings.feature_flag(f"use_{lms_name}_ajax") is not None: + if settings.feature_flag(cache_flag) is not None: value = await cache[cache_key] if value is not None: # Translate keys if the service is Google, otherwise return raw JSON @@ -212,7 +213,7 @@ async def raw_ajax(runtime, target_url, lms_name, base_url=None, **kwargs): # Log the AJAX request and response learning_observer.log_event.log_ajax(target_url, response, request) # Cache the response if the feature flag is enabled - if settings.feature_flag(f"use_{lms_name}_ajax") is not None: + if settings.feature_flag(cache_flag) is not None: await cache.set(cache_key, json.dumps(response, indent=2)) # Translate keys if the service is Google, otherwise return raw JSON if lms_name == 'google': @@ -235,8 +236,7 @@ async def raw_google_ajax(runtime, target_url, **kwargs): return await raw_ajax(runtime, target_url, 'google', **kwargs) async def raw_canvas_ajax(runtime, target_url, **kwargs): - default_server = settings.pmss_settings.default_server(types=['lms', 'canvas_oauth']) - base_url = f'https://{default_server}/api/v1/' + base_url = settings.pmss_settings.lms_api(types=['lms', 'canvas_oauth']) kwargs.setdefault('retry', True) return await raw_ajax(runtime, target_url, 'canvas', base_url, **kwargs) diff --git a/learning_observer/learning_observer/rosters.py b/learning_observer/learning_observer/rosters.py index 8d24fb971..4832a7684 100644 --- a/learning_observer/learning_observer/rosters.py +++ b/learning_observer/learning_observer/rosters.py @@ -362,12 +362,10 @@ def init(): ) elif roster_source in ['test', 'filesystem']: ajax = synthetic_ajax - elif roster_source in ["google_api"]: + elif roster_source in ["google_api", "canvas"]: ajax = combined_ajax elif roster_source in ["all"]: ajax = all_ajax - elif roster_source in ["canvas"]: - ajax = combined_ajax else: raise learning_observer.prestartup.StartupCheck( "Settings file `roster_data` element should have `source` field\n" @@ -415,13 +413,18 @@ async def courselist(request): ''' List all of the courses a teacher manages: Helper ''' - # New code - if settings.pmss_settings.source(types=['roster_data']) in ["google_api"]: - runtime = learning_observer.runtime.Runtime(request) - return await learning_observer.google.courses(runtime) - elif settings.pmss_settings.source(types=['roster_data']) in ["canvas"]: - runtime = learning_observer.runtime.Runtime(request) - return await learning_observer.canvas.courses(runtime) + + # A map of LMSes to their respective handler functions + lms_map = { + "google_api": learning_observer.google.courses, + "canvas": learning_observer.canvas.courses + } + + runtime = learning_observer.runtime.Runtime(request) + + roster_source = settings.pmss_settings.source(types=['roster_data']) + if roster_source in lms_map: + return await lms_map[roster_source](runtime) # Legacy code course_list = await ajax( @@ -463,12 +466,18 @@ async def courseroster(request, course_id): ''' List all of the students in a course: Helper ''' - if settings.pmss_settings.source(types=['roster_data']) in ["google_api"]: - runtime = learning_observer.runtime.Runtime(request) - return await learning_observer.google.roster(runtime, courseId=course_id) - elif settings.pmss_settings.source(types=['roster_data']) in ["canvas"]: - runtime = learning_observer.runtime.Runtime(request) - return await learning_observer.canvas.roster(runtime, courseId=course_id) + + # A map of LMSes to their respective handler functions + lms_map = { + "google_api": learning_observer.google.roster, + "canvas": learning_observer.canvas.roster + } + + runtime = learning_observer.runtime.Runtime(request) + + roster_source = settings.pmss_settings.source(types=['roster_data']) + if roster_source in lms_map: + return await lms_map[roster_source](runtime, courseId=course_id) roster = await ajax( request, diff --git a/learning_observer/learning_observer/routes.py b/learning_observer/learning_observer/routes.py index 9837f6c1f..d2f8633eb 100644 --- a/learning_observer/learning_observer/routes.py +++ b/learning_observer/learning_observer/routes.py @@ -33,20 +33,6 @@ from learning_observer.log_event import debug_log, startup_state from learning_observer.utility_handlers import * -import pmss - - -pmss.register_field( - name='source', - type='roster_source', - description='Source to use for student class rosters. This can be\n'\ - '`all`: aggregate all available students into a single class\n'\ - '`test`: use sample course and student files\n'\ - '`filesystem`: read rosters defined on filesystem\n'\ - '`google_api`: fetch from Google API\n'\ - '`canvas`: fetch from Canvas API', - required=True -) def add_routes(app): From 1594ac931b7bb90264576ae94307e6de7443e3bb Mon Sep 17 00:00:00 2001 From: JohnDamilola Date: Wed, 28 Aug 2024 12:30:22 -0400 Subject: [PATCH 15/20] Refactor code abstractions --- docs/config.md | 2 +- .../learning_observer/auth/social_sso.py | 38 +++++++++---------- learning_observer/learning_observer/canvas.py | 7 ++-- .../learning_observer/constants.py | 4 ++ .../learning_observer/creds.yaml.example | 2 +- learning_observer/learning_observer/google.py | 4 +- .../learning_observer/lms_integration.py | 16 ++++---- .../learning_observer/rosters.py | 18 ++++----- learning_observer/learning_observer/routes.py | 5 ++- 9 files changed, 50 insertions(+), 46 deletions(-) diff --git a/docs/config.md b/docs/config.md index 84f9ea673..878395527 100644 --- a/docs/config.md +++ b/docs/config.md @@ -138,7 +138,7 @@ kvs: The `roster_data` section configures the source of roster data. -- `source`: The source of roster data, such as 'filesystem', 'google_api', 'all', or 'test'. +- `source`: The source of roster data, such as 'filesystem', 'google', 'all', or 'test'. Example: diff --git a/learning_observer/learning_observer/auth/social_sso.py b/learning_observer/learning_observer/auth/social_sso.py index 5b8a046d8..7dee70f44 100644 --- a/learning_observer/learning_observer/auth/social_sso.py +++ b/learning_observer/learning_observer/auth/social_sso.py @@ -96,18 +96,6 @@ required=True ) -pmss.register_field( - name='source', - type='roster_source', - description='Source to use for student class rosters. This can be\n'\ - '`all`: aggregate all available students into a single class\n'\ - '`test`: use sample course and student files\n'\ - '`filesystem`: read rosters defined on filesystem\n'\ - '`google_api`: fetch from Google API\n'\ - '`canvas`: fetch from Canvas API', - required=True -) - DEFAULT_GOOGLE_SCOPES = [ 'https://www.googleapis.com/auth/userinfo.profile', @@ -154,7 +142,7 @@ async def social_handler(request): user = await _google(request) - await handle_lms_request_authorization(request) + await _set_lms_header_information(request) if constants.USER_ID in user: await learning_observer.auth.utils.update_session_user_info(request, user) @@ -169,18 +157,26 @@ async def social_handler(request): return aiohttp.web.HTTPFound(url) -async def handle_lms_request_authorization(request): +async def determine_roster_source(request): """ - Handles the authorization of the specified Learning Management System (LMS) - by checking the roster data source and delegating the request to the appropriate handler - based on the data source type. + Retrieve the data source type for roster data from the PMSS settings """ - # Retrieve the data source type for roster data from the PMSS settings roster_source = settings.pmss_settings.source(types=['roster_data']) - + return roster_source + +async def _set_lms_header_information(request, roster_source): + """ + Handles the authorization of the specified Learning Management System (LMS) + based on the roster data source and delegating the request to the appropriate handler + based on the data source type. + """ + lms_map = { + constants.CANVAS: _canvas + } + # Handle the request depending on the roster source - if roster_source == 'canvas': - await _canvas(request) + if roster_source in lms_map: + return await lms_map[roster_source](request) async def _store_teacher_info_for_background_process(id, request): diff --git a/learning_observer/learning_observer/canvas.py b/learning_observer/learning_observer/canvas.py index 356674f55..1c0418af4 100755 --- a/learning_observer/learning_observer/canvas.py +++ b/learning_observer/learning_observer/canvas.py @@ -4,9 +4,10 @@ import learning_observer.util import learning_observer.auth import learning_observer.lms_integration +import learning_observer.constants as constants -LMS_NAME = "canvas" +LMS_NAME = constants.CANVAS CANVAS_ENDPOINTS = list(map(lambda x: learning_observer.lms_integration.Endpoint(*x, "", None, LMS_NAME), [ ("course_list", "/courses"), @@ -44,7 +45,7 @@ def clean_course_roster(canvas_json): } if 'external_ids' not in student_json: student_json['external_ids'] = [] - student_json['external_ids'].append({"source": "canvas", "id": integration_id}) + student_json['external_ids'].append({"source": constants.CANVAS, "id": integration_id}) students_updated.append(student) return students_updated @@ -63,4 +64,4 @@ def clean_course_assignment_list(canvas_json): canvas_lms = CanvasLMS() def initialize_canvas_routes(app): - canvas_lms.initialize_routes(app) \ No newline at end of file + canvas_lms.initialize_routes(app) diff --git a/learning_observer/learning_observer/constants.py b/learning_observer/learning_observer/constants.py index e0d879033..070c29b20 100644 --- a/learning_observer/learning_observer/constants.py +++ b/learning_observer/learning_observer/constants.py @@ -18,3 +18,7 @@ USER = 'user' # common user id reference for user object USER_ID = 'user_id' + +# used to identify LMSes +GOOGLE = 'google' +CANVAS = 'canvas' diff --git a/learning_observer/learning_observer/creds.yaml.example b/learning_observer/learning_observer/creds.yaml.example index 0bc3e4070..7dd0f9b79 100644 --- a/learning_observer/learning_observer/creds.yaml.example +++ b/learning_observer/learning_observer/creds.yaml.example @@ -60,7 +60,7 @@ kvs: type: redis_ephemeral expiry: 60 roster_data: - source: all # Can be set to google_api, all, test, or filesystem + source: all # Can be set to google, all, test, or filesystem aio: # User session; used for log-ins. session_secret: {unique-aio-session-key} # This should be a unique secret key for YOUR deployment session_max_age: 3600 # In seconds. This may be short for auth dev (e.g. <5 mins), intermediate for deploy (a few hours?), and long for e.g. testing other parts of the system (or set to null, for lifetime of the browser) diff --git a/learning_observer/learning_observer/google.py b/learning_observer/learning_observer/google.py index b40d57faa..f0960b3a4 100644 --- a/learning_observer/learning_observer/google.py +++ b/learning_observer/learning_observer/google.py @@ -37,7 +37,7 @@ cache = None -LMS_NAME = "google" +LMS_NAME = constants.GOOGLE GOOGLE_FIELDS = [ @@ -117,7 +117,7 @@ def clean_course_roster(google_json): # For the present there is only one external id so we will add that directly. if 'external_ids' not in student_json['profile']: student_json['profile']['external_ids'] = [] - student_json['profile']['external_ids'].append({"source": "google", "id": google_id}) + student_json['profile']['external_ids'].append({"source": constants.GOOGLE, "id": google_id}) return students @register_cleaner_with_endpoints("course_list", "courses") diff --git a/learning_observer/learning_observer/lms_integration.py b/learning_observer/learning_observer/lms_integration.py index babbb4dc0..5f4cd269d 100644 --- a/learning_observer/learning_observer/lms_integration.py +++ b/learning_observer/learning_observer/lms_integration.py @@ -173,8 +173,8 @@ async def raw_ajax(runtime, target_url, lms_name, base_url=None, **kwargs): # Retrieve session and determine the appropriate headers based on the service session = await aiohttp_session.get_session(request) headers = { - 'google': request[constants.AUTH_HEADERS], - 'canvas': session.get(constants.CANVAS_AUTH_HEADERS) + constants.GOOGLE: request[constants.AUTH_HEADERS], + constants.CANVAS: session.get(constants.CANVAS_AUTH_HEADERS) } # Ensure Google requests are authenticated @@ -221,24 +221,26 @@ async def raw_ajax(runtime, target_url, lms_name, base_url=None, **kwargs): response, learning_observer.google.GOOGLE_TO_SNAKE ) + # Return response for other LMSes else: + # Raise an exception for non-successful HTTP responses resp.raise_for_status() return response # Handle 401 errors for Canvas with an optional retry except aiohttp.ClientResponseError as e: - if lms_name == 'canvas' and e.status == 401 and retry: + if lms_name == constants.CANVAS and e.status == 401 and retry: new_tokens = await learning_observer.auth.social_sso._canvas(request) if 'access_token' in new_tokens: return await raw_ajax(runtime, target_url, lms_name, base_url, **kwargs) raise async def raw_google_ajax(runtime, target_url, **kwargs): - return await raw_ajax(runtime, target_url, 'google', **kwargs) + return await raw_ajax(runtime, target_url, constants.GOOGLE, **kwargs) async def raw_canvas_ajax(runtime, target_url, **kwargs): base_url = settings.pmss_settings.lms_api(types=['lms', 'canvas_oauth']) kwargs.setdefault('retry', True) - return await raw_ajax(runtime, target_url, 'canvas', base_url, **kwargs) + return await raw_ajax(runtime, target_url, constants.CANVAS, base_url, **kwargs) class LMS: @@ -246,8 +248,8 @@ def __init__(self, lms_name, endpoints): self.lms_name = lms_name self.endpoints = endpoints self.raw_ajax_function = { - 'google': raw_google_ajax, - 'canvas': raw_canvas_ajax + constants.GOOGLE: raw_google_ajax, + constants.CANVAS: raw_canvas_ajax } def initialize_routes(self, app): diff --git a/learning_observer/learning_observer/rosters.py b/learning_observer/learning_observer/rosters.py index 4832a7684..266b80911 100644 --- a/learning_observer/learning_observer/rosters.py +++ b/learning_observer/learning_observer/rosters.py @@ -88,7 +88,7 @@ COURSE_URL = 'https://classroom.googleapis.com/v1/courses' ROSTER_URL = 'https://classroom.googleapis.com/v1/courses/{courseid}/students' -pmss.parser('roster_source', parent='string', choices=['google_api', 'all', 'test', 'canvas', 'filesystem'], transform=None) +pmss.parser('roster_source', parent='string', choices=['google', 'all', 'test', 'canvas', 'filesystem'], transform=None) pmss.register_field( name='source', type='roster_source', @@ -96,7 +96,7 @@ '`all`: aggregate all available students into a single class\n'\ '`test`: use sample course and student files\n'\ '`filesystem`: read rosters defined on filesystem\n'\ - '`google_api`: fetch from Google API\n'\ + '`google`: fetch from Google API\n'\ '`canvas`: fetch from Canvas API', required=True ) @@ -179,7 +179,7 @@ def adjust_external_gc_ids(resp_json): student_json[constants.USER_ID] = google_id # For the present there is only one external id so we will add that directly. - ext_ids = [{"source": "google", "id": google_id}] + ext_ids = [{"source": constants.GOOGLE, "id": google_id}] student_profile['external_ids'] = ext_ids @@ -362,7 +362,7 @@ def init(): ) elif roster_source in ['test', 'filesystem']: ajax = synthetic_ajax - elif roster_source in ["google_api", "canvas"]: + elif roster_source in [constants.GOOGLE, constants.CANVAS]: ajax = combined_ajax elif roster_source in ["all"]: ajax = all_ajax @@ -371,7 +371,7 @@ def init(): "Settings file `roster_data` element should have `source` field\n" "set to either:\n" " test (retrieve from files courses.json and students.json)\n" - " google_api (retrieve roster data from Google)\n" + " google (retrieve roster data from Google)\n" " canvas (retrieve roster data from Canvas)\n" " filesystem (retrieve roster data from file system hierarchy\n" " all (retrieve roster data as all students)" @@ -416,8 +416,8 @@ async def courselist(request): # A map of LMSes to their respective handler functions lms_map = { - "google_api": learning_observer.google.courses, - "canvas": learning_observer.canvas.courses + constants.GOOGLE: learning_observer.google.courses, + constants.CANVAS: learning_observer.canvas.courses } runtime = learning_observer.runtime.Runtime(request) @@ -469,8 +469,8 @@ async def courseroster(request, course_id): # A map of LMSes to their respective handler functions lms_map = { - "google_api": learning_observer.google.roster, - "canvas": learning_observer.canvas.roster + constants.GOOGLE: learning_observer.google.roster, + constants.CANVAS: learning_observer.canvas.roster } runtime = learning_observer.runtime.Runtime(request) diff --git a/learning_observer/learning_observer/routes.py b/learning_observer/learning_observer/routes.py index d2f8633eb..c49f8a7dc 100644 --- a/learning_observer/learning_observer/routes.py +++ b/learning_observer/learning_observer/routes.py @@ -29,6 +29,7 @@ import learning_observer.paths as paths import learning_observer.settings as settings +import learning_observer.constants as constants from learning_observer.log_event import debug_log, startup_state @@ -179,8 +180,8 @@ def register_lms_routes(app): # Define a mapping of LMS names to their respective route initialization functions LMS_ROUTES_MAP = { - 'google': learning_observer.google.initialize_google_routes, - 'canvas': learning_observer.canvas.initialize_canvas_routes + constants.GOOGLE: learning_observer.google.initialize_google_routes, + constants.CANVAS: learning_observer.canvas.initialize_canvas_routes } # Retrieve the active roster data source from the settings (e.g., 'google', 'canvas'). From c679533392abd4d8df3872210613233a88e079e7 Mon Sep 17 00:00:00 2001 From: JohnDamilola Date: Thu, 29 Aug 2024 13:32:11 -0400 Subject: [PATCH 16/20] Refactor code abstractions --- docs/config.md | 2 +- .../learning_observer/auth/handlers.py | 10 ++++++++-- .../learning_observer/auth/social_sso.py | 4 +++- .../learning_observer/creds.yaml.example | 2 +- .../learning_observer/lms_integration.py | 13 ++++++------- learning_observer/learning_observer/rosters.py | 8 ++++---- learning_observer/learning_observer/routes.py | 18 +----------------- 7 files changed, 24 insertions(+), 33 deletions(-) diff --git a/docs/config.md b/docs/config.md index 878395527..84f9ea673 100644 --- a/docs/config.md +++ b/docs/config.md @@ -138,7 +138,7 @@ kvs: The `roster_data` section configures the source of roster data. -- `source`: The source of roster data, such as 'filesystem', 'google', 'all', or 'test'. +- `source`: The source of roster data, such as 'filesystem', 'google_api', 'all', or 'test'. Example: diff --git a/learning_observer/learning_observer/auth/handlers.py b/learning_observer/learning_observer/auth/handlers.py index 4506afdaa..ebe6f586e 100644 --- a/learning_observer/learning_observer/auth/handlers.py +++ b/learning_observer/learning_observer/auth/handlers.py @@ -55,8 +55,14 @@ async def user_from_session(request): ''' session = await aiohttp_session.get_session(request) session_user = session.get(constants.USER, None) - if constants.AUTH_HEADERS in session: - request[constants.AUTH_HEADERS] = session[constants.AUTH_HEADERS] + + header_keys = [constants.AUTH_HEADERS, constants.CANVAS_AUTH_HEADERS] + + # Set headers in the request if they exist in the session + for key in header_keys: + if key in session: + request[key] = session[key] + return session_user diff --git a/learning_observer/learning_observer/auth/social_sso.py b/learning_observer/learning_observer/auth/social_sso.py index 7dee70f44..99f7fed4b 100644 --- a/learning_observer/learning_observer/auth/social_sso.py +++ b/learning_observer/learning_observer/auth/social_sso.py @@ -141,8 +141,10 @@ async def social_handler(request): ) user = await _google(request) + + roster_source = await determine_roster_source(request) - await _set_lms_header_information(request) + await _set_lms_header_information(request, roster_source) if constants.USER_ID in user: await learning_observer.auth.utils.update_session_user_info(request, user) diff --git a/learning_observer/learning_observer/creds.yaml.example b/learning_observer/learning_observer/creds.yaml.example index 7dd0f9b79..0bc3e4070 100644 --- a/learning_observer/learning_observer/creds.yaml.example +++ b/learning_observer/learning_observer/creds.yaml.example @@ -60,7 +60,7 @@ kvs: type: redis_ephemeral expiry: 60 roster_data: - source: all # Can be set to google, all, test, or filesystem + source: all # Can be set to google_api, all, test, or filesystem aio: # User session; used for log-ins. session_secret: {unique-aio-session-key} # This should be a unique secret key for YOUR deployment session_max_age: 3600 # In seconds. This may be short for auth dev (e.g. <5 mins), intermediate for deploy (a few hours?), and long for e.g. testing other parts of the system (or set to null, for lifetime of the browser) diff --git a/learning_observer/learning_observer/lms_integration.py b/learning_observer/learning_observer/lms_integration.py index 5f4cd269d..7087ebd0f 100644 --- a/learning_observer/learning_observer/lms_integration.py +++ b/learning_observer/learning_observer/lms_integration.py @@ -170,15 +170,14 @@ async def raw_ajax(runtime, target_url, lms_name, base_url=None, **kwargs): # Extract 'retry' flag from kwargs (defaults to False) retry = kwargs.pop('retry', False) - # Retrieve session and determine the appropriate headers based on the service - session = await aiohttp_session.get_session(request) + # mapping to determine the appropriate headers based on the service headers = { - constants.GOOGLE: request[constants.AUTH_HEADERS], - constants.CANVAS: session.get(constants.CANVAS_AUTH_HEADERS) + constants.GOOGLE: request.get(constants.AUTH_HEADERS), + constants.CANVAS: request.get(constants.CANVAS_AUTH_HEADERS) } # Ensure Google requests are authenticated - if lms_name == 'google' and constants.AUTH_HEADERS not in request: + if lms_name == constants.GOOGLE and constants.AUTH_HEADERS not in request: raise aiohttp.web.HTTPUnauthorized(text="Please log in") # Construct the full URL using the base URL if provided, otherwise use the target URL directly @@ -196,7 +195,7 @@ async def raw_ajax(runtime, target_url, lms_name, base_url=None, **kwargs): value = await cache[cache_key] if value is not None: # Translate keys if the service is Google, otherwise return raw JSON - if lms_name == 'google': + if lms_name == constants.GOOGLE: return learning_observer.util.translate_json_keys( json.loads(value), learning_observer.google.GOOGLE_TO_SNAKE @@ -216,7 +215,7 @@ async def raw_ajax(runtime, target_url, lms_name, base_url=None, **kwargs): if settings.feature_flag(cache_flag) is not None: await cache.set(cache_key, json.dumps(response, indent=2)) # Translate keys if the service is Google, otherwise return raw JSON - if lms_name == 'google': + if lms_name == constants.GOOGLE: return learning_observer.util.translate_json_keys( response, learning_observer.google.GOOGLE_TO_SNAKE diff --git a/learning_observer/learning_observer/rosters.py b/learning_observer/learning_observer/rosters.py index 266b80911..a5f65a6ce 100644 --- a/learning_observer/learning_observer/rosters.py +++ b/learning_observer/learning_observer/rosters.py @@ -88,7 +88,7 @@ COURSE_URL = 'https://classroom.googleapis.com/v1/courses' ROSTER_URL = 'https://classroom.googleapis.com/v1/courses/{courseid}/students' -pmss.parser('roster_source', parent='string', choices=['google', 'all', 'test', 'canvas', 'filesystem'], transform=None) +pmss.parser('roster_source', parent='string', choices=['google_api', 'all', 'test', 'canvas', 'filesystem'], transform=None) pmss.register_field( name='source', type='roster_source', @@ -96,7 +96,7 @@ '`all`: aggregate all available students into a single class\n'\ '`test`: use sample course and student files\n'\ '`filesystem`: read rosters defined on filesystem\n'\ - '`google`: fetch from Google API\n'\ + '`google_api`: fetch from Google API\n'\ '`canvas`: fetch from Canvas API', required=True ) @@ -362,7 +362,7 @@ def init(): ) elif roster_source in ['test', 'filesystem']: ajax = synthetic_ajax - elif roster_source in [constants.GOOGLE, constants.CANVAS]: + elif roster_source in ['google_api', constants.CANVAS]: ajax = combined_ajax elif roster_source in ["all"]: ajax = all_ajax @@ -371,7 +371,7 @@ def init(): "Settings file `roster_data` element should have `source` field\n" "set to either:\n" " test (retrieve from files courses.json and students.json)\n" - " google (retrieve roster data from Google)\n" + " google_api (retrieve roster data from Google)\n" " canvas (retrieve roster data from Canvas)\n" " filesystem (retrieve roster data from file system hierarchy\n" " all (retrieve roster data as all students)" diff --git a/learning_observer/learning_observer/routes.py b/learning_observer/learning_observer/routes.py index c49f8a7dc..18a832aea 100644 --- a/learning_observer/learning_observer/routes.py +++ b/learning_observer/learning_observer/routes.py @@ -170,28 +170,12 @@ def tracemalloc_handler(request): def register_lms_routes(app): """ Register routes for the various Learning Management Systems (LMS). - - This function maps each LMS to its corresponding route initialization function - and registers the routes based on the active roster data source. Parameters: - app: An instance of aiohttp.web.Application where the routes will be registered. """ - - # Define a mapping of LMS names to their respective route initialization functions - LMS_ROUTES_MAP = { - constants.GOOGLE: learning_observer.google.initialize_google_routes, - constants.CANVAS: learning_observer.canvas.initialize_canvas_routes - } - - # Retrieve the active roster data source from the settings (e.g., 'google', 'canvas'). - roster_source = settings.pmss_settings.source(types=['roster_data']) - - # Call the corresponding route initialization function for the active LMS. - if roster_source in LMS_ROUTES_MAP: - LMS_ROUTES_MAP[roster_source](app) - learning_observer.google.initialize_google_routes(app) + learning_observer.canvas.initialize_canvas_routes(app) def register_debug_routes(app): From fe20d06f710fefd0dd6533de7df0541ee04d78c3 Mon Sep 17 00:00:00 2001 From: JohnDamilola Date: Thu, 29 Aug 2024 14:12:21 -0400 Subject: [PATCH 17/20] Fix imports --- learning_observer/learning_observer/lms_integration.py | 1 - modules/writing_observer/writing_observer/aggregator.py | 6 +----- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/learning_observer/learning_observer/lms_integration.py b/learning_observer/learning_observer/lms_integration.py index 7087ebd0f..ebeac5749 100644 --- a/learning_observer/learning_observer/lms_integration.py +++ b/learning_observer/learning_observer/lms_integration.py @@ -3,7 +3,6 @@ import string import aiohttp import aiohttp.web -import aiohttp_session import learning_observer import learning_observer.runtime diff --git a/modules/writing_observer/writing_observer/aggregator.py b/modules/writing_observer/writing_observer/aggregator.py index 8d378d07d..6a81b422a 100644 --- a/modules/writing_observer/writing_observer/aggregator.py +++ b/modules/writing_observer/writing_observer/aggregator.py @@ -275,9 +275,6 @@ async def update_reconstruct_reducer_with_google_api(runtime, doc_ids): We use a closure here to make use of memoization so we do not update the KVS every time we call this method. """ - - import learning_observer.google - from learning_observer.stream_analytics.fields import KeyField, KeyStateType, EventField @learning_observer.cache.async_memoization() async def fetch_doc_from_google(student, doc_id): @@ -287,6 +284,7 @@ async def fetch_doc_from_google(student, doc_id): """ if student is None or doc_id is None or len(doc_id) == 0: return None + import learning_observer.google kvs = learning_observer.kvs.KVS() @@ -339,8 +337,6 @@ async def fetch_doc_from_google(student): :return: The text of the latest document """ import learning_observer.google - from learning_observer.stream_analytics.fields import KeyField, KeyStateType, EventField - kvs = learning_observer.kvs.KVS() From 5bb359ea702d83fa6034f5fdee03887a8a9c1ced Mon Sep 17 00:00:00 2001 From: JohnDamilola Date: Sat, 31 Aug 2024 21:58:04 -0400 Subject: [PATCH 18/20] Add documentation for Canvas LMS integration --- docs/lms_integrations/canvas.md | 107 ++++++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 docs/lms_integrations/canvas.md diff --git a/docs/lms_integrations/canvas.md b/docs/lms_integrations/canvas.md new file mode 100644 index 000000000..2f15f3038 --- /dev/null +++ b/docs/lms_integrations/canvas.md @@ -0,0 +1,107 @@ +## Canvas LMS Documentation: +Reference: https://canvas.instructure.com/doc/api/file.oauth.html + +This guide will walk you through the process of obtaining the `client_id`, `client_secret`, and `refresh_token` for interacting with the Canvas LMS API. These credentials are essential for making authenticated API requests to Canvas. + +### Prerequisites + +- You need to have administrator access to the Canvas LMS instance. + +### Steps to Obtain the `client_id` and `client_secret` + +1. **Log in to Your Canvas LMS Account**: + - Go to your Canvas LMS instance and log in with your administrator credentials. + +2. **Navigate to the Developer Keys Section**: + - From the Canvas dashboard, click on the **Admin** panel located on the left-hand side. + - Select the specific account (usually your institution's name) where you want to manage developer keys. + - Scroll down and click on **Developer Keys** in the left-hand menu under the **Settings** section. + +3. **Create a New Developer Key**: + - In the Developer Keys section, click the **+ Developer Key** button at the top-right corner. + - Choose **API Key** from the dropdown menu. + +4. **Fill Out the Developer Key Details**: + - **Name**: Enter a name for the Developer Key (e.g., "My Canvas API Integration"). + - **Owner's Email**: Enter administrator's email. + - **Redirect URIs**: Provide the redirect URI that will handle OAuth callbacks. This is typically a URL on your institution server where you handle OAuth responses. + +5. **Save and Enable the Developer Key**: + - After filling out the required information, click **Save Key**. + - Ensure the key is **enabled** by toggling the switch next to your newly created key. + +6. **Obtain the `client_id` and `client_secret`**: + - After saving, your `client_id` and `client_secret` will be displayed in the list of developer keys. + - **Client ID**: This is usually displayed as a numeric value in the details column. + - **Client Secret**: Click on the `show key` button and it will display the `client_secret`. + +### Steps to Obtain the `refresh_token` + +1. **Redirect User to Canvas Authorization Endpoint**: + - To obtain the `refresh_token`, you need to perform an OAuth flow. + - Direct the user to the Canvas OAuth authorization endpoint: + ``` + https://canvas.instructure.com/login/oauth2/auth?client_id=YOUR_CLIENT_ID&response_type=code&redirect_uri=YOUR_REDIRECT_URI + ``` + - Replace `YOUR_CLIENT_ID` with the `client_id` obtained earlier and `YOUR_REDIRECT_URI` with the redirect URI you configured. + +2. **User Authorizes the Application**: + - The user will be prompted to log in (if not already logged in) and authorize the application to access their Canvas data. + +3. **Handle the Authorization Code**: + - After the user authorizes the application, they will be redirected to the `redirect_uri` you provided, with an authorization `code` appended as a query parameter. + - Example: `https://your-redirect-uri.com?code=AUTHORIZATION_CODE` + +4. **Exchange the Authorization Code for a Refresh Token**: + - Use the authorization `code` to request an access token and refresh token by making a POST request to the Canvas token endpoint: + ``` + POST https://canvas.instructure.com/login/oauth2/token + ``` + - Include the following parameters in the request body: + - `client_id`: Your Canvas `client_id` + - `client_secret`: Your Canvas `client_secret` + - `redirect_uri`: Your `redirect_uri` used in the authorization request + - `code`: The authorization code you received + - `grant_type`: Set this to `authorization_code` + + - Example of the POST request in `curl`: + ```bash + curl -X POST https://canvas.instructure.com/login/oauth2/token \ + -F 'client_id=YOUR_CLIENT_ID' \ + -F 'client_secret=YOUR_CLIENT_SECRET' \ + -F 'redirect_uri=YOUR_REDIRECT_URI' \ + -F 'code=AUTHORIZATION_CODE' \ + -F 'grant_type=authorization_code' + ``` + +5. **Extract the Refresh Token**: + - The response to the token request will include an `access_token`, a `refresh_token`, and other token information. + - **Refresh Token**: This token can be used to obtain new access tokens without requiring the user to re-authorize. + +### Example JSON Response from Token Request + +```json +{ + "access_token": "ACCESS_TOKEN", + "token_type": "Bearer", + "expires_in": 3600, + "refresh_token": "REFRESH_TOKEN", + "user": { + "id": 12345, + "name": "John Doe", + "sortable_name": "Doe, John", + "short_name": "John" + } +} +``` + +- **`refresh_token`**: The value you will need to store securely for future use. + +### Important Notes + +- **Security**: The `client_id`, `client_secret`, and `refresh_token` should be stored securely. Do not expose them in client-side code or public repositories. +- **Token Expiration**: The `access_token` typically expires after a short period (e.g., 1 hour). The `refresh_token` does not expire as quickly and can be used to obtain new `access_token`s. + +### Conclusion + +By following these steps, you will obtain the necessary credentials (`client_id`, `client_secret`, and `refresh_token`) to interact with the Canvas LMS API programmatically. These credentials are essential for making authenticated requests to access and manage Canvas resources through the API. \ No newline at end of file From f40360904b225853f90a1499e8b334d7f932b068 Mon Sep 17 00:00:00 2001 From: JohnDamilola Date: Tue, 15 Oct 2024 15:15:40 -0400 Subject: [PATCH 19/20] Add docstrings and code cleanups --- .../learning_observer/auth/social_sso.py | 17 +- learning_observer/learning_observer/canvas.py | 2 - .../learning_observer/lms_integration.py | 241 ++++++++++++++++-- 3 files changed, 220 insertions(+), 40 deletions(-) diff --git a/learning_observer/learning_observer/auth/social_sso.py b/learning_observer/learning_observer/auth/social_sso.py index 99f7fed4b..6d7f83895 100644 --- a/learning_observer/learning_observer/auth/social_sso.py +++ b/learning_observer/learning_observer/auth/social_sso.py @@ -140,9 +140,9 @@ async def social_handler(request): "We only handle Google logins. Non-google Provider" ) - user = await _google(request) + user = await _handle_google_authorization(request) - roster_source = await determine_roster_source(request) + roster_source = settings.pmss_settings.source(types=['roster_data']) await _set_lms_header_information(request, roster_source) @@ -159,13 +159,6 @@ async def social_handler(request): return aiohttp.web.HTTPFound(url) -async def determine_roster_source(request): - """ - Retrieve the data source type for roster data from the PMSS settings - """ - roster_source = settings.pmss_settings.source(types=['roster_data']) - return roster_source - async def _set_lms_header_information(request, roster_source): """ Handles the authorization of the specified Learning Management System (LMS) @@ -173,7 +166,7 @@ async def _set_lms_header_information(request, roster_source): based on the data source type. """ lms_map = { - constants.CANVAS: _canvas + constants.CANVAS: _handle_canvas_authorization } # Handle the request depending on the roster source @@ -248,7 +241,7 @@ async def _process_student_documents(student): await _process_student_documents(student) # TODO saved skipped doc ids somewhere? -async def _canvas(request): +async def _handle_canvas_authorization(request): ''' Handle Canvas authorization ''' @@ -277,7 +270,7 @@ async def _canvas(request): return data -async def _google(request): +async def _handle_google_authorization(request): ''' Handle Google login ''' diff --git a/learning_observer/learning_observer/canvas.py b/learning_observer/learning_observer/canvas.py index 1c0418af4..46d73cadf 100755 --- a/learning_observer/learning_observer/canvas.py +++ b/learning_observer/learning_observer/canvas.py @@ -1,7 +1,5 @@ import functools -import learning_observer.log_event -import learning_observer.util import learning_observer.auth import learning_observer.lms_integration import learning_observer.constants as constants diff --git a/learning_observer/learning_observer/lms_integration.py b/learning_observer/learning_observer/lms_integration.py index ebeac5749..15508a71d 100644 --- a/learning_observer/learning_observer/lms_integration.py +++ b/learning_observer/learning_observer/lms_integration.py @@ -21,10 +21,35 @@ cache = None class Endpoint(recordclass.make_dataclass("Endpoint", ["name", "remote_url", "doc", "cleaners", "lms"], defaults=["", None])): + """ + The Endpoint class represents an API endpoint, allowing for parameter extraction, + URL construction, and cleaner (function) management. + + Attributes: + name (str): The name of the endpoint. + remote_url (str): The remote URL of the endpoint, which may contain parameters. + doc (str): Documentation or description of the endpoint. + cleaners (dict): A dictionary of cleaner functions associated with the endpoint. + lms (str): The learning management system (LMS) that the endpoint belongs to. + """ + def arguments(self): + """ + Extracts the parameters from the remote URL. + + Returns: + list: A list of parameters extracted from the remote_url. + """ return extract_parameters_from_format_string(self.remote_url) def _local_url(self): + """ + Constructs the local URL based on the LMS, endpoint name, and any parameters. + + Returns: + str: The constructed local URL in the format "/{lms}/{name}/{parameters}". + If there are no parameters, the URL will be "/{lms}/{name}". + """ parameters = "}/{".join(self.arguments()) base_url = f"/{self.lms}/{self.name}" if len(parameters) == 0: @@ -33,6 +58,15 @@ def _local_url(self): return base_url + "/{" + parameters + "}" def _add_cleaner(self, name, cleaner): + """ + Adds a cleaner function to the endpoint, assigning it a name. If the cleaner + doesn't have a local URL, one is generated. + + Args: + name (str): The name to associate with the cleaner. + cleaner (dict): The cleaner function to be added, optionally containing + additional metadata such as its local URL. + """ if self.cleaners is None: self.cleaners = dict() self.cleaners[name] = cleaner @@ -40,6 +74,12 @@ def _add_cleaner(self, name, cleaner): cleaner['local_url'] = self._local_url + "/" + name def _cleaners(self): + """ + Retrieves the list of cleaner functions associated with the endpoint. + + Returns: + list: A list of cleaner functions, or an empty list if no cleaners exist. + """ if self.cleaners is None: return [] else: @@ -50,33 +90,59 @@ def extract_parameters_from_format_string(format_string): Extracts parameters from a format string. E.g. >>> ("hello {hi} my {bye}")] ['hi', 'bye'] + + Args: + format_string (str): The format string containing parameters enclosed in braces. + + Returns: + list: A list of parameter names extracted from the format string. ''' return [f[1] for f in string.Formatter().parse(format_string) if f[1] is not None] def raw_access_partial(raw_ajax_function, target_url, name=None): ''' - This is a helper which allows us to create a function which calls specific - Google APIs. + Creates an asynchronous function that calls a specific LMS API. - To test this, try: + This helper function allows you to wrap an AJAX function to easily + call a specific API endpoint. - print(await raw_document(request, documentId="some_google_doc_id")) + Args: + raw_ajax_function (callable): The function to be called for making the AJAX request. + target_url (str): The target URL for the API call. + name (str, optional): The name to assign to the created function. + + Returns: + callable: An asynchronous function that can be called to perform the AJAX request. ''' - async def caller(request, **kwargs): + async def ajax_caller(request, **kwargs): ''' Make an AJAX request to LMS + + Args: + request: The incoming request object. + **kwargs: Additional keyword arguments to pass to the raw AJAX function. + + Returns: + The response from the raw AJAX function. ''' return await raw_ajax_function(request, target_url, **kwargs) - setattr(caller, "__qualname__", name) + setattr(ajax_caller, "__qualname__", name) - return caller + return ajax_caller def api_docs_handler(endpoints): ''' - Return a list of available endpoints. - + Returns a list of available endpoints in a human-readable format. + Eventually, we should also document available function calls + + Args: + endpoints (list): A list of Endpoint objects to document. + + Returns: + aiohttp.web.Response: A response object containing the documentation of endpoints. ''' + response = "URL Endpoints:\n\n" for endpoint in endpoints: response += f"{endpoint._local_url()}\n" @@ -88,10 +154,21 @@ def api_docs_handler(endpoints): def register_cleaner(data_source, cleaner_name, endpoints): ''' - This will register a cleaner function, for export both as a web service + Registers a cleaner function, allowing it to be exported both as a web service and as a local function call. + + Args: + data_source (str): The name of the data source to associate with the cleaner. + cleaner_name (str): The name of the cleaner function to register. + endpoints (list): A list of Endpoint objects to search for the data source. + + Returns: + callable: A decorator for registering the cleaner function. + + Raises: + AttributeError: If the data source is not found in the endpoints. ''' - def decorator(f): + def add_cleaner(f): found = False for endpoint in endpoints: if endpoint.name == data_source: @@ -109,34 +186,108 @@ def decorator(f): raise AttributeError(f"Data source {data_source} invalid; not found in endpoints.") return f - return decorator + return add_cleaner def make_ajax_raw_handler(raw_ajax_function, remote_url): + ''' + Creates an AJAX passthrough handler that calls a raw AJAX function. + + This function handles requests and passes them to the specified raw AJAX function, + returning the response as a JSON response. + + Args: + raw_ajax_function (callable): The raw AJAX function to call. + remote_url (str): The URL to which the AJAX request is sent. + + Returns: + callable: An asynchronous function that handles AJAX requests. + ''' async def ajax_passthrough(request): + ''' + Handle the AJAX request by calling the raw AJAX function. + + Args: + request: The incoming request object. + + Returns: + aiohttp.web.json_response: A JSON response containing the result of the AJAX function. + ''' runtime = learning_observer.runtime.Runtime(request) response = await raw_ajax_function(runtime, remote_url, retry=True, **request.match_info) return aiohttp.web.json_response(response) return ajax_passthrough def make_cleaner_handler(raw_function, cleaner_function, name=None): + ''' + Creates a handler for the cleaner function. + + This function will process the input from the raw function, apply the cleaner, + and return the cleaned response. + + Args: + raw_function (callable): The raw function to call. + cleaner_function (callable): The function to clean the response from the raw function. + name (str, optional): The name to assign to the created function. + + Returns: + callable: An asynchronous function that handles requests and cleans the responses. + ''' async def cleaner_handler(request): + ''' + Handle the request by applying the cleaner function to the raw function's response. + + Args: + request: The incoming request object. + + Returns: + aiohttp.web.json_response: A JSON response containing the cleaned data. + ''' + # Call the raw function with the request and match_info as parameters response = cleaner_function(await raw_function(request, **request.match_info)) + + # Determine the response type and return appropriately if isinstance(response, dict) or isinstance(response, list): - return aiohttp.web.json_response(response) + return aiohttp.web.json_response(response) # Return JSON response for dict or list elif isinstance(response, str): - return aiohttp.web.Response(text=response) + return aiohttp.web.Response(text=response) # Return plain text response if it's a string else: - raise AttributeError(f"Invalid response type: {type(response)}") + raise AttributeError(f"Invalid response type: {type(response)}") # Handle unexpected response types if name is not None: setattr(cleaner_handler, "__qualname__", name + "_handler") return cleaner_handler def make_cleaner_function(raw_function, cleaner_function, name=None): + """ + Creates a cleaner function that processes the output of a raw function. + + This function wraps a raw function and a cleaner function, allowing the cleaner + to be applied to the response of the raw function. + + Args: + raw_function (callable): The function that makes the raw API call. + cleaner_function (callable): The function that cleans the response. + name (str, optional): The name to assign to the created cleaner function. + + Returns: + callable: An asynchronous cleaner function that calls the raw function + and processes its output with the cleaner function. + """ async def cleaner_local(request, **kwargs): + """ + Handles the request, calls the raw function, and applies the cleaner function. + + Args: + request: The incoming request object. + **kwargs: Additional keyword arguments for the raw function. + + Returns: + The cleaned response from the cleaner function. + """ lms_response = await raw_function(request, **kwargs) clean = cleaner_function(lms_response) return clean + if name is not None: setattr(cleaner_local, "__qualname__", name) return cleaner_local @@ -227,22 +378,43 @@ async def raw_ajax(runtime, target_url, lms_name, base_url=None, **kwargs): # Handle 401 errors for Canvas with an optional retry except aiohttp.ClientResponseError as e: if lms_name == constants.CANVAS and e.status == 401 and retry: - new_tokens = await learning_observer.auth.social_sso._canvas(request) + new_tokens = await learning_observer.auth.social_sso._handle_canvas_authorization(request) if 'access_token' in new_tokens: return await raw_ajax(runtime, target_url, lms_name, base_url, **kwargs) raise +# Abstract raw_ajax for each LMS to specify their different arguments + async def raw_google_ajax(runtime, target_url, **kwargs): + """Make an authenticated AJAX call to the Google API.""" return await raw_ajax(runtime, target_url, constants.GOOGLE, **kwargs) async def raw_canvas_ajax(runtime, target_url, **kwargs): + """Make an authenticated AJAX call to the Canvas API.""" base_url = settings.pmss_settings.lms_api(types=['lms', 'canvas_oauth']) + # This is used to request the access token again in order to retry the ajax call one more time kwargs.setdefault('retry', True) return await raw_ajax(runtime, target_url, constants.CANVAS, base_url, **kwargs) class LMS: + """ + The LMS class represents a Learning Management System, encapsulating + the necessary information and methods for API interactions. + + Attributes: + lms_name (str): The name of the LMS (e.g., 'google', 'canvas'). + endpoints (list): A list of Endpoint objects that represent the API endpoints. + raw_ajax_function (dict): A dictionary mapping LMS names to their respective AJAX functions. + """ def __init__(self, lms_name, endpoints): + """ + Initializes the LMS instance with the specified name and endpoints. + + Args: + lms_name (str): The name of the LMS. + endpoints (list): A list of Endpoint objects. + """ self.lms_name = lms_name self.endpoints = endpoints self.raw_ajax_function = { @@ -251,37 +423,54 @@ def __init__(self, lms_name, endpoints): } def initialize_routes(self, app): + """ + Initializes the API routes for the specified LMS within the given web application. + + This method sets up the endpoint routes and associates them with their corresponding + handler functions. + + Args: + app: An instance of the aiohttp web application to which routes will be added. + """ + + # Add the main API documentation route app.add_routes([ aiohttp.web.get(f"/{self.lms_name}", lambda _: api_docs_handler(self.endpoints)) ]) + # Iterate through the endpoints to set up routes for each one for e in self.endpoints: - function_name = f"raw_{e.name}" + function_name = f"raw_{e.name}" # Construct the function name for the raw AJAX function raw_function = raw_access_partial( - raw_ajax_function = self.raw_ajax_function[self.lms_name], - target_url = e.remote_url, - name = e.name + raw_ajax_function = self.raw_ajax_function[self.lms_name], # Get the appropriate raw AJAX function + target_url = e.remote_url, # Use the endpoint's remote URL + name = e.name # Set the name for the function ) - globals()[function_name] = raw_function + globals()[function_name] = raw_function # Register the raw function globally + + # Add routes for each cleaner associated with the endpoint cleaners = e._cleaners() for c in cleaners: app.add_routes([ aiohttp.web.get( - cleaners[c]['local_url'], - make_cleaner_handler(raw_function, cleaners[c]['function'], name=cleaners[c]['name']) + cleaners[c]['local_url'], # The local URL for the cleaner + make_cleaner_handler(raw_function, cleaners[c]['function'], name=cleaners[c]['name']) # Handler for the cleaner ) ]) - lms_module = getattr(learning_observer, self.lms_name) + lms_module = getattr(learning_observer, self.lms_name) # Get the module for the LMS + # Create the cleaner function and set it in the LMS module cleaner_function = make_cleaner_function( raw_function, cleaners[c]['function'], name=cleaners[c]['name'] ) setattr(lms_module, cleaners[c]['name'], cleaner_function) + + # Add the main route for the endpoint app.add_routes([ aiohttp.web.get(e._local_url(), make_ajax_raw_handler( - self.raw_ajax_function[self.lms_name], - e.remote_url + self.raw_ajax_function[self.lms_name], # The raw AJAX function for the LMS + e.remote_url # The endpoint's remote URL )) ]) \ No newline at end of file From 5705757c980ae48e769f5474a54671b787639f84 Mon Sep 17 00:00:00 2001 From: JohnDamilola Date: Tue, 15 Oct 2024 15:41:47 -0400 Subject: [PATCH 20/20] Clean up code with linting --- .../learning_observer/auth/handlers.py | 3 - .../learning_observer/auth/social_sso.py | 19 ++-- learning_observer/learning_observer/canvas.py | 10 ++- learning_observer/learning_observer/google.py | 14 ++- .../learning_observer/lms_integration.py | 89 +++++++++++-------- .../learning_observer/rosters.py | 22 ++--- learning_observer/learning_observer/routes.py | 4 +- .../learning_observer/settings.py | 7 +- 8 files changed, 93 insertions(+), 75 deletions(-) diff --git a/learning_observer/learning_observer/auth/handlers.py b/learning_observer/learning_observer/auth/handlers.py index ebe6f586e..11575ed6e 100644 --- a/learning_observer/learning_observer/auth/handlers.py +++ b/learning_observer/learning_observer/auth/handlers.py @@ -55,14 +55,11 @@ async def user_from_session(request): ''' session = await aiohttp_session.get_session(request) session_user = session.get(constants.USER, None) - header_keys = [constants.AUTH_HEADERS, constants.CANVAS_AUTH_HEADERS] - # Set headers in the request if they exist in the session for key in header_keys: if key in session: request[key] = session[key] - return session_user diff --git a/learning_observer/learning_observer/auth/social_sso.py b/learning_observer/learning_observer/auth/social_sso.py index 6d7f83895..f2d79bda1 100644 --- a/learning_observer/learning_observer/auth/social_sso.py +++ b/learning_observer/learning_observer/auth/social_sso.py @@ -79,8 +79,7 @@ pmss.register_field( name='fetch_additional_info_from_teacher_on_login', type=pmss.pmsstypes.TYPES.boolean, - description='Whether we should start an additional task that will '\ - 'fetch all text from current rosters.', + description='Whether we should start an additional task that will fetch all text from current rosters.', default=False ) pmss.register_field( @@ -141,7 +140,7 @@ async def social_handler(request): ) user = await _handle_google_authorization(request) - + roster_source = settings.pmss_settings.source(types=['roster_data']) await _set_lms_header_information(request, roster_source) @@ -162,13 +161,13 @@ async def social_handler(request): async def _set_lms_header_information(request, roster_source): """ Handles the authorization of the specified Learning Management System (LMS) - based on the roster data source and delegating the request to the appropriate handler + based on the roster data source and delegating the request to the appropriate handler based on the data source type. - """ + """ lms_map = { constants.CANVAS: _handle_canvas_authorization } - + # Handle the request depending on the roster source if roster_source in lms_map: return await lms_map[roster_source](request) @@ -241,16 +240,17 @@ async def _process_student_documents(student): await _process_student_documents(student) # TODO saved skipped doc ids somewhere? + async def _handle_canvas_authorization(request): ''' Handle Canvas authorization ''' if 'error' in request.query: return {} - + token_uri = settings.pmss_settings.token_uri(types=['lms', 'canvas_oauth']) url = token_uri - + params = { "grant_type": "refresh_token", 'client_id': settings.pmss_settings.client_id(types=['lms', 'canvas_oauth']), @@ -267,9 +267,10 @@ async def _handle_canvas_authorization(request): session = await aiohttp_session.get_session(request) session[constants.CANVAS_AUTH_HEADERS] = canvas_headers request[constants.CANVAS_AUTH_HEADERS] = canvas_headers - + return data + async def _handle_google_authorization(request): ''' Handle Google login diff --git a/learning_observer/learning_observer/canvas.py b/learning_observer/learning_observer/canvas.py index 46d73cadf..de94161ee 100755 --- a/learning_observer/learning_observer/canvas.py +++ b/learning_observer/learning_observer/canvas.py @@ -16,11 +16,11 @@ register_cleaner_with_endpoints = functools.partial(learning_observer.lms_integration.register_cleaner, endpoints=CANVAS_ENDPOINTS) - + class CanvasLMS(learning_observer.lms_integration.LMS): def __init__(self): super().__init__(lms_name=LMS_NAME, endpoints=CANVAS_ENDPOINTS) - + @register_cleaner_with_endpoints("course_roster", "roster") def clean_course_roster(canvas_json): students = canvas_json @@ -52,14 +52,16 @@ def clean_course_list(canvas_json): courses = canvas_json courses.sort(key=lambda x: x.get('name', 'ZZ')) return courses - + @register_cleaner_with_endpoints("course_assignments", "assignments") def clean_course_assignment_list(canvas_json): assignments = canvas_json assignments.sort(key=lambda x: x.get('name', 'ZZ')) return assignments - + + canvas_lms = CanvasLMS() + def initialize_canvas_routes(app): canvas_lms.initialize_routes(app) diff --git a/learning_observer/learning_observer/google.py b/learning_observer/learning_observer/google.py index f0960b3a4..831298244 100644 --- a/learning_observer/learning_observer/google.py +++ b/learning_observer/learning_observer/google.py @@ -71,6 +71,7 @@ register_cleaner_with_endpoints = functools.partial(learning_observer.lms_integration.register_cleaner, endpoints=GOOGLE_ENDPOINTS) + # Google Docs def _force_text_length(text, length): ''' @@ -84,6 +85,7 @@ def _force_text_length(text, length): ''' return text[:length] + " " * (length - len(text)) + def get_error_details(error): messages = { 403: 'Student working on private document.', @@ -93,6 +95,7 @@ def get_error_details(error): message = messages.get(code, 'Unknown error.') return {'error': {'code': code, 'message': message}} + class GoogleLMS(learning_observer.lms_integration.LMS): def __init__(self): super().__init__(lms_name=LMS_NAME, endpoints=GOOGLE_ENDPOINTS) @@ -163,13 +166,13 @@ def extract_text_from_google_doc_json( if align: for f in flat: text = f.get('textRun', {}).get('content', None) - if text != None: + if text is not None: length = f['endIndex'] - f['startIndex'] text_chunks.append(_force_text_length(text, length)) else: for f in flat: text = f.get('textRun', {}).get('content', None) - if text != None: + if text is not None: text_chunks.append(text) text = ''.join(text_chunks) @@ -220,13 +223,16 @@ def connect_to_google_cache(): '```\ngoogle_cache:\n type: filesystem\n path: ./learning_observer/static_data/google\n'\ ' subdirs: true\n```\nOR\n'\ '```\ngoogle_cache:\n type: redis_ephemeral\n expiry: 600\n```' - raise learning_observer.prestartup.StartupCheck("Google KVS: " + error_text) - + raise learning_observer.prestartup.StartupCheck("Google KVS: " + error_text) + + google_lms = GoogleLMS() + def initialize_google_routes(app): google_lms.initialize_routes(app) + if __name__ == '__main__': import json import sys diff --git a/learning_observer/learning_observer/lms_integration.py b/learning_observer/learning_observer/lms_integration.py index 15508a71d..4a8fbef68 100644 --- a/learning_observer/learning_observer/lms_integration.py +++ b/learning_observer/learning_observer/lms_integration.py @@ -20,11 +20,12 @@ cache = None + class Endpoint(recordclass.make_dataclass("Endpoint", ["name", "remote_url", "doc", "cleaners", "lms"], defaults=["", None])): """ The Endpoint class represents an API endpoint, allowing for parameter extraction, URL construction, and cleaner (function) management. - + Attributes: name (str): The name of the endpoint. remote_url (str): The remote URL of the endpoint, which may contain parameters. @@ -32,7 +33,7 @@ class Endpoint(recordclass.make_dataclass("Endpoint", ["name", "remote_url", "do cleaners (dict): A dictionary of cleaner functions associated with the endpoint. lms (str): The learning management system (LMS) that the endpoint belongs to. """ - + def arguments(self): """ Extracts the parameters from the remote URL. @@ -64,7 +65,7 @@ def _add_cleaner(self, name, cleaner): Args: name (str): The name to associate with the cleaner. - cleaner (dict): The cleaner function to be added, optionally containing + cleaner (dict): The cleaner function to be added, optionally containing additional metadata such as its local URL. """ if self.cleaners is None: @@ -85,12 +86,13 @@ def _cleaners(self): else: return self.cleaners + def extract_parameters_from_format_string(format_string): ''' Extracts parameters from a format string. E.g. >>> ("hello {hi} my {bye}")] ['hi', 'bye'] - + Args: format_string (str): The format string containing parameters enclosed in braces. @@ -99,6 +101,7 @@ def extract_parameters_from_format_string(format_string): ''' return [f[1] for f in string.Formatter().parse(format_string) if f[1] is not None] + def raw_access_partial(raw_ajax_function, target_url, name=None): ''' Creates an asynchronous function that calls a specific LMS API. @@ -117,7 +120,7 @@ def raw_access_partial(raw_ajax_function, target_url, name=None): async def ajax_caller(request, **kwargs): ''' Make an AJAX request to LMS - + Args: request: The incoming request object. **kwargs: Additional keyword arguments to pass to the raw AJAX function. @@ -130,10 +133,11 @@ async def ajax_caller(request, **kwargs): return ajax_caller + def api_docs_handler(endpoints): ''' Returns a list of available endpoints in a human-readable format. - + Eventually, we should also document available function calls Args: @@ -142,7 +146,7 @@ def api_docs_handler(endpoints): Returns: aiohttp.web.Response: A response object containing the documentation of endpoints. ''' - + response = "URL Endpoints:\n\n" for endpoint in endpoints: response += f"{endpoint._local_url()}\n" @@ -152,6 +156,7 @@ def api_docs_handler(endpoints): response += "\n\n Globals:" return aiohttp.web.Response(text=response) + def register_cleaner(data_source, cleaner_name, endpoints): ''' Registers a cleaner function, allowing it to be exported both as a web service @@ -164,7 +169,7 @@ def register_cleaner(data_source, cleaner_name, endpoints): Returns: callable: A decorator for registering the cleaner function. - + Raises: AttributeError: If the data source is not found in the endpoints. ''' @@ -188,6 +193,7 @@ def add_cleaner(f): return add_cleaner + def make_ajax_raw_handler(raw_ajax_function, remote_url): ''' Creates an AJAX passthrough handler that calls a raw AJAX function. @@ -217,6 +223,7 @@ async def ajax_passthrough(request): return aiohttp.web.json_response(response) return ajax_passthrough + def make_cleaner_handler(raw_function, cleaner_function, name=None): ''' Creates a handler for the cleaner function. @@ -244,24 +251,25 @@ async def cleaner_handler(request): ''' # Call the raw function with the request and match_info as parameters response = cleaner_function(await raw_function(request, **request.match_info)) - + # Determine the response type and return appropriately if isinstance(response, dict) or isinstance(response, list): - return aiohttp.web.json_response(response) # Return JSON response for dict or list + return aiohttp.web.json_response(response) # Return JSON response for dict or list elif isinstance(response, str): - return aiohttp.web.Response(text=response) # Return plain text response if it's a string + return aiohttp.web.Response(text=response) # Return plain text response if it's a string else: - raise AttributeError(f"Invalid response type: {type(response)}") # Handle unexpected response types + raise AttributeError(f"Invalid response type: {type(response)}") # Handle unexpected response types if name is not None: setattr(cleaner_handler, "__qualname__", name + "_handler") return cleaner_handler + def make_cleaner_function(raw_function, cleaner_function, name=None): """ Creates a cleaner function that processes the output of a raw function. - This function wraps a raw function and a cleaner function, allowing the cleaner + This function wraps a raw function and a cleaner function, allowing the cleaner to be applied to the response of the raw function. Args: @@ -270,7 +278,7 @@ def make_cleaner_function(raw_function, cleaner_function, name=None): name (str, optional): The name to assign to the created cleaner function. Returns: - callable: An asynchronous cleaner function that calls the raw function + callable: An asynchronous cleaner function that calls the raw function and processes its output with the cleaner function. """ async def cleaner_local(request, **kwargs): @@ -287,23 +295,24 @@ async def cleaner_local(request, **kwargs): lms_response = await raw_function(request, **kwargs) clean = cleaner_function(lms_response) return clean - + if name is not None: setattr(cleaner_local, "__qualname__", name) return cleaner_local + async def raw_ajax(runtime, target_url, lms_name, base_url=None, **kwargs): """ - Make an authenticated AJAX call to a specified service (e.g., Google, Canvas), handling + Make an authenticated AJAX call to a specified service (e.g., Google, Canvas), handling authorization, caching, and retries. Parameters: - runtime: An instance of the Runtime class containing request information. - lms_name: A string indicating the name of the service ('google' or 'canvas'). - target_url: The URL endpoint to be called, with optional formatting using kwargs. - - base_url: An optional base URL for the service. If provided, it will be prefixed + - base_url: An optional base URL for the service. If provided, it will be prefixed to target_url. - - kwargs: Additional keyword arguments to format the target_url or control behavior + - kwargs: Additional keyword arguments to format the target_url or control behavior (e.g., retry). Returns: @@ -316,10 +325,10 @@ async def raw_ajax(runtime, target_url, lms_name, base_url=None, **kwargs): # Retrieve the incoming request and active user request = runtime.get_request() user = await learning_observer.auth.get_active_user(request) - + # Extract 'retry' flag from kwargs (defaults to False) retry = kwargs.pop('retry', False) - + # mapping to determine the appropriate headers based on the service headers = { constants.GOOGLE: request.get(constants.AUTH_HEADERS), @@ -329,7 +338,7 @@ async def raw_ajax(runtime, target_url, lms_name, base_url=None, **kwargs): # Ensure Google requests are authenticated if lms_name == constants.GOOGLE and constants.AUTH_HEADERS not in request: raise aiohttp.web.HTTPUnauthorized(text="Please log in") - + # Construct the full URL using the base URL if provided, otherwise use the target URL directly if base_url: url = base_url + target_url.format(**kwargs) @@ -338,7 +347,7 @@ async def raw_ajax(runtime, target_url, lms_name, base_url=None, **kwargs): # Generate a unique cache key based on the service, user, and request URL cache_key = f"raw_{lms_name}/" + learning_observer.auth.encode_id('session', user[constants.USER_ID]) + '/' + learning_observer.util.url_pathname(url) - + cache_flag = f"use_{lms_name}_ajax" # Check cache and return cached response if available if settings.feature_flag(cache_flag) is not None: @@ -350,7 +359,7 @@ async def raw_ajax(runtime, target_url, lms_name, base_url=None, **kwargs): json.loads(value), learning_observer.google.GOOGLE_TO_SNAKE ) - else: + else: return json.loads(value) # Make the actual AJAX call to the service @@ -383,12 +392,14 @@ async def raw_ajax(runtime, target_url, lms_name, base_url=None, **kwargs): return await raw_ajax(runtime, target_url, lms_name, base_url, **kwargs) raise + # Abstract raw_ajax for each LMS to specify their different arguments async def raw_google_ajax(runtime, target_url, **kwargs): """Make an authenticated AJAX call to the Google API.""" return await raw_ajax(runtime, target_url, constants.GOOGLE, **kwargs) + async def raw_canvas_ajax(runtime, target_url, **kwargs): """Make an authenticated AJAX call to the Canvas API.""" base_url = settings.pmss_settings.lms_api(types=['lms', 'canvas_oauth']) @@ -426,13 +437,13 @@ def initialize_routes(self, app): """ Initializes the API routes for the specified LMS within the given web application. - This method sets up the endpoint routes and associates them with their corresponding + This method sets up the endpoint routes and associates them with their corresponding handler functions. Args: app: An instance of the aiohttp web application to which routes will be added. """ - + # Add the main API documentation route app.add_routes([ aiohttp.web.get(f"/{self.lms_name}", lambda _: api_docs_handler(self.endpoints)) @@ -440,25 +451,25 @@ def initialize_routes(self, app): # Iterate through the endpoints to set up routes for each one for e in self.endpoints: - function_name = f"raw_{e.name}" # Construct the function name for the raw AJAX function + function_name = f"raw_{e.name}" # Construct the function name for the raw AJAX function raw_function = raw_access_partial( - raw_ajax_function = self.raw_ajax_function[self.lms_name], # Get the appropriate raw AJAX function - target_url = e.remote_url, # Use the endpoint's remote URL - name = e.name # Set the name for the function + raw_ajax_function=self.raw_ajax_function[self.lms_name], # Get the appropriate raw AJAX function + target_url=e.remote_url, # Use the endpoint's remote URL + name=e.name # Set the name for the function ) - globals()[function_name] = raw_function # Register the raw function globally - + globals()[function_name] = raw_function # Register the raw function globally + # Add routes for each cleaner associated with the endpoint cleaners = e._cleaners() for c in cleaners: app.add_routes([ aiohttp.web.get( - cleaners[c]['local_url'], # The local URL for the cleaner - make_cleaner_handler(raw_function, cleaners[c]['function'], name=cleaners[c]['name']) # Handler for the cleaner + cleaners[c]['local_url'], # The local URL for the cleaner + make_cleaner_handler(raw_function, cleaners[c]['function'], name=cleaners[c]['name']) # Handler for the cleaner ) ]) - lms_module = getattr(learning_observer, self.lms_name) # Get the module for the LMS - + lms_module = getattr(learning_observer, self.lms_name) # Get the module for the LMS + # Create the cleaner function and set it in the LMS module cleaner_function = make_cleaner_function( raw_function, @@ -466,11 +477,11 @@ def initialize_routes(self, app): name=cleaners[c]['name'] ) setattr(lms_module, cleaners[c]['name'], cleaner_function) - + # Add the main route for the endpoint app.add_routes([ aiohttp.web.get(e._local_url(), make_ajax_raw_handler( - self.raw_ajax_function[self.lms_name], # The raw AJAX function for the LMS - e.remote_url # The endpoint's remote URL + self.raw_ajax_function[self.lms_name], # The raw AJAX function for the LMS + e.remote_url # The endpoint's remote URL )) - ]) \ No newline at end of file + ]) diff --git a/learning_observer/learning_observer/rosters.py b/learning_observer/learning_observer/rosters.py index a5f65a6ce..92face58b 100644 --- a/learning_observer/learning_observer/rosters.py +++ b/learning_observer/learning_observer/rosters.py @@ -92,11 +92,11 @@ pmss.register_field( name='source', type='roster_source', - description='Source to use for student class rosters. This can be\n'\ - '`all`: aggregate all available students into a single class\n'\ - '`test`: use sample course and student files\n'\ - '`filesystem`: read rosters defined on filesystem\n'\ - '`google_api`: fetch from Google API\n'\ + description='Source to use for student class rosters. This can be\n' + '`all`: aggregate all available students into a single class\n' + '`test`: use sample course and student files\n' + '`filesystem`: read rosters defined on filesystem\n' + '`google_api`: fetch from Google API\n' '`canvas`: fetch from Canvas API', required=True ) @@ -413,15 +413,15 @@ async def courselist(request): ''' List all of the courses a teacher manages: Helper ''' - + # A map of LMSes to their respective handler functions lms_map = { constants.GOOGLE: learning_observer.google.courses, constants.CANVAS: learning_observer.canvas.courses } - + runtime = learning_observer.runtime.Runtime(request) - + roster_source = settings.pmss_settings.source(types=['roster_data']) if roster_source in lms_map: return await lms_map[roster_source](runtime) @@ -466,15 +466,15 @@ async def courseroster(request, course_id): ''' List all of the students in a course: Helper ''' - + # A map of LMSes to their respective handler functions lms_map = { constants.GOOGLE: learning_observer.google.roster, constants.CANVAS: learning_observer.canvas.roster } - + runtime = learning_observer.runtime.Runtime(request) - + roster_source = settings.pmss_settings.source(types=['roster_data']) if roster_source in lms_map: return await lms_map[roster_source](runtime, courseId=course_id) diff --git a/learning_observer/learning_observer/routes.py b/learning_observer/learning_observer/routes.py index 18a832aea..b3c193141 100644 --- a/learning_observer/learning_observer/routes.py +++ b/learning_observer/learning_observer/routes.py @@ -166,11 +166,11 @@ def tracemalloc_handler(request): # and figuring stuff out, this feels safest to put last. register_wsgi_routes(app) - + def register_lms_routes(app): """ Register routes for the various Learning Management Systems (LMS). - + Parameters: - app: An instance of aiohttp.web.Application where the routes will be registered. """ diff --git a/learning_observer/learning_observer/settings.py b/learning_observer/learning_observer/settings.py index 1c1de6aa6..8582507a0 100644 --- a/learning_observer/learning_observer/settings.py +++ b/learning_observer/learning_observer/settings.py @@ -39,6 +39,7 @@ args = None parser = None + def str_to_bool(arg): if isinstance(arg, bool): return arg @@ -123,9 +124,9 @@ def parse_and_validate_arguments(): pmss.register_field( name='run_mode', type='run_mode', - description="Set which mode the server is running in.\n"\ - "`dev` for local development with full debugging\n"\ - "`deploy` for running on a server with better performance\n"\ + description="Set which mode the server is running in.\n" + "`dev` for local development with full debugging\n" + "`deploy` for running on a server with better performance\n" "`interactive` for processing data offline", required=True )