diff --git a/backend/Makefile b/backend/Makefile index 9d6d9e2..f6a21c9 100644 --- a/backend/Makefile +++ b/backend/Makefile @@ -66,7 +66,7 @@ stop-worker: -$(PYTHON) -m celery -A core control shutdown \ --destination=worker@$(shell hostname) || true -test: test-models test-header-validation-task test-syntax-task test-syntax-header-validation-task test-schema-task +test: test-models test-header-validation-task test-syntax-task test-syntax-header-validation-task test-schema-task test-magic-and-av-task test-models: MEDIA_ROOT=./apps/ifc_validation/fixtures $(PYTHON) manage.py test apps/ifc_validation_models --settings apps.ifc_validation_models.test_settings --debug-mode --verbosity 3 @@ -83,6 +83,9 @@ test-syntax-task: test-schema-task: MEDIA_ROOT=./apps/ifc_validation/fixtures $(PYTHON) manage.py test apps.ifc_validation.tests.tests_schema_validation_task --settings apps.ifc_validation.test_settings --debug-mode --verbosity 3 +test-magic-and-av-task: + MEDIA_ROOT=./apps/ifc_validation/fixtures $(PYTHON) manage.py test apps.ifc_validation.tests.tests_magic_clamav_task --settings apps.ifc_validation.test_settings --debug-mode --verbosity 3 + archive-dry-run: $(PYTHON) manage.py archive_requests --days 180 --all --dry-run diff --git a/backend/apps/ifc_validation/admin.py b/backend/apps/ifc_validation/admin.py index 391f835..aaee4cb 100644 --- a/backend/apps/ifc_validation/admin.py +++ b/backend/apps/ifc_validation/admin.py @@ -450,7 +450,8 @@ class ModelAdmin(BaseAdmin, NonAdminAddable): "status_industry_practices", "status_mvd", "status_bsdd", - "status_signatures" + "status_signatures", + "status_magic_clamav", ]}), ('Auditing Information', {"classes": ("wide"), "fields": [("created",), ("updated")]}) ] diff --git a/backend/apps/ifc_validation/chart_views.py b/backend/apps/ifc_validation/chart_views.py index b8a9a75..3c574eb 100644 --- a/backend/apps/ifc_validation/chart_views.py +++ b/backend/apps/ifc_validation/chart_views.py @@ -65,6 +65,7 @@ "prereq": "#b4acb4", "digital_signatures": "#73d0d8", "inst_completion": "#e76565", + "magic_and_av": "#a29bfe", } SYNTAX_TASK_TYPES = { @@ -83,6 +84,7 @@ "INDUSTRY": ("Industry", COLORS["industry"]), "PREREQ": ("Prereq", COLORS["prereq"]), "INST_COMPLETION": ("Inst Completion", COLORS["inst_completion"]), + "MAGIC_AND_CLAMAV": ("Magic/AV", COLORS["magic_and_av"]), } SECONDS_PER_MINUTE = 60 diff --git a/backend/apps/ifc_validation/checks/ifc_gherkin_rules b/backend/apps/ifc_validation/checks/ifc_gherkin_rules index 8aeadc9..04c930e 160000 --- a/backend/apps/ifc_validation/checks/ifc_gherkin_rules +++ b/backend/apps/ifc_validation/checks/ifc_gherkin_rules @@ -1 +1 @@ -Subproject commit 8aeadc9203c8297b0c5e03bd701c538070c9a651 +Subproject commit 04c930e3630bfe05978ddf9f1468d73df889652d diff --git a/backend/apps/ifc_validation/fixtures/eicar_testfile.ifc b/backend/apps/ifc_validation/fixtures/eicar_testfile.ifc new file mode 100644 index 0000000..e69de29 diff --git a/backend/apps/ifc_validation/tasks/__init__.py b/backend/apps/ifc_validation/tasks/__init__.py index 603604b..a3c97a8 100644 --- a/backend/apps/ifc_validation/tasks/__init__.py +++ b/backend/apps/ifc_validation/tasks/__init__.py @@ -13,7 +13,8 @@ normative_rules_ip_validation_subtask, bsdd_validation_subtask, industry_practices_subtask, - instance_completion_subtask + instance_completion_subtask, + magic_clamav_subtask ) __all__ = [ @@ -28,4 +29,5 @@ "normative_rules_ip_validation_subtask", "industry_practices_subtask", "instance_completion_subtask", + "magic_clamav_subtask" ] \ No newline at end of file diff --git a/backend/apps/ifc_validation/tasks/check_programs.py b/backend/apps/ifc_validation/tasks/check_programs.py index 50e5d26..3ef7893 100644 --- a/backend/apps/ifc_validation/tasks/check_programs.py +++ b/backend/apps/ifc_validation/tasks/check_programs.py @@ -1,10 +1,15 @@ import os import sys import json +import shutil import subprocess from typing import List from dataclasses import dataclass +# pip install filetype +import filetype +from filetype.types import archive + from apps.ifc_validation_models.settings import TASK_TIMEOUT_LIMIT from apps.ifc_validation_models.models import ValidationTask @@ -112,6 +117,57 @@ def check_header(context:TaskContext): return context +def check_magic_and_clamav(context:TaskContext): + result = {} + ty = filetype.guess(context.file_path) + if type(ty) in (type(None), archive.Zip): + # happy path, continue + # some support for zipbombs + clamscan = shutil.which('clamscan') + if clamscan: + proc = run_subprocess( + task=context.task, + command=[ + clamscan, + '--alert-exceeds-max', '--max-recursion=2', '--max-files=10', '--max-scansize=256M', '--max-filesize=128M', + '--stdout', + context.file_path] + ) + if proc.returncode != 0: + result = { + 'invalid': f'suspicious file\n\n{proc.stdout}\n{proc.stderr}' + } + else: + result = { + 'valid': 'unknown type' if ty is None else ty.mime + } + else: + print('WARNING: clamscan not installed') + result = { + 'warn': 'clamscan not installed' + } + else: + try: + mime = ty.mime + except: + mime = 'unknown mime type' + result = { + 'invalid': mime + } + if 'invalid' in result: + print('REMOVING:', context.file_path) + # we do not unlink() the file because it would create DB issues + # when a file with the exact same name is reuploaded and it is not + # made unique. + with open(context.file_path, 'w'): + pass + context.result = { + "success": 'warn' not in result, + "valid": 'invalid' not in result, + 'output': next(iter(result.values())), + } + return context + def check_digital_signatures(context:TaskContext): proc = run_subprocess( task=context.task, diff --git a/backend/apps/ifc_validation/tasks/configs.py b/backend/apps/ifc_validation/tasks/configs.py index 029b992..28068db 100644 --- a/backend/apps/ifc_validation/tasks/configs.py +++ b/backend/apps/ifc_validation/tasks/configs.py @@ -44,17 +44,18 @@ def _load_function(module, prefix, type): ) # define task info -header_syntax = make_task(type=ValidationTask.Type.HEADER_SYNTAX, increment=5, field='status_header_syntax', stage="serial") -header = make_task(type=ValidationTask.Type.HEADER, increment=10, field='status_header', stage="serial") -syntax = make_task(type=ValidationTask.Type.SYNTAX, increment=5, field='status_syntax', stage="serial") -prerequisites = make_task(type=ValidationTask.Type.PREREQUISITES, increment=10, field='status_prereq', stage="serial") -schema = make_task(type=ValidationTask.Type.SCHEMA, increment=10, field='status_schema') -digital_signatures = make_task(type=ValidationTask.Type.DIGITAL_SIGNATURES, increment=5, field='status_signatures') -bsdd = make_task(type=ValidationTask.Type.BSDD, increment=0, field='status_bsdd') -normative_ia = make_task(type=ValidationTask.Type.NORMATIVE_IA, increment=20, field='status_ia') -normative_ip = make_task(type=ValidationTask.Type.NORMATIVE_IP, increment=20, field='status_ip') -industry_practices = make_task(type=ValidationTask.Type.INDUSTRY_PRACTICES, increment=10, field='status_industry_practices') -instance_completion = make_task(type=ValidationTask.Type.INSTANCE_COMPLETION, increment=5, field=None, stage="final") +magic_clamav = make_task(type=ValidationTask.Type.MAGIC_AND_CLAMAV, increment=5, field='status_magic_clamav', stage="serial") +header_syntax = make_task(type=ValidationTask.Type.HEADER_SYNTAX, increment=5, field='status_header_syntax', stage="serial") +header = make_task(type=ValidationTask.Type.HEADER, increment=10, field='status_header', stage="serial") +syntax = make_task(type=ValidationTask.Type.SYNTAX, increment=5, field='status_syntax', stage="serial") +prerequisites = make_task(type=ValidationTask.Type.PREREQUISITES, increment=5, field='status_prereq', stage="serial") +schema = make_task(type=ValidationTask.Type.SCHEMA, increment=10, field='status_schema') +digital_signatures = make_task(type=ValidationTask.Type.DIGITAL_SIGNATURES, increment=5, field='status_signatures') +bsdd = make_task(type=ValidationTask.Type.BSDD, increment=0, field='status_bsdd') +normative_ia = make_task(type=ValidationTask.Type.NORMATIVE_IA, increment=20, field='status_ia') +normative_ip = make_task(type=ValidationTask.Type.NORMATIVE_IP, increment=20, field='status_ip') +industry_practices = make_task(type=ValidationTask.Type.INDUSTRY_PRACTICES, increment=10, field='status_industry_practices') +instance_completion = make_task(type=ValidationTask.Type.INSTANCE_COMPLETION, increment=5, field=None, stage="final") # block tasks on error post_tasks = [digital_signatures, schema, normative_ia, normative_ip, industry_practices, instance_completion] @@ -64,10 +65,15 @@ def _load_function(module, prefix, type): # register ALL_TASKS = [ + magic_clamav, header_syntax, header, syntax, prerequisites, schema, digital_signatures, bsdd, normative_ia, normative_ip, industry_practices, instance_completion, ] + +# av check blocks everything +magic_clamav.blocks = [t for t in ALL_TASKS if t is not magic_clamav] + class TaskRegistry: def __init__(self, config_map: dict[str, TaskConfig]): self._configs = config_map diff --git a/backend/apps/ifc_validation/tasks/processing/__init__.py b/backend/apps/ifc_validation/tasks/processing/__init__.py index 40fea8e..53313ee 100644 --- a/backend/apps/ifc_validation/tasks/processing/__init__.py +++ b/backend/apps/ifc_validation/tasks/processing/__init__.py @@ -10,4 +10,5 @@ from .schema import process_schema from .header import process_header from .digital_signatures import process_digital_signatures -from .bsdd import process_bsdd \ No newline at end of file +from .bsdd import process_bsdd +from .magicav import process_magic_and_clamav \ No newline at end of file diff --git a/backend/apps/ifc_validation/tasks/processing/magicav.py b/backend/apps/ifc_validation/tasks/processing/magicav.py new file mode 100644 index 0000000..aed8b1a --- /dev/null +++ b/backend/apps/ifc_validation/tasks/processing/magicav.py @@ -0,0 +1,13 @@ +from .. import TaskContext, with_model +from apps.ifc_validation_models.models import Model + + +def process_magic_and_clamav(context:TaskContext): + output, success, valid = (context.result.get(k) for k in ("output", "success", "valid")) + + with with_model(context.request.id) as model: + agg_status = Model.Status.VALID if valid else Model.Status.INVALID + setattr(model, context.config.status_field.name, agg_status) + + model.save(update_fields=[context.config.status_field.name]) + return f'Magic and av check completed:\nsuccess = {success}\nvalid = {valid}\noutput = {output}' diff --git a/backend/apps/ifc_validation/tasks/task_runner.py b/backend/apps/ifc_validation/tasks/task_runner.py index 799f7d9..b440b2b 100644 --- a/backend/apps/ifc_validation/tasks/task_runner.py +++ b/backend/apps/ifc_validation/tasks/task_runner.py @@ -203,6 +203,7 @@ def ifc_file_validation_task(self, id, file_name, *args, **kwargs): workflow_completed = on_workflow_completed.s(id=id, file_name=file_name) serial_tasks = chain( + magic_clamav_subtask.s(id=id, file_name=file_name), header_syntax_validation_subtask.s(id=id, file_name=file_name), header_validation_subtask.s(id=id, file_name=file_name), syntax_validation_subtask.s(id=id, file_name=file_name), @@ -254,3 +255,5 @@ def ifc_file_validation_task(self, id, file_name, *args, **kwargs): bsdd_validation_subtask = task_factory(ValidationTask.Type.BSDD) industry_practices_subtask = task_factory(ValidationTask.Type.INDUSTRY_PRACTICES) + +magic_clamav_subtask = task_factory(ValidationTask.Type.MAGIC_AND_CLAMAV) diff --git a/backend/apps/ifc_validation/tests/tests_magic_clamav_task.py b/backend/apps/ifc_validation/tests/tests_magic_clamav_task.py new file mode 100644 index 0000000..7fecdaa --- /dev/null +++ b/backend/apps/ifc_validation/tests/tests_magic_clamav_task.py @@ -0,0 +1,70 @@ +import datetime + +from django.test import TransactionTestCase +from django.contrib.auth.models import User + +from apps.ifc_validation_models.models import * +from apps.ifc_validation.tasks.utils import get_absolute_file_path + +from ..tasks import magic_clamav_subtask + +class MagicClamAVTaskTestCase(TransactionTestCase): + + def set_user_context(): + user = User.objects.create(id=1, username='SYSTEM', is_active=True) + set_user_context(user) + + def test_magic_clamav_task_detects_valid_ifc_file(self): + + # arrange + MagicClamAVTaskTestCase.set_user_context() + request = ValidationRequest.objects.create( + file_name='valid_file.ifc', + file='valid_file.ifc', + size=1 + ) + request.mark_as_initiated() + + # act + task = magic_clamav_subtask( + prev_result={'is_valid': True, 'reason': 'test'}, + id=request.id, + file_name=request.file_name + ) + print(task) + + # assert + model = Model.objects.get(id=request.id) + self.assertIsNotNone(model) + self.assertEqual(model.status_magic_clamav, Model.Status.VALID) + + def test_magic_clamav_task_detects_eicar_testfile(self): + + # arrange + MagicClamAVTaskTestCase.set_user_context() + request = ValidationRequest.objects.create( + file_name='eicar_testfile.ifc', + file='eicar_testfile.ifc', + size=68 + ) + request.mark_as_initiated() + + # make sure the test file contains eicar test string + file_path = get_absolute_file_path(request.file.name) + with open(file_path, 'w') as f: + EICAR_TEST_STRING = 'X5O!P%@AP[4\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-FILE!$H+H*' + f.write(EICAR_TEST_STRING) + + # act + task = magic_clamav_subtask( + prev_result={'is_valid': True, 'reason': 'test'}, + id=request.id, + file_name=request.file_name + ) + print(task) + + # assert + model = Model.objects.get(id=request.id) + self.assertIsNotNone(model) + self.assertEqual(model.status_magic_clamav, Model.Status.INVALID) + \ No newline at end of file diff --git a/backend/apps/ifc_validation_bff/views_legacy.py b/backend/apps/ifc_validation_bff/views_legacy.py index e0cc352..e8aa299 100644 --- a/backend/apps/ifc_validation_bff/views_legacy.py +++ b/backend/apps/ifc_validation_bff/views_legacy.py @@ -197,6 +197,7 @@ def format_request(request): ), "status_ind": "p" if (request.model is None or request.model.status_industry_practices is None) else request.model.status_industry_practices, "status_signatures": "p" if (request.model is None or request.model.status_signatures is None) else request.model.status_signatures, + "status_magic_clamav": "p" if (request.model is None or request.model.status_magic_clamav is None) else request.model.status_magic_clamav, "deleted": 0, # TODO "commit_id": None # TODO } diff --git a/backend/apps/ifc_validation_models b/backend/apps/ifc_validation_models index 3a4c020..f9551c6 160000 --- a/backend/apps/ifc_validation_models +++ b/backend/apps/ifc_validation_models @@ -1 +1 @@ -Subproject commit 3a4c02072fe55da0b5f2960e9808fe1890ec9445 +Subproject commit f9551c601babe74ff9dd4c39a511bb0014163d08 diff --git a/backend/requirements.txt b/backend/requirements.txt index 7720e5a..0bb367e 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -46,6 +46,7 @@ shapely==2.1.1 python-ranges==1.2.2 pyproj==3.7.1 python-dateutil==2.9.0.post0 +filetype==1.2.0 # dev django-debug-toolbar==6.0.0 diff --git a/docker/backend/Dockerfile b/docker/backend/Dockerfile index 99b15cc..bf480e0 100644 --- a/docker/backend/Dockerfile +++ b/docker/backend/Dockerfile @@ -72,8 +72,11 @@ RUN set -ex && \ postgresql-client \ netcat-openbsd \ procps \ - htop && \ + htop \ + clamav && \ update-ca-certificates -f && \ + # clamav initial setup + freshclam && \ # cleanup apt-get -y clean && \ apt-get autoremove -y && \ diff --git a/frontend/src/DashboardTable.js b/frontend/src/DashboardTable.js index f5ceeec..e7ffaa1 100644 --- a/frontend/src/DashboardTable.js +++ b/frontend/src/DashboardTable.js @@ -434,9 +434,12 @@ export default function DashboardTable({ models }) { } - evt.stopPropagation()}> - {'Download file'} - + { + (row.status_magic_clamav != 'i') ? + evt.stopPropagation()}> + {'Download file'} + : 'File unavailable' + } );