diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index eb9838ff..975b3c1b 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -32,4 +32,4 @@ jobs: python -m pip install -e . - name: Run pyright - run: pyright \ No newline at end of file + run: pyright glitch \ No newline at end of file diff --git a/.github/workflows/rego_python.yml b/.github/workflows/rego_python.yml new file mode 100644 index 00000000..579f5029 --- /dev/null +++ b/.github/workflows/rego_python.yml @@ -0,0 +1,188 @@ +name: rego-python + +on: + push: + paths: + - ".github/workflows/rego-python.yml" + - "glitch/rego/rego_python/**" + workflow_dispatch: + +permissions: + contents: write + +jobs: + build-binaries: + name: Build Rego Python binaries + runs-on: ${{ matrix.os_runner }} + strategy: + matrix: + include: + # Linux + - os: linux + arch: amd64 + os_runner: ubuntu-latest + - os: linux + arch: arm64 + os_runner: ubuntu-24.04-arm + + # macOS + - os: darwin + arch: amd64 + os_runner: macos-15-intel + - os: darwin + arch: arm64 + os_runner: macos-latest + + # Windows + - os: windows + arch: amd64 + os_runner: windows-latest + #- os: windows + # arch: arm64 + # os_runner: windows-11-arm + # This fails the build because the runner is still limited + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v4 + with: + go-version: 'stable' + cache-dependency-path: glitch/rego/rego_python/src/rego_python/go/go.sum + + - name: Build binary + shell: bash + run: | + OS=${{ matrix.os }} + ARCH=${{ matrix.arch }} + + # Determine file extension + if [ "$OS" = "windows" ]; then + EXT="dll" + elif [ "$OS" = "darwin" ]; then + EXT="dylib" + else + EXT="so" + fi + + OUTPUT="../bin/librego-$OS-$ARCH.$EXT" + echo "Building $OUTPUT ..." + + cd glitch/rego/rego_python/src/rego_python/go + GOOS=$OS GOARCH=$ARCH go build -o "$OUTPUT" -buildmode=c-shared regolib.go + rm -f ../bin/librego-*.h + + - name: List bin folder + run: ls -l glitch/rego/rego_python/src/rego_python/bin/ + + - name: Upload binary artifact + uses: actions/upload-artifact@v4 + with: + name: librego-${{ matrix.os }}-${{ matrix.arch }} + path: glitch/rego/rego_python/src/rego_python/bin/librego-${{ matrix.os }}-${{ matrix.arch }}.* + retention-days: 1 + + build-package: + name: Build Python package + runs-on: ubuntu-latest + needs: build-binaries + + steps: + - uses: actions/checkout@v4 + + # Download all binary artifacts into bin/ + - uses: actions/download-artifact@v4 + with: + path: glitch/rego/rego_python/src/rego_python/bin + merge-multiple: true + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.x" + + - name: Install build tools + run: | + python -m pip install --upgrade build twine + + - name: Build wheel and sdist + working-directory: glitch/rego/rego_python + run: python -m build + + - name: Upload package artifacts + uses: actions/upload-artifact@v4 + with: + name: python-package + path: glitch/rego/rego_python/dist/* + retention-days: 1 + + release: + name: Create GitHub Release + runs-on: ubuntu-latest + needs: build-package + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Extract version and check if tag exists + id: get_version + run: | + VERSION=$(grep '^version = ' glitch/rego/rego_python/pyproject.toml | sed -E 's/version = "(.*)"/\1/') + echo "version=$VERSION" >> $GITHUB_OUTPUT + echo "Version extracted: $VERSION" + + TAG=rego_python-v$VERSION + if git rev-parse "$TAG" >/dev/null 2>&1; then + echo "Tag $TAG already exists, skipping release" + echo "tag_exists=true" >> $GITHUB_OUTPUT + else + echo "Tag $TAG does not exist, will create release" + echo "tag_exists=false" >> $GITHUB_OUTPUT + fi + + - name: Create tag for release + if: steps.get_version.outputs.tag_exists == 'false' + run: | + git config user.name "rego_python-release-bot" + git config user.email "rego_python@users.noreply.github.com" + TAG=rego_python-v${{ steps.get_version.outputs.version }} + git tag $TAG + git push origin $TAG + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Download built package and binaries + if: steps.get_version.outputs.tag_exists == 'false' + uses: actions/download-artifact@v4 + with: + path: release_assets + merge-multiple: true + + - name: List downloaded assets + if: steps.get_version.outputs.tag_exists == 'false' + run: | + echo "Listing contents of release_assets directory:" + ls -lR release_assets + + - name: Create GitHub Release + if: steps.get_version.outputs.tag_exists == 'false' + uses: softprops/action-gh-release@v2 + with: + tag_name: "rego_python-v${{ steps.get_version.outputs.version }}" + name: "Rego Python v${{ steps.get_version.outputs.version }}" + files: | + release_assets/*.whl + release_assets/*.tar.gz + release_assets/*.so + release_assets/*.dylib + release_assets/*.dll + generate_release_notes: false + draft: false + prerelease: false + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index c5ed5a05..a58bd8c4 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -15,6 +15,16 @@ jobs: - uses: ruby/setup-ruby@v1 with: ruby-version: '2.7.4' + - name: Set up Go + uses: actions/setup-go@v4 + with: + go-version: 'stable' + cache-dependency-path: glitch/rego/rego_python/src/rego_python/go/go.sum + - name: Build Rego library + run: | + cd glitch/rego/rego_python/src/rego_python/go + GOOS=linux GOARCH=amd64 go build -o "../bin/librego-linux-amd64.so" -buildmode=c-shared regolib.go + rm -f ../bin/librego-*.h - name: Install Python 3 uses: actions/setup-python@v4 with: @@ -23,7 +33,6 @@ jobs: run: | python -m pip install --upgrade pip python -m pip install -e . + python -m pip install pytest - name: Run tests with pytest - run: | - cd glitch - python -m unittest discover tests \ No newline at end of file + run: python -m pytest tests \ No newline at end of file diff --git a/.github/workflows/vscode-extension.yml b/.github/workflows/vscode-extension.yml new file mode 100644 index 00000000..758c4279 --- /dev/null +++ b/.github/workflows/vscode-extension.yml @@ -0,0 +1,59 @@ +name: Publish VSCode Extension + +on: + push: + branches: + - main + - nuno/test/workflow + workflow_dispatch: + +jobs: + publish: + runs-on: ubuntu-latest + defaults: + run: + working-directory: vscode-extension/glitch + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 2 + + - name: Setup Node.js + uses: actions/setup-node@v3 + with: + node-version: '20' + + - name: Check if version changed + id: version-check + run: | + npm install -g @vscode/vsce + + LOCAL_VERSION=$(node -p "require('./package.json').version") + PUBLISHER=$(node -p "require('./package.json').publisher") + NAME=$(node -p "require('./package.json').name") + + MARKETPLACE_VERSION=$(vsce show "${PUBLISHER}.${NAME}" --json 2>/dev/null | node -p "JSON.parse(require('fs').readFileSync(0, 'utf8')).versions[0].version" 2>/dev/null || echo "0.0.0") + + echo "local=$LOCAL_VERSION" >> $GITHUB_OUTPUT + echo "marketplace=$MARKETPLACE_VERSION" >> $GITHUB_OUTPUT + + if [ "$LOCAL_VERSION" != "$MARKETPLACE_VERSION" ]; then + echo "changed=true" >> $GITHUB_OUTPUT + echo "Local version ($LOCAL_VERSION) differs from marketplace ($MARKETPLACE_VERSION)" + else + echo "changed=false" >> $GITHUB_OUTPUT + echo "Version unchanged ($LOCAL_VERSION)" + fi + + - name: Install dependencies + if: steps.version-check.outputs.changed == 'true' + run: npm ci + + - name: Compile extension + if: steps.version-check.outputs.changed == 'true' + run: npm run compile + + - name: Publish to Visual Studio Marketplace + if: steps.version-check.outputs.changed == 'true' + run: npx vsce publish -p ${{ secrets.VSCE_PAT }} diff --git a/.gitignore b/.gitignore index 6b0d3eb5..d7a3afbd 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,8 @@ __pycache__/ # C extensions *.so +glitch/rego/go/library/librego.h +glitch/rego/rego_python/src/rego_python/bin/* # Distribution / packaging .Python @@ -130,3 +132,4 @@ dmypy.json out errors +.vscode diff --git a/README.md b/README.md index 067ea0bd..06b346b4 100644 --- a/README.md +++ b/README.md @@ -58,6 +58,58 @@ To install GLITCH using Poetry, run: poetry install ``` +### Rego + +Some smell checks (design and security) are implemented in [Rego](https://www.openpolicyagent.org/docs/latest/policy-language/). To use these checks, you need the appropriate Rego binary for your platform. + +#### Option 1: Download Pre-built Binary (Recommended) + +Download the binary for your platform from the [Rego Python release](https://github.com/sr-lab/GLITCH/releases). + +Available binaries: +| Platform | Architecture | Binary Name | +|----------|--------------|-------------| +| Linux | x86_64/amd64 | `librego-linux-amd64.so` | +| Linux | arm64 | `librego-linux-arm64.so` | +| macOS | Intel (x86_64) | `librego-darwin-amd64.dylib` | +| macOS | Apple Silicon | `librego-darwin-arm64.dylib` | +| Windows | x86_64/amd64 | `librego-windows-amd64.dll` | + +After downloading, place the binary in: +``` +glitch/rego/rego_python/src/rego_python/bin/ +``` + +#### Option 2: Build from Source + +If you need a binary not listed above or prefer to build from source, you need [Go](https://go.dev/doc/install) installed. + +```bash +cd glitch/rego/rego_python/src/rego_python/go + +# Set your target OS and architecture +OS=linux # Options: linux, darwin, windows +ARCH=amd64 # Options: amd64, arm64 + +# Determine file extension +if [ "$OS" = "windows" ]; then + EXT="dll" +elif [ "$OS" = "darwin" ]; then + EXT="dylib" +else + EXT="so" +fi + +GOOS=$OS GOARCH=$ARCH go build -o "../bin/librego-$OS-$ARCH.$EXT" -buildmode=c-shared regolib.go +``` + +#### Verifying Rego is Working + +To verify the Rego binary is correctly installed, run: +```bash +python -c "from glitch.rego.rego_python.src.rego_python import run_rego; print('Rego is working!')" +``` + **WARNING**: _For now, the GLITCH VSCode extension does not function if GLITCH is installed via Poetry. Since Poetry uses virtual environments it does not create a binary for GLITCH available in the user's PATH, which is required for @@ -75,12 +127,12 @@ Please read `usage` > `Docker` To explore all available options, use the command: ``` -glitch --help +glitch lint --help ``` To analyze a file or folder and retrieve CSV results, use the following command: ``` -glitch --tech (chef|puppet|ansible|terraform) --csv --config PATH_TO_CONFIG PATH_TO_FILE_OR_FOLDER +glitch lint --tech (chef|puppet|ansible|terraform) --csv --config PATH_TO_CONFIG PATH_TO_FILE_OR_FOLDER ``` If you want to consider the module structure you can add the flag ```--module```. @@ -89,13 +141,13 @@ If you want to consider the module structure you can add the flag ```--module``` If GLITCH was installed using Poetry, execute GLITCH commands as follows: ``` -poetry run glitch --help +poetry run glitch lint --help ``` Alternatively, you can use `poetry shell`: ``` poetry shell -glitch --help +glitch lint --help ``` ### Docker @@ -112,9 +164,9 @@ docker run --rm -v /Users/user/.../project:/glitch:ro glitch --tech terraform . ## Tests -To run the tests for GLITCH go to the folder ```glitch``` and run: +To run the tests for GLITCH run the following command: ``` -python -m unittest discover tests +poetry run pytest -s ``` ## Configs diff --git a/glitch/__main__.py b/glitch/__main__.py index 34aa53f7..97118f1f 100644 --- a/glitch/__main__.py +++ b/glitch/__main__.py @@ -1,12 +1,12 @@ import json import tqdm +import functools import click, os, sys from pathlib import Path -from typing import Tuple, List, Set, Optional, TextIO, Dict +from typing import Tuple, List, Set, Optional, TextIO, Dict, Any from glitch.analysis.rules import Error, RuleVisitor -from glitch.helpers import get_smell_types, get_smells -from glitch.parsers.docker import DockerParser +from glitch.helpers import get_smell_types, get_smells, ini_to_json_dict from glitch.stats.print import print_stats from glitch.stats.stats import FileStats from glitch.tech import Tech @@ -18,15 +18,22 @@ from glitch.parsers.terraform import TerraformParser from glitch.parsers.gha import GithubActionsParser from glitch.exceptions import throw_exception -from pkg_resources import resource_filename +from glitch.repair.interactive.main import run_infrafix +from importlib.resources import files from copy import deepcopy from concurrent.futures import ThreadPoolExecutor, Future, as_completed +from glitch.rego.engine import load_rego_from_path, run_analyses +from glitch.rego.rego_python.src.rego_python import is_rego_available, get_rego_error # NOTE: These are necessary in order for python to load the visitors. # Otherwise, python will not consider these types of rules. from glitch.analysis.design.visitor import DesignVisitor # type: ignore -from glitch.analysis.security import SecurityVisitor # type: ignore +from glitch.analysis.security.visitor import SecurityVisitor # type: ignore + + +def __get_resource_path(resource: str) -> str: + return str(files("glitch").joinpath(resource)) def __parse_and_check( @@ -36,21 +43,76 @@ def __parse_and_check( parser: Parser, analyses: List[RuleVisitor], stats: FileStats, + config_rego: Dict[str, Dict[str, List[str]]], + rego_modules: Dict[str, str], ) -> Set[Error]: errors: Set[Error] = set() inter = parser.parse(path, type, module) # Avoids problems with multiple threads (and possibly multiple files) # sharing the same object - analyses = deepcopy(analyses) + analyses = deepcopy(analyses) if inter != None: for analysis in analyses: errors.update(analysis.check(inter)) + + inputRego = json.dumps(inter.as_dict(), indent=2) + + errors.update(run_analyses(inputRego, config_rego, rego_modules)) + stats.compute(inter) return errors +def __filter_analysis( + smell_types: Tuple[str, ...], config: str, tech: Tech +) -> Tuple[Dict[str, str], List[RuleVisitor]]: + rego_modules: Dict[str, str] = {} + python_analyses: List[RuleVisitor] = [] + + rego_lib_path = __get_resource_path("rego/queries/library/glitch_lib.rego") + if not os.path.exists(rego_lib_path): + raise FileNotFoundError("The rego query library does not exist.") + load_rego_from_path(rego_lib_path, rego_modules) + + for smell_type in smell_types: + smells: List[str] = get_smells([smell_type], tech) + fallback: Set[str] = set() + + for smell in smells: + rego_path = __get_resource_path(f"rego/queries/{smell_type}/{smell}.rego") + if os.path.exists(rego_path): + load_rego_from_path(rego_path, rego_modules) + else: + fallback.add(smell) + + if len(fallback) > 0: + match smell_type: + case "design": + visitor = DesignVisitor(tech, fallback) + case "security": + visitor = SecurityVisitor(tech, fallback) + case _: + raise ValueError(f"Invalid smell type: {smell_type}") + + visitor.config(config) + python_analyses.append(visitor) + + return rego_modules, python_analyses + + +def __get_tech(tech: str) -> Tech: + for t in Tech: + if t.tech == tech: + return t + + raise click.BadOptionUsage( + "tech", + f"Invalid value for 'tech': '{tech}' is not a valid technology.", + ) + + def __print_errors(errors: Set[Error], f: TextIO, linter: bool, csv: bool) -> None: errors_sorted = sorted(errors, key=lambda e: (e.path, e.line, e.code)) if linter: @@ -71,8 +133,6 @@ def __get_parser(tech: Tech) -> Parser: return ChefParser() elif tech == Tech.puppet: return PuppetParser() - elif tech == Tech.docker: - return DockerParser() elif tech == Tech.terraform: return TerraformParser() elif tech == Tech.gha: @@ -111,26 +171,38 @@ def __get_paths_and_title( return paths, title -def repr_mode( - type: UnitBlockType, - path: str, - module: bool, - parser: Parser, -) -> None: - inter = parser.parse(path, type, module) - if inter != None: - print(json.dumps(inter.as_dict(), indent=2)) - - -@click.command( - help="PATH is the file or folder to analyze. OUTPUT is an optional file to which we can redirect the smells output." -) -@click.option( - "--tech", - type=click.Choice([t.tech for t in Tech]), - required=True, - help="The IaC technology to be considered.", +def __common_params(func: Any) -> Any: + @click.option( + "--tech", + type=click.Choice([t.tech for t in Tech]), + required=True, + help="The IaC technology to be considered.", + ) + @click.option( + "--type", + type=click.Choice([t.value for t in UnitBlockType]), + default=UnitBlockType.unknown, + help="The type of the scripts being analyzed.", + ) + @click.argument("path", type=click.Path(exists=True), required=True) + @functools.wraps(func) + def wrapper(*args: Any, **kwargs: Any): + return func(*args, **kwargs) + + return wrapper + + +@click.group() +def cli(): + pass + + +@cli.command( + help="Lint the IaC scripts in the given PATH.\n" + "PATH is the file or folder to analyze.\n" + "OUTPUT is an optional file to which we can redirect the smells output." ) +@__common_params @click.option( "--table-format", type=click.Choice(("prettytable", "latex")), @@ -138,12 +210,6 @@ def repr_mode( default="prettytable", help="The presentation format of the tables that summarize the run.", ) -@click.option( - "--type", - type=click.Choice([t.value for t in UnitBlockType]), - default=UnitBlockType.unknown, - help="The type of the scripts being analyzed.", -) @click.option( "--config", type=click.Path(), @@ -180,22 +246,14 @@ def repr_mode( multiple=True, help="The type of smell_types being analyzed.", ) -@click.option( - "--mode", - type=click.Choice(["smell_detector", "repr"]), - help="The mode the tool is running in. If the mode is 'repr', the output is the intermediate representation." - "Defaults to 'smell_detector'.", - default="smell_detector", -) @click.option( "--n-workers", type=int, help="Number of parallel workers to use. Defaults to 1.", default=1, ) -@click.argument("path", type=click.Path(exists=True), required=True) @click.argument("output", type=click.Path(), required=False) -def glitch( +def lint( tech: str, # type: ignore type: str, path: str, @@ -206,22 +264,22 @@ def glitch( output: Optional[str], table_format: str, linter: bool, - mode: str, n_workers: int, ): - for t in Tech: - if t.tech == tech: - tech: Tech = t - break - else: - raise click.BadOptionUsage( - "tech", - f"Invalid value for 'tech': '{tech}' is not a valid technology.", - ) - + tech: Tech = __get_tech(tech) type = UnitBlockType(type) module = folder_strategy == "module" + if not is_rego_available(): + click.echo( + f"Error: Rego library is not available. {get_rego_error()}", err=True + ) + click.echo( + "Please build or install the Rego library. See README for instructions.", + err=True, + ) + sys.exit(1) + if config != "configs/default.ini" and not os.path.exists(config): raise click.BadOptionUsage( "config", f"Invalid value for 'config': Path '{config}' does not exist." @@ -231,29 +289,19 @@ def glitch( "config", f"Invalid value for 'config': Path '{config}' should be a file." ) elif config == "configs/default.ini": - config_path = resource_filename("glitch", "configs/default.ini") - else: - config_path = config + config = __get_resource_path("configs/default.ini") parser = __get_parser(tech) if tech == Tech.terraform: - config_path = resource_filename("glitch", "configs/terraform.ini") + config = __get_resource_path("configs/terraform.ini") file_stats = FileStats() - if mode == "repr": - repr_mode(type, path, module, parser) - return - if smell_types == (): smell_types = get_smell_types() - analyses: List[RuleVisitor] = [] - rules = RuleVisitor.__subclasses__() - for r in rules: - if smell_types == () or r.get_name() in smell_types: - analysis = r(tech) - analysis.config(config_path) - analyses.append(analysis) + config_rego = ini_to_json_dict(config) + + rego_modules, analyses = __filter_analysis(smell_types, config, tech) errors: List[Error] = [] paths: Set[str] @@ -266,12 +314,22 @@ def glitch( for p in paths: futures.append( executor.submit( - __parse_and_check, type, p, module, parser, analyses, file_stats + __parse_and_check, + type, + p, + module, + parser, + analyses, + file_stats, + config_rego, + rego_modules, ) ) future_to_path[futures[-1]] = p f = sys.stdout if output is None else open(output, "w") + if csv: + print("PATH,LINE,ERROR,DESCRIPTION,CODE", file=f) for future in tqdm.tqdm(as_completed(futures), total=len(futures), desc=title): try: new_errors = future.result() @@ -286,8 +344,43 @@ def glitch( print_stats(errors, get_smells(smell_types, tech), file_stats, table_format) +@cli.command() +@__common_params +@click.argument("pid", type=str, required=True) +def infrafix( + path: str, + pid: str, + tech: str, # type: ignore + type: UnitBlockType, +): + tech: Tech = __get_tech(tech) + parser = __get_parser(tech) + run_infrafix(path, pid, parser, type, tech) + + +@cli.command() +@__common_params +@click.option( + "--module", + is_flag=True, + default=False, + help="True if the path is a module, false otherwise.", +) +def repr( + path: str, + type: UnitBlockType, + tech: str, # type: ignore + module: bool, +) -> None: + tech: Tech = __get_tech(tech) + parser = __get_parser(tech) + inter = parser.parse(path, type, module) + if inter != None: + print(json.dumps(inter.as_dict(), indent=2)) + + def main() -> None: - glitch(prog_name="glitch") + cli(prog_name="glitch") if __name__ == "__main__": diff --git a/glitch/tests/__init__.py b/glitch/analysis/checkers/__init__.py similarity index 100% rename from glitch/tests/__init__.py rename to glitch/analysis/checkers/__init__.py diff --git a/glitch/analysis/checkers/string_checker.py b/glitch/analysis/checkers/string_checker.py new file mode 100644 index 00000000..9b9a601e --- /dev/null +++ b/glitch/analysis/checkers/string_checker.py @@ -0,0 +1,12 @@ +from glitch.analysis.checkers.transverse_checker import TransverseChecker +from typing import Callable + +from glitch.repr.inter import String + + +class StringChecker(TransverseChecker): + def __init__(self, str_check: Callable[[str], bool]) -> None: + self.str_check = str_check + + def check_string(self, expr: String) -> bool: + return self.str_check(expr.value) diff --git a/glitch/analysis/checkers/transverse_checker.py b/glitch/analysis/checkers/transverse_checker.py new file mode 100644 index 00000000..b6904ab5 --- /dev/null +++ b/glitch/analysis/checkers/transverse_checker.py @@ -0,0 +1,96 @@ +from abc import ABC +from glitch.analysis.rules import Checker + +from glitch.repr.inter import ( + Array, + BinaryOperation, + ConditionalStatement, + FunctionCall, + Hash, + MethodCall, + UnaryOperation, + BlockExpr, + Undef, + KeyValue, + Block, + AtomicUnit, + UnitBlock, +) + + +class TransverseChecker(Checker, ABC): + def check_array(self, expr: Array) -> bool: + for v in expr.value: + if self.check(v): + return True + return False + + def check_hash(self, expr: Hash) -> bool: + for k, v in expr.value.items(): + if self.check(k) or self.check(v): + return True + return False + + def check_function_call(self, expr: FunctionCall) -> bool: + for arg in expr.args: + if self.check(arg): + return True + return False + + def check_method_call(self, expr: MethodCall) -> bool: + if self.check(expr.receiver): + return True + for arg in expr.args: + if self.check(arg): + return True + return False + + def check_unary_operation(self, expr: UnaryOperation) -> bool: + return self.check(expr.expr) + + def check_binary_operation(self, expr: BinaryOperation) -> bool: + return self.check(expr.left) or self.check(expr.right) + + def check_conditional_statement(self, expr: ConditionalStatement) -> bool: + return ( + self.check_block(expr) + or self.check(expr.condition) + or (expr.else_statement is not None and self.check(expr.else_statement)) + ) + + def check_blockexpr(self, element: BlockExpr) -> bool: + for stmt in element.statements: + if self.check(stmt): + return True + return False + + def check_undef(self, expr: Undef) -> bool: + return False + + def check_keyvalue(self, element: KeyValue) -> bool: + return self.check(element.value) + + def check_block(self, element: Block) -> bool: + if isinstance(element, UnitBlock): + for stmt in element.variables: + if self.check(stmt): + return True + for stmt in element.atomic_units: + if self.check(stmt): + return True + for stmt in element.unit_blocks: + if self.check(stmt): + return True + + for stmt in element.statements: + if self.check(stmt): + return True + return False + + def check_atomicunit(self, element: AtomicUnit) -> bool: + if self.check(element.name): + return True + for attr in element.attributes: + if self.check(attr): + return True + return self.check_block(element) diff --git a/glitch/analysis/checkers/var_checker.py b/glitch/analysis/checkers/var_checker.py new file mode 100644 index 00000000..b0f66c68 --- /dev/null +++ b/glitch/analysis/checkers/var_checker.py @@ -0,0 +1,7 @@ +from glitch.analysis.checkers.transverse_checker import TransverseChecker +from glitch.repr.inter import VariableReference + + +class VariableChecker(TransverseChecker): + def check_var_reference(self, expr: VariableReference) -> bool: + return True diff --git a/glitch/analysis/design/duplicate_block.py b/glitch/analysis/design/duplicate_block.py index 1def6026..afab6465 100644 --- a/glitch/analysis/design/duplicate_block.py +++ b/glitch/analysis/design/duplicate_block.py @@ -4,7 +4,7 @@ from glitch.repr.inter import * -class UnguardedVariable(DesignSmellChecker): +class DuplicateBlock(DesignSmellChecker): def __get_line(self, i: int, lines: List[Tuple[int, int]]): for j, line in lines: if i < j: diff --git a/glitch/analysis/design/imperative_abstraction.py b/glitch/analysis/design/imperative_abstraction.py index f58c5840..2963ad75 100644 --- a/glitch/analysis/design/imperative_abstraction.py +++ b/glitch/analysis/design/imperative_abstraction.py @@ -4,6 +4,9 @@ from glitch.analysis.design.smell_checker import DesignSmellChecker from glitch.repr.inter import * +# Deprecated: This rule has been rewritten in Rego. +# This is kept for reference and possible future use since there are no tests for the Rego version yet. + class ImperativeAbstraction(DesignSmellChecker): def __count_atomic_units(self, ub: UnitBlock) -> Tuple[int, int]: diff --git a/glitch/analysis/design/improper_alignment.py b/glitch/analysis/design/improper_alignment.py index d1536f25..571e64f2 100644 --- a/glitch/analysis/design/improper_alignment.py +++ b/glitch/analysis/design/improper_alignment.py @@ -31,8 +31,9 @@ def ignore_techs() -> List[Tech]: def check(self, element: CodeElement, file: str) -> List[Error]: if isinstance(element, AtomicUnit): identation = None + for a in element.attributes: - first_line = a.code.split("\n")[0] + first_line = self.code_lines[a.line - 1] curr_id = len(first_line) - len(first_line.lstrip()) if identation is None: diff --git a/glitch/analysis/design/long_resource.py b/glitch/analysis/design/long_resource.py deleted file mode 100644 index 2cd5bbad..00000000 --- a/glitch/analysis/design/long_resource.py +++ /dev/null @@ -1,20 +0,0 @@ -from typing import List -from glitch.analysis.rules import Error -from glitch.analysis.design.smell_checker import DesignSmellChecker -from glitch.analysis.design.visitor import DesignVisitor -from glitch.repr.inter import * - - -class TooManyVariables(DesignSmellChecker): - def check(self, element: CodeElement, file: str) -> List[Error]: - if isinstance(element, AtomicUnit) and element.type in DesignVisitor.EXEC: - lines = 0 - for attr in element.attributes: - for line in attr.code.split("\n"): - if line.strip() != "": - lines += 1 - - if lines > 7: - return [Error("design_long_resource", element, file, repr(element))] - - return [] diff --git a/glitch/analysis/design/misplaced_attribute.py b/glitch/analysis/design/misplaced_attribute.py deleted file mode 100644 index eaed5af2..00000000 --- a/glitch/analysis/design/misplaced_attribute.py +++ /dev/null @@ -1,64 +0,0 @@ -from typing import List, Optional -from glitch.analysis.rules import Error -from glitch.analysis.design.smell_checker import DesignSmellChecker -from glitch.tech import Tech -from glitch.repr.inter import * - - -class ChefMisplacedAttribute(DesignSmellChecker): - @staticmethod - def tech() -> Optional[Tech]: - return Tech.chef - - def check(self, element: CodeElement, file: str) -> List[Error]: - if isinstance(element, AtomicUnit): - order: List[int] = [] - for attribute in element.attributes: - if attribute.name == "source": - order.append(1) - elif attribute.name in ["owner", "group"]: - order.append(2) - elif attribute.name == "mode": - order.append(3) - elif attribute.name == "action": - order.append(4) - - if order != sorted(order): - return [ - Error("design_misplaced_attribute", element, file, repr(element)) - ] - return [] - - -class PuppetMisplacedAttribute(DesignSmellChecker): - @staticmethod - def tech() -> Optional[Tech]: - return Tech.puppet - - def check(self, element: CodeElement, file: str) -> List[Error]: - if isinstance(element, AtomicUnit): - for i, attr in enumerate(element.attributes): - if attr.name == "ensure" and i != 0: - return [ - Error( - "design_misplaced_attribute", - element, - file, - repr(element), - ) - ] - elif isinstance(element, UnitBlock): - optional = False - for attr in element.attributes: - if attr.value is not None: - optional = True - elif optional: - return [ - Error( - "design_misplaced_attribute", - element, - file, - repr(element), - ) - ] - return [] diff --git a/glitch/analysis/design/multifaceted_abstraction.py b/glitch/analysis/design/multifaceted_abstraction.py deleted file mode 100644 index c7f0bc00..00000000 --- a/glitch/analysis/design/multifaceted_abstraction.py +++ /dev/null @@ -1,32 +0,0 @@ -from typing import List -from glitch.analysis.rules import Error -from glitch.analysis.design.smell_checker import DesignSmellChecker -from glitch.analysis.design.visitor import DesignVisitor -from glitch.repr.inter import * - - -class TooManyVariables(DesignSmellChecker): - def check(self, element: CodeElement, file: str) -> List[Error]: - if isinstance(element, AtomicUnit) and element.type in DesignVisitor.EXEC: - if isinstance(element.name, str) and ( - "&&" in element.name or ";" in element.name or "|" in element.name - ): - return [ - Error( - "design_multifaceted_abstraction", element, file, repr(element) - ) - ] - else: - for attribute in element.attributes: - value = repr(attribute.value) - if "&&" in value or ";" in value or "|" in value: - return [ - Error( - "design_multifaceted_abstraction", - element, - file, - repr(element), - ) - ] - - return [] diff --git a/glitch/analysis/design/too_many_variables.py b/glitch/analysis/design/too_many_variables.py deleted file mode 100644 index 78a0ff4d..00000000 --- a/glitch/analysis/design/too_many_variables.py +++ /dev/null @@ -1,32 +0,0 @@ -from typing import List -from glitch.analysis.rules import Error -from glitch.analysis.design.smell_checker import DesignSmellChecker -from glitch.repr.inter import * - - -class TooManyVariables(DesignSmellChecker): - def __count_variables(self, vars: List[KeyValue]) -> int: - count = 0 - for var in vars: - if isinstance(var.value, type(None)): - count += self.__count_variables(var.keyvalues) - else: - count += 1 - return count - - def check(self, element: CodeElement, file: str) -> List[Error]: - if isinstance(element, UnitBlock) and element.type != UnitBlockType.vars: - # The UnitBlock should not be of type vars, because these files are supposed to only - # have variables - if ( - self.__count_variables(element.variables) / max(len(self.code_lines), 1) > 0.3 # type: ignore - ): - return [ - Error( - "implementation_too_many_variables", - element, - file, - repr(element), - ) - ] - return [] diff --git a/glitch/analysis/design/visitor.py b/glitch/analysis/design/visitor.py index d946535b..6bd4ac10 100644 --- a/glitch/analysis/design/visitor.py +++ b/glitch/analysis/design/visitor.py @@ -5,16 +5,41 @@ from glitch.analysis.rules import Error, RuleVisitor from glitch.tech import Tech from glitch.repr.inter import * -from typing import List +from typing import List, Type from glitch.analysis.design.smell_checker import DesignSmellChecker class DesignVisitor(RuleVisitor): - def __init__(self, tech: Tech) -> None: + def __init__(self, tech: Tech, fallback: set[str]) -> None: super().__init__(tech) + from glitch.analysis.design.unguarded_variable import UnguardedVariable + from glitch.analysis.design.duplicate_block import DuplicateBlock + from glitch.analysis.design.imperative_abstraction import ImperativeAbstraction + from glitch.analysis.design.improper_alignment import ( + ImproperAlignmentTabs, + ImproperAlignment, + PuppetImproperAlignment, + ) + from glitch.analysis.design.long_statement import LongStatement + + DESIGN_CHECKER_ERRORS: Dict[Type[DesignSmellChecker], str] = { + UnguardedVariable: "implementation_unguarded_variable", + DuplicateBlock: "design_duplicate_block", + ImperativeAbstraction: "design_imperative_abstraction", + ImproperAlignmentTabs: "implementation_improper_alignment", + ImproperAlignment: "implementation_improper_alignment", + PuppetImproperAlignment: "implementation_improper_alignment", + LongStatement: "implementation_long_statement", + } + self.checkers: List[DesignSmellChecker] = [] for child in DesignSmellChecker.__subclasses__(): + error_name = DESIGN_CHECKER_ERRORS.get(child) + + if error_name is not None and error_name not in fallback: + continue + if (child.tech() is None and tech not in child.ignore_techs()) or ( child.tech() is not None and child.tech() == tech ): @@ -28,6 +53,7 @@ def __init__(self, tech: Tech) -> None: self.variable_stack: List[int] = [] self.variables_names: List[str] = [] self.first_code_line = inf + self.code_lines: List[str] = [] @staticmethod def get_name() -> str: @@ -58,18 +84,12 @@ def check_unitblock(self, u: UnitBlock, file: str) -> List[Error]: if u.path != "": with open(u.path, "r") as f: try: - code_lines = f.readlines() + self.code_lines = f.readlines() f.seek(0, 0) except UnicodeDecodeError: return [] else: - code_lines = [] - - self.first_non_comm_line = inf - for i, line in enumerate(code_lines): - if not line.startswith(self.comment): - self.first_non_comm_line = i + 1 - break + self.code_lines = [] self.variable_stack.append(len(self.variables_names)) for attr in u.attributes: @@ -97,7 +117,7 @@ def check_unitblock(self, u: UnitBlock, file: str) -> List[Error]: # errors.append(Error('design_unnecessary_abstraction', u, file, repr(u))) for checker in self.checkers: - checker.code_lines = code_lines + checker.code_lines = self.code_lines checker.variables_names = self.variables_names errors += checker.check(u, file) @@ -120,6 +140,7 @@ def check_condition(self, c: ConditionalStatement, file: str) -> list[Error]: def check_atomicunit(self, au: AtomicUnit, file: str) -> list[Error]: errors = super().check_atomicunit(au, file) for checker in self.checkers: + checker.code_lines = self.code_lines errors += checker.check(au, file) return errors @@ -134,10 +155,7 @@ def check_variable(self, v: Variable, file: str) -> list[Error]: return [] def check_comment(self, c: Comment, file: str) -> list[Error]: - errors: List[Error] = [] - if c.line >= self.first_non_comm_line: - errors.append(Error("design_avoid_comments", c, file, repr(c))) - return errors + return [] # NOTE: in the end of the file to avoid circular import diff --git a/glitch/analysis/rules.py b/glitch/analysis/rules.py index ff8f3354..4f351e56 100644 --- a/glitch/analysis/rules.py +++ b/glitch/analysis/rules.py @@ -2,6 +2,7 @@ from abc import ABC, abstractmethod from glitch.tech import Tech from glitch.repr.inter import * +from glitch.repr.inter import UNDEFINED_POSITION ErrorValue = Dict[Tech | str, Dict[str, str] | str] ErrorDict = Dict[str, ErrorValue] @@ -23,9 +24,6 @@ class Error: "sec_no_default_switch": "Missing default case statement - Not handling every possible input combination might allow an attacker to trigger an error for an unhandled value. (CWE-478)", "sec_full_permission_filesystem": "Full permission to the filesystem - Files should not have full permissions to every user. (CWE-732)", "sec_obsolete_command": "Use of obsolete command or function - Avoid using obsolete or deprecated commands and functions. (CWE-477)", - Tech.docker: { - "sec_non_official_image": "Use of non-official Docker image - Use of non-official images should be avoided or taken into careful consideration. (CWE-829)", - }, Tech.terraform: { "sec_integrity_policy": "Integrity Policy - Image tag is prone to be mutable or integrity monitoring is disabled. (CWE-471)", "sec_ssl_tls_policy": "SSL/TLS/mTLS Policy - Developers should use SSL/TLS/mTLS protocols and their secure versions. (CWE-326)", @@ -88,26 +86,42 @@ def __init__( self.opt_msg = opt_msg if isinstance(self.el, CodeElement): - self.line = self.el.line + if ( + self.code == "sec_no_default_switch" + and type(self.el) is ConditionalStatement + ): + # Special case: use line number of condition + # TODO: improve this hack for edge cases on parsing errors + # self.line = self.el.condition.line + # In the ideal case, we would use the line of the condition but this can break in some cases + # Temporary fix: use line of case - 1 + self.line = self.el.line - 1 + else: + self.line = self.el.line else: self.line = -1 def to_csv(self) -> str: repr = self.repr.split("\n")[0].strip() if self.opt_msg: - return f"{self.path},{self.line},{self.code},{repr},{self.opt_msg}" + return f"{self.path},{self.line},{self.code},{self.opt_msg},{repr}" else: - return f"{self.path},{self.line},{self.code},{repr},-" + return f"{self.path},{self.line},{self.code},-,{repr}" def __repr__(self) -> str: with open(self.path) as f: line = ( f.readlines()[self.line - 1].strip() - if self.line != -1 + if self.line != UNDEFINED_POSITION else self.repr.split("\n")[0] ) if self.opt_msg: line += f"\n-> {self.opt_msg}" + + if self.line == UNDEFINED_POSITION: + return ( + f"{self.path}\nIssue: {Error.ALL_ERRORS[self.code]}\n" + f"{line}\n" + ) return ( f"{self.path}\nIssue on line {self.line}: {Error.ALL_ERRORS[self.code]}\n" + f"{line}\n" @@ -129,6 +143,122 @@ def __eq__(self, other: Any): Error.agglomerate_errors() +class Checker(ABC): + def check(self, element: CodeElement) -> bool: + if isinstance(element, String): + return self.check_string(element) + elif isinstance(element, Integer): + return self.check_integer(element) + elif isinstance(element, Float): + return self.check_float(element) + elif isinstance(element, Complex): + return self.check_complex(element) + elif isinstance(element, Boolean): + return self.check_boolean(element) + elif isinstance(element, Null): + return self.check_null(element) + elif isinstance(element, Undef): + return self.check_undef(element) + elif isinstance(element, Hash): + return self.check_hash(element) + elif isinstance(element, Array): + return self.check_array(element) + elif isinstance(element, VariableReference): + return self.check_var_reference(element) + elif isinstance(element, FunctionCall): + return self.check_function_call(element) + elif isinstance(element, MethodCall): + return self.check_method_call(element) + elif isinstance(element, UnaryOperation): + return self.check_unary_operation(element) + elif isinstance(element, BinaryOperation): + return self.check_binary_operation(element) + elif isinstance(element, ConditionalStatement): + return self.check_conditional_statement(element) + elif isinstance(element, BlockExpr): + return self.check_blockexpr(element) + elif isinstance(element, AddArgs): + for e in element.value: + if not self.check(e): + return False + return True + elif isinstance(element, KeyValue): + return self.check_keyvalue(element) + elif isinstance(element, Block): + return self.check_block(element) + elif isinstance(element, AtomicUnit): + return self.check_atomicunit(element) + elif isinstance(element, Comment): + return self.check_comment(element) + elif isinstance(element, Dependency): + return self.check_dependency(element) + else: + return False + + def check_string(self, expr: String) -> bool: + return False + + def check_integer(self, expr: Integer) -> bool: + return False + + def check_float(self, expr: Float) -> bool: + return False + + def check_complex(self, expr: Complex) -> bool: + return False + + def check_boolean(self, expr: Boolean) -> bool: + return False + + def check_null(self, expr: Null) -> bool: + return False + + def check_hash(self, expr: Hash) -> bool: + return False + + def check_array(self, expr: Array) -> bool: + return False + + def check_var_reference(self, expr: VariableReference) -> bool: + return False + + def check_function_call(self, expr: FunctionCall) -> bool: + return False + + def check_method_call(self, expr: MethodCall) -> bool: + return False + + def check_unary_operation(self, expr: UnaryOperation) -> bool: + return False + + def check_binary_operation(self, expr: BinaryOperation) -> bool: + return False + + def check_conditional_statement(self, expr: ConditionalStatement) -> bool: + return False + + def check_blockexpr(self, element: BlockExpr) -> bool: + return False + + def check_undef(self, expr: Undef) -> bool: + return False + + def check_keyvalue(self, element: KeyValue) -> bool: + return False + + def check_block(self, element: Block) -> bool: + return False + + def check_atomicunit(self, element: AtomicUnit) -> bool: + return False + + def check_comment(self, element: Comment) -> bool: + return False + + def check_dependency(self, element: Dependency) -> bool: + return False + + class RuleVisitor(ABC): def __init__(self, tech: Tech) -> None: super().__init__() @@ -243,9 +373,6 @@ def check_comment(self, c: Comment, file: str) -> list[Error]: pass -Error.agglomerate_errors() - - class SmellChecker(ABC): def __init__(self) -> None: self.code: Optional[Project | UnitBlock | Module] = None diff --git a/glitch/analysis/security.py b/glitch/analysis/security.py deleted file mode 100644 index e91416cc..00000000 --- a/glitch/analysis/security.py +++ /dev/null @@ -1,556 +0,0 @@ -import os -import re -import json -import glitch -import configparser -from urllib.parse import urlparse -from glitch.analysis.rules import Error, RuleVisitor, SmellChecker -from nltk.tokenize import WordPunctTokenizer # type: ignore -from typing import Tuple, List, Optional - -from glitch.tech import Tech -from glitch.repr.inter import * - -from glitch.analysis.terraform.smell_checker import TerraformSmellChecker - - -class SecurityVisitor(RuleVisitor): - __URL_REGEX = r"^(http:\/\/www\.|https:\/\/www\.|http:\/\/|https:\/\/)?[a-z0-9]+([_\-\.]{1}[a-z0-9]+)*\.[a-z]{2,5}(:[0-9]{1,5})?(\/.*)?$" - - class NonOfficialImageSmell(SmellChecker): - def check(self, element: CodeElement, file: str) -> List[Error]: - return [] - - class DockerNonOfficialImageSmell(SmellChecker): - def check(self, element: CodeElement, file: str) -> List[Error]: - if ( - not isinstance(element, UnitBlock) - or element.name is None - or "Dockerfile" in element.name - ): - return [] - image = element.name.split(":") - if image[0] not in SecurityVisitor._DOCKER_OFFICIAL_IMAGES: - return [Error("sec_non_official_image", element, file, repr(element))] - return [] - - def __init__(self, tech: Tech) -> None: - super().__init__(tech) - self.checkers: List[SmellChecker] = [] - - if tech == Tech.terraform: - for child in TerraformSmellChecker.__subclasses__(): - self.checkers.append(child()) - - if tech == Tech.docker: - self.non_off_img = SecurityVisitor.DockerNonOfficialImageSmell() - else: - self.non_off_img = SecurityVisitor.NonOfficialImageSmell() - - @staticmethod - def get_name() -> str: - return "security" - - def config(self, config_path: str) -> None: - config = configparser.ConfigParser() - config.read(config_path) - SecurityVisitor.__WRONG_WORDS = json.loads( - config["security"]["suspicious_words"] - ) - SecurityVisitor.__PASSWORDS = json.loads(config["security"]["passwords"]) - SecurityVisitor.__USERS = json.loads(config["security"]["users"]) - SecurityVisitor.__PROFILE = json.loads(config["security"]["profile"]) - SecurityVisitor.__SECRETS = json.loads(config["security"]["secrets"]) - SecurityVisitor.__MISC_SECRETS = json.loads(config["security"]["misc_secrets"]) - SecurityVisitor.__ROLES = json.loads(config["security"]["roles"]) - SecurityVisitor.__DOWNLOAD = json.loads( - config["security"]["download_extensions"] - ) - SecurityVisitor.__SSH_DIR = json.loads(config["security"]["ssh_dirs"]) - SecurityVisitor.__ADMIN = json.loads(config["security"]["admin"]) - SecurityVisitor.__CHECKSUM = json.loads(config["security"]["checksum"]) - SecurityVisitor.__CRYPT = json.loads(config["security"]["weak_crypt"]) - SecurityVisitor.__CRYPT_WHITELIST = json.loads( - config["security"]["weak_crypt_whitelist"] - ) - SecurityVisitor.__URL_WHITELIST = json.loads( - config["security"]["url_http_white_list"] - ) - SecurityVisitor.__SECRETS_WHITELIST = json.loads( - config["security"]["secrets_white_list"] - ) - SecurityVisitor.__SENSITIVE_DATA = json.loads( - config["security"]["sensitive_data"] - ) - SecurityVisitor.__SECRET_ASSIGN = json.loads( - config["security"]["secret_value_assign"] - ) - SecurityVisitor.__GITHUB_ACTIONS = json.loads( - config["security"]["github_actions_resources"] - ) - - if self.tech == Tech.terraform: - SecurityVisitor.INTEGRITY_POLICY = json.loads( - config["security"]["integrity_policy"] - ) - SecurityVisitor.HTTPS_CONFIGS = json.loads( - config["security"]["ensure_https"] - ) - SecurityVisitor.SSL_TLS_POLICY = json.loads( - config["security"]["ssl_tls_policy"] - ) - SecurityVisitor.DNSSEC_CONFIGS = json.loads( - config["security"]["ensure_dnssec"] - ) - SecurityVisitor.PUBLIC_IP_CONFIGS = json.loads( - config["security"]["use_public_ip"] - ) - SecurityVisitor.POLICY_KEYWORDS = json.loads( - config["security"]["policy_keywords"] - ) - SecurityVisitor.ACCESS_CONTROL_CONFIGS = json.loads( - config["security"]["insecure_access_control"] - ) - SecurityVisitor.AUTHENTICATION = json.loads( - config["security"]["authentication"] - ) - SecurityVisitor.POLICY_ACCESS_CONTROL = json.loads( - config["security"]["policy_insecure_access_control"] - ) - SecurityVisitor.POLICY_AUTHENTICATION = json.loads( - config["security"]["policy_authentication"] - ) - SecurityVisitor.MISSING_ENCRYPTION = json.loads( - config["security"]["missing_encryption"] - ) - SecurityVisitor.CONFIGURATION_KEYWORDS = json.loads( - config["security"]["configuration_keywords"] - ) - SecurityVisitor.ENCRYPT_CONFIG = json.loads( - config["security"]["encrypt_configuration"] - ) - SecurityVisitor.FIREWALL_CONFIGS = json.loads( - config["security"]["firewall"] - ) - SecurityVisitor.MISSING_THREATS_DETECTION_ALERTS = json.loads( - config["security"]["missing_threats_detection_alerts"] - ) - SecurityVisitor.PASSWORD_KEY_POLICY = json.loads( - config["security"]["password_key_policy"] - ) - SecurityVisitor.KEY_MANAGEMENT = json.loads( - config["security"]["key_management"] - ) - SecurityVisitor.NETWORK_SECURITY_RULES = json.loads( - config["security"]["network_security_rules"] - ) - SecurityVisitor.PERMISSION_IAM_POLICIES = json.loads( - config["security"]["permission_iam_policies"] - ) - SecurityVisitor.GOOGLE_IAM_MEMBER = json.loads( - config["security"]["google_iam_member_resources"] - ) - SecurityVisitor.LOGGING = json.loads(config["security"]["logging"]) - SecurityVisitor.GOOGLE_SQL_DATABASE_LOG_FLAGS = json.loads( - config["security"]["google_sql_database_log_flags"] - ) - SecurityVisitor.POSSIBLE_ATTACHED_RESOURCES = json.loads( - config["security"]["possible_attached_resources_aws_route53"] - ) - SecurityVisitor.VERSIONING = json.loads(config["security"]["versioning"]) - SecurityVisitor.NAMING = json.loads(config["security"]["naming"]) - SecurityVisitor.REPLICATION = json.loads(config["security"]["replication"]) - - SecurityVisitor.__FILE_COMMANDS = json.loads( - config["security"]["file_commands"] - ) - SecurityVisitor.__SHELL_RESOURCES = json.loads( - config["security"]["shell_resources"] - ) - SecurityVisitor.__IP_BIND_COMMANDS = json.loads( - config["security"]["ip_binding_commands"] - ) - SecurityVisitor.__OBSOLETE_COMMANDS = self._load_data_file("obsolete_commands") - SecurityVisitor._DOCKER_OFFICIAL_IMAGES = self._load_data_file( - "official_docker_images" - ) - - @staticmethod - def _load_data_file(file: str) -> List[str]: - folder_path = os.path.dirname(os.path.realpath(glitch.__file__)) - with open(os.path.join(folder_path, "files", file)) as f: - content = f.readlines() - return [c.strip() for c in content] - - def check_atomicunit(self, au: AtomicUnit, file: str) -> List[Error]: - errors = super().check_atomicunit(au, file) - - for item in SecurityVisitor.__FILE_COMMANDS: - if item not in au.type: - continue - for a in au.attributes: - values = [a.value] - for value in values: - if not isinstance(value, str): - continue - if a.name in ["mode", "m"] and re.search( - r"(?:^0?777$)|(?:(?:^|(?:ugo)|o|a)\+[rwx]{3})", value - ): - errors.append( - Error("sec_full_permission_filesystem", a, file, repr(a)) - ) - - for attribute in au.attributes: - if ( - au.type in SecurityVisitor.__GITHUB_ACTIONS - and attribute.name == "plaintext_value" - ): - errors.append(Error("sec_hard_secr", attribute, file, repr(attribute))) - - if au.type in SecurityVisitor.__OBSOLETE_COMMANDS: - errors.append(Error("sec_obsolete_command", au, file, repr(au))) - elif any(au.type.endswith(res) for res in SecurityVisitor.__SHELL_RESOURCES): - for attr in au.attributes: - if ( - isinstance(attr.value, str) - and attr.value.split(" ")[0] in SecurityVisitor.__OBSOLETE_COMMANDS - ): - errors.append(Error("sec_obsolete_command", attr, file, repr(attr))) - break - - for checker in self.checkers: - checker.code = self.code - errors += checker.check(au, file) - - if self.__is_http_url(au.name): - errors.append(Error("sec_https", au, file, repr(au))) - if self.__is_weak_crypt(au.type, au.name): - errors.append(Error("sec_weak_crypt", au, file, repr(au))) - - return errors - - def check_dependency(self, d: Dependency, file: str) -> List[Error]: - return [] - - def __check_keyvalue(self, c: KeyValue, file: str) -> List[Error]: - errors: List[Error] = [] - c.name = c.name.strip().lower() - - if isinstance(c.value, type(None)): - for child in c.keyvalues: - errors += self.check_element(child, file) - return errors - elif isinstance(c.value, str): # type: ignore - c.value = c.value.strip().lower() - else: - errors += self.check_element(c.value, file) - c.value = repr(c.value) - - if self.__is_http_url(c.value): - errors.append(Error("sec_https", c, file, repr(c))) - - if ( - re.match(r"(?:https?://|^)0.0.0.0", c.value) - or (c.name == "ip" and c.value in {"*", "::"}) - or ( - c.name in SecurityVisitor.__IP_BIND_COMMANDS - and (c.value == True or c.value in {"*", "::"}) # type: ignore - ) - ): - errors.append(Error("sec_invalid_bind", c, file, repr(c))) - - if self.__is_weak_crypt(c.value, c.name): - errors.append(Error("sec_weak_crypt", c, file, repr(c))) - - for check in SecurityVisitor.__CHECKSUM: - if check in c.name and (c.value == "no" or c.value == "false"): - errors.append(Error("sec_no_int_check", c, file, repr(c))) - break - - for item in SecurityVisitor.__ROLES + SecurityVisitor.__USERS: - if re.match(r"[_A-Za-z0-9$\/\.\[\]-]*{text}\b".format(text=item), c.name): - if len(c.value) > 0 and not c.has_variable: - for admin in SecurityVisitor.__ADMIN: - if admin in c.value: - errors.append(Error("sec_def_admin", c, file, repr(c))) - break - - def get_au( - c: Project | Module | UnitBlock | None, name: str, type: str - ) -> AtomicUnit | None: - if isinstance(c, Project): - module_name = os.path.basename(os.path.dirname(file)) - for m in c.modules: - if m.name == module_name: - return get_au(m, name, type) - elif isinstance(c, Module): - for ub in c.blocks: - au = get_au(ub, name, type) - if au is not None: - return au - elif isinstance(c, UnitBlock): - for au in c.atomic_units: - if au.type == type and au.name == name: - return au - return None - - def get_module_var( - c: Project | Module | UnitBlock | None, name: str - ) -> Variable | None: - if isinstance(c, Project): - module_name = os.path.basename(os.path.dirname(file)) - for m in c.modules: - if m.name == module_name: - return get_module_var(m, name) - elif isinstance(c, Module): - for ub in c.blocks: - var = get_module_var(ub, name) - if var is not None: - return var - elif isinstance(c, UnitBlock): - for var in c.variables: - if var.name == name: - return var - return None - - # only for terraform - var = None - if c.has_variable and self.tech == Tech.terraform: - value = re.sub(r"^\${(.*)}$", r"\1", c.value) - if value.startswith( - "var." - ): # input variable (atomic unit with type variable) - au = get_au(self.code, value.strip("var."), "variable") - if au != None: - for attribute in au.attributes: - if attribute.name == "default": - var = attribute - elif value.startswith("local."): # local value (variable) - var = get_module_var(self.code, value.strip("local.")) - - for item in ( - SecurityVisitor.__PASSWORDS - + SecurityVisitor.__SECRETS - + SecurityVisitor.__USERS - ): - if ( - re.match(r"[_A-Za-z0-9$\/\.\[\]-]*{text}\b".format(text=item), c.name) - and c.name.split("[")[0] - not in SecurityVisitor.__SECRETS_WHITELIST + SecurityVisitor.__PROFILE - ): - if not c.has_variable or var: - if not c.has_variable: - if item in SecurityVisitor.__PASSWORDS and len(c.value) == 0: - errors.append(Error("sec_empty_pass", c, file, repr(c))) - break - if var is not None: - if ( - item in SecurityVisitor.__PASSWORDS - and var.value != None - and len(var.value) == 0 - ): - errors.append(Error("sec_empty_pass", c, file, repr(c))) - break - - errors.append(Error("sec_hard_secr", c, file, repr(c))) - if item in SecurityVisitor.__PASSWORDS: - errors.append(Error("sec_hard_pass", c, file, repr(c))) - elif item in SecurityVisitor.__USERS: - errors.append(Error("sec_hard_user", c, file, repr(c))) - - break - - for item in SecurityVisitor.__SSH_DIR: - if item.lower() in c.name: - if len(c.value) > 0 and "/id_rsa" in c.value: - errors.append(Error("sec_hard_secr", c, file, repr(c))) - - for item in SecurityVisitor.__MISC_SECRETS: - if ( - re.match( - r"([_A-Za-z0-9$-]*[-_]{text}([-_].*)?$)|(^{text}([-_].*)?$)".format( - text=item - ), - c.name, - ) - and len(c.value) > 0 - and not c.has_variable - ): - errors.append(Error("sec_hard_secr", c, file, repr(c))) - - for item in SecurityVisitor.__SENSITIVE_DATA: - if item.lower() in c.name: - for item_value in SecurityVisitor.__SECRET_ASSIGN: - if item_value in c.value.lower(): - errors.append(Error("sec_hard_secr", c, file, repr(c))) - if "password" in item_value: - errors.append(Error("sec_hard_pass", c, file, repr(c))) - - if c.has_variable and var is not None: - c.has_variable = var.has_variable - c.value = var.value - - for checker in self.checkers: - checker.code = self.code - errors += checker.check(c, file) - - return errors - - def check_attribute(self, a: Attribute, file: str) -> list[Error]: - return self.__check_keyvalue(a, file) - - def check_variable(self, v: Variable, file: str) -> list[Error]: - return self.__check_keyvalue(v, file) - - def check_comment(self, c: Comment, file: str) -> List[Error]: - errors: List[Error] = [] - lines = c.content.split("\n") - stop = False - for word in SecurityVisitor.__WRONG_WORDS: - for line in lines: - tokenizer = WordPunctTokenizer() - tokens = tokenizer.tokenize(line.lower()) # type: ignore - if word in tokens: - errors.append(Error("sec_susp_comm", c, file, line)) - stop = True - if stop: - break - return errors - - def check_condition(self, c: ConditionalStatement, file: str) -> List[Error]: - errors = super().check_condition(c, file) - if c.type != ConditionalStatement.ConditionType.SWITCH: - return errors - - condition = c - has_default = False - - while condition != None: - if condition.is_default: - has_default = True - break - condition = condition.else_statement - - if not has_default: - return errors + [Error("sec_no_default_switch", c, file, repr(c))] - - return errors - - def check_unitblock(self, u: UnitBlock, file: str) -> List[Error]: - errors = super().check_unitblock(u, file) - - # Missing integrity check changed to unit block since in Docker the integrity check is not an attribute of the - # atomic unit but can be done on another atomic unit inside the same unit block. - missing_integrity_checks = {} - for au in u.atomic_units: - result = self.check_integrity_check(au, file) - if result is not None: - missing_integrity_checks[result[0]] = result[1] - continue - f = SecurityVisitor.check_has_checksum(au) - if f is not None: - if f in missing_integrity_checks: - del missing_integrity_checks[f] - - errors += missing_integrity_checks.values() - errors += self.non_off_img.check(u, file) - - return errors - - @staticmethod - def check_integrity_check(au: AtomicUnit, path: str) -> Optional[Tuple[str, Error]]: - for item in SecurityVisitor.__DOWNLOAD: - if not isinstance(au.name, str): - continue - - if not re.search( - r"(http|https|www)[^ ,]*\.{text}".format(text=item), au.name - ): - continue - if SecurityVisitor.__has_integrity_check(au.attributes): - return None - return os.path.basename(au.name), Error( - "sec_no_int_check", au, path, repr(au) - ) - - for a in au.attributes: - value = ( - a.value.strip().lower() - if isinstance(a.value, str) - else repr(a.value).strip().lower() - ) - - for item in SecurityVisitor.__DOWNLOAD: - if not re.search( - r"(http|https|www)[^ ,]*\.{text}".format(text=item), value - ): - continue - if SecurityVisitor.__has_integrity_check(au.attributes): - return None - return os.path.basename(a.value), Error( # type: ignore - "sec_no_int_check", au, path, repr(a) - ) # type: ignore - return None - - @staticmethod - def check_has_checksum(au: AtomicUnit) -> Optional[str]: - if au.type not in SecurityVisitor.__CHECKSUM or au.name is None: - return None - if any(d in au.name for d in SecurityVisitor.__DOWNLOAD): - return os.path.basename(au.name) - - for a in au.attributes: - value = ( - a.value.strip().lower() - if isinstance(a.value, str) - else repr(a.value).strip().lower() - ) - if any(d in value for d in SecurityVisitor.__DOWNLOAD): - return os.path.basename(au.name) - return None - - @staticmethod - def __has_integrity_check(attributes: List[Attribute]) -> bool: - for attr in attributes: - name = attr.name.strip().lower() - if any([check in name for check in SecurityVisitor.__CHECKSUM]): - return True - return False - - @staticmethod - def __is_http_url(value: str | None) -> bool: - if value is None: - return False - - if ( - re.match(SecurityVisitor.__URL_REGEX, value) - and ("http" in value or "www" in value) - and "https" not in value - ): - return True - try: - parsed_url = urlparse(value) - return ( - parsed_url.scheme == "http" - and parsed_url.hostname not in SecurityVisitor.__URL_WHITELIST - ) - except ValueError: - return False - - @staticmethod - def __is_weak_crypt(value: str, name: str | None) -> bool: - if name is None: - return False - - if any(crypt in value for crypt in SecurityVisitor.__CRYPT): - whitelist = any( - word in name or word in value - for word in SecurityVisitor.__CRYPT_WHITELIST - ) - return not whitelist - return False - - -# NOTE: in the end of the file to avoid circular import -# Imports all the classes defined in the __init__.py file -from glitch.analysis.terraform import * diff --git a/glitch/analysis/security/__init__.py b/glitch/analysis/security/__init__.py new file mode 100644 index 00000000..d1bb5cae --- /dev/null +++ b/glitch/analysis/security/__init__.py @@ -0,0 +1,7 @@ +import os +from typing import List + +__all__: List[str] = [] +for file in os.listdir(os.path.dirname(__file__)): + if file.endswith(".py") and file != "__init__.py" and file != "visitor.py": + __all__.append(file[:-3]) # type: ignore diff --git a/glitch/analysis/security/smell_checker.py b/glitch/analysis/security/smell_checker.py new file mode 100644 index 00000000..bce312e1 --- /dev/null +++ b/glitch/analysis/security/smell_checker.py @@ -0,0 +1,5 @@ +from glitch.analysis.rules import SmellChecker + + +class SecuritySmellChecker(SmellChecker): + pass diff --git a/glitch/analysis/security/visitor.py b/glitch/analysis/security/visitor.py new file mode 100644 index 00000000..a1ff992f --- /dev/null +++ b/glitch/analysis/security/visitor.py @@ -0,0 +1,287 @@ +import os +import json +import glitch +import configparser +from glitch.analysis.rules import Error, RuleVisitor, SmellChecker +from nltk.tokenize import WordPunctTokenizer # type: ignore +from typing import List, Type, Dict + +from glitch.tech import Tech +from glitch.repr.inter import * + +from glitch.analysis.terraform.smell_checker import TerraformSmellChecker +from glitch.analysis.security.smell_checker import SecuritySmellChecker + + +class SecurityVisitor(RuleVisitor): + class NonOfficialImageSmell(SmellChecker): + def check(self, element: CodeElement, file: str) -> List[Error]: + return [] + + def __init__(self, tech: Tech, fallback: set[str]) -> None: + super().__init__(tech) + + SECURITY_CHECKER_ERRORS: Dict[Type[SecuritySmellChecker], List[str]] = {} + + from glitch.analysis.terraform.access_control import TerraformAccessControl + from glitch.analysis.terraform.attached_resource import ( + TerraformAttachedResource, + ) + from glitch.analysis.terraform.authentication import TerraformAuthentication + from glitch.analysis.terraform.dns_policy import TerraformDnsWithoutDnssec + from glitch.analysis.terraform.firewall_misconfig import ( + TerraformFirewallMisconfig, + ) + from glitch.analysis.terraform.http_without_tls import TerraformHttpWithoutTls + from glitch.analysis.terraform.integrity_policy import TerraformIntegrityPolicy + from glitch.analysis.terraform.key_management import TerraformKeyManagement + from glitch.analysis.terraform.logging import TerraformLogging + from glitch.analysis.terraform.missing_encryption import ( + TerraformMissingEncryption, + ) + from glitch.analysis.terraform.naming import TerraformNaming + from glitch.analysis.terraform.network_policy import ( + TerraformNetworkSecurityRules, + ) + from glitch.analysis.terraform.permission_iam_policies import ( + TerraformPermissionIAMPolicies, + ) + from glitch.analysis.terraform.public_ip import TerraformPublicIp + from glitch.analysis.terraform.replication import TerraformReplication + from glitch.analysis.terraform.sensitive_iam_action import ( + TerraformSensitiveIAMAction, + ) + from glitch.analysis.terraform.ssl_tls_policy import TerraformSslTlsPolicy + from glitch.analysis.terraform.threats_detection import ( + TerraformThreatsDetection, + ) + from glitch.analysis.terraform.versioning import TerraformVersioning + from glitch.analysis.terraform.weak_password_key_policy import ( + TerraformWeakPasswordKeyPolicy, + ) + + TERRAFORM_CHECKER_ERRORS: Dict[Type[TerraformSmellChecker], str] = { + TerraformAccessControl: "sec_access_control", + TerraformAttachedResource: "sec_attached_resource", + TerraformAuthentication: "sec_authentication", + TerraformDnsWithoutDnssec: "sec_dnssec", + TerraformFirewallMisconfig: "sec_firewall_misconfig", + TerraformHttpWithoutTls: "sec_https", + TerraformIntegrityPolicy: "sec_integrity_policy", + TerraformKeyManagement: "sec_key_management", + TerraformLogging: "sec_logging", + TerraformMissingEncryption: "sec_missing_encryption", + TerraformNaming: "sec_naming", + TerraformNetworkSecurityRules: "sec_network_security_rules", + TerraformPermissionIAMPolicies: "sec_permission_iam_policies", + TerraformPublicIp: "sec_public_ip", + TerraformReplication: "sec_replication", + TerraformSensitiveIAMAction: "sec_sensitive_iam_action", + TerraformSslTlsPolicy: "sec_ssl_tls_policy", + TerraformThreatsDetection: "sec_threats_detection_alerts", + TerraformVersioning: "sec_versioning", + TerraformWeakPasswordKeyPolicy: "sec_weak_password_key_policy", + } + + self.checkers: List[SmellChecker] = [] + + for child in SecuritySmellChecker.__subclasses__(): + error_name = SECURITY_CHECKER_ERRORS.get(child, []) + + if not any(name in fallback for name in error_name): + continue + + self.checkers.append(child()) + + if tech == Tech.terraform: + # Some Terraform checkers handle Terraform-specific patterns that complement + # generic Rego checks for the same smell code. These should always run. + ALWAYS_RUN_CHECKERS = {TerraformHttpWithoutTls} + + for child in TerraformSmellChecker.__subclasses__(): + error_name = TERRAFORM_CHECKER_ERRORS.get(child) + + if error_name is None: + continue + + # Run if: smell is in fallback (no Rego) OR checker is in always-run list + if error_name not in fallback and child not in ALWAYS_RUN_CHECKERS: + continue + + self.checkers.append(child()) + + self.non_off_img = SecurityVisitor.NonOfficialImageSmell() + + @staticmethod + def get_name() -> str: + return "security" + + def config(self, config_path: str) -> None: + config = configparser.ConfigParser() + config.read(config_path) + SecurityVisitor.WRONG_WORDS = json.loads(config["security"]["suspicious_words"]) + SecurityVisitor.PASSWORDS = json.loads(config["security"]["passwords"]) + SecurityVisitor.USERS = json.loads(config["security"]["users"]) + SecurityVisitor.PROFILE = json.loads(config["security"]["profile"]) + SecurityVisitor.SECRETS = json.loads(config["security"]["secrets"]) + SecurityVisitor.MISC_SECRETS = json.loads(config["security"]["misc_secrets"]) + SecurityVisitor.ROLES = json.loads(config["security"]["roles"]) + SecurityVisitor.DOWNLOAD = json.loads(config["security"]["download_extensions"]) + SecurityVisitor.SSH_DIR = json.loads(config["security"]["ssh_dirs"]) + SecurityVisitor.ADMIN = json.loads(config["security"]["admin"]) + SecurityVisitor.CHECKSUM = json.loads(config["security"]["checksum"]) + SecurityVisitor.CRYPT = json.loads(config["security"]["weak_crypt"]) + SecurityVisitor.CRYPT_WHITELIST = json.loads( + config["security"]["weak_crypt_whitelist"] + ) + SecurityVisitor.URL_WHITELIST = json.loads( + config["security"]["url_http_white_list"] + ) + SecurityVisitor.SECRETS_WHITELIST = json.loads( + config["security"]["secrets_white_list"] + ) + SecurityVisitor.SENSITIVE_DATA = json.loads( + config["security"]["sensitive_data"] + ) + SecurityVisitor.SECRET_ASSIGN = json.loads( + config["security"]["secret_value_assign"] + ) + SecurityVisitor.GITHUB_ACTIONS = json.loads( + config["security"]["github_actions_resources"] + ) + + if self.tech == Tech.terraform: + SecurityVisitor.INTEGRITY_POLICY = json.loads( + config["security"]["integrity_policy"] + ) + SecurityVisitor.HTTPS_CONFIGS = json.loads( + config["security"]["ensure_https"] + ) + SecurityVisitor.SSL_TLS_POLICY = json.loads( + config["security"]["ssl_tls_policy"] + ) + SecurityVisitor.DNSSEC_CONFIGS = json.loads( + config["security"]["ensure_dnssec"] + ) + SecurityVisitor.PUBLIC_IP_CONFIGS = json.loads( + config["security"]["use_public_ip"] + ) + SecurityVisitor.POLICY_KEYWORDS = json.loads( + config["security"]["policy_keywords"] + ) + SecurityVisitor.ACCESS_CONTROL_CONFIGS = json.loads( + config["security"]["insecure_access_control"] + ) + SecurityVisitor.AUTHENTICATION = json.loads( + config["security"]["authentication"] + ) + SecurityVisitor.POLICY_ACCESS_CONTROL = json.loads( + config["security"]["policy_insecure_access_control"] + ) + SecurityVisitor.POLICY_AUTHENTICATION = json.loads( + config["security"]["policy_authentication"] + ) + SecurityVisitor.MISSING_ENCRYPTION = json.loads( + config["security"]["missing_encryption"] + ) + SecurityVisitor.CONFIGURATION_KEYWORDS = json.loads( + config["security"]["configuration_keywords"] + ) + SecurityVisitor.ENCRYPT_CONFIG = json.loads( + config["security"]["encrypt_configuration"] + ) + SecurityVisitor.FIREWALL_CONFIGS = json.loads( + config["security"]["firewall"] + ) + SecurityVisitor.MISSING_THREATS_DETECTION_ALERTS = json.loads( + config["security"]["missing_threats_detection_alerts"] + ) + SecurityVisitor.PASSWORD_KEY_POLICY = json.loads( + config["security"]["password_key_policy"] + ) + SecurityVisitor.KEY_MANAGEMENT = json.loads( + config["security"]["key_management"] + ) + SecurityVisitor.NETWORK_SECURITY_RULES = json.loads( + config["security"]["network_security_rules"] + ) + SecurityVisitor.PERMISSION_IAM_POLICIES = json.loads( + config["security"]["permission_iam_policies"] + ) + SecurityVisitor.GOOGLE_IAM_MEMBER = json.loads( + config["security"]["google_iam_member_resources"] + ) + SecurityVisitor.LOGGING = json.loads(config["security"]["logging"]) + SecurityVisitor.GOOGLE_SQL_DATABASE_LOG_FLAGS = json.loads( + config["security"]["google_sql_database_log_flags"] + ) + SecurityVisitor.POSSIBLE_ATTACHED_RESOURCES = json.loads( + config["security"]["possible_attached_resources_aws_route53"] + ) + SecurityVisitor.VERSIONING = json.loads(config["security"]["versioning"]) + SecurityVisitor.NAMING = json.loads(config["security"]["naming"]) + SecurityVisitor.REPLICATION = json.loads(config["security"]["replication"]) + + SecurityVisitor.FILE_COMMANDS = json.loads(config["security"]["file_commands"]) + SecurityVisitor.SHELL_RESOURCES = json.loads( + config["security"]["shell_resources"] + ) + SecurityVisitor.IP_BIND_COMMANDS = json.loads( + config["security"]["ip_binding_commands"] + ) + SecurityVisitor.OBSOLETE_COMMANDS = self._load_data_file("obsolete_commands") + + @staticmethod + def _load_data_file(file: str) -> List[str]: + folder_path = os.path.dirname(os.path.realpath(glitch.__file__)) + with open(os.path.join(folder_path, "files", file)) as f: + content = f.readlines() + return [c.strip() for c in content] + + def check_atomicunit(self, au: AtomicUnit, file: str) -> List[Error]: + errors = super().check_atomicunit(au, file) + + for checker in self.checkers: + checker.code = self.code + errors += checker.check(au, file) + + return errors + + def check_dependency(self, d: Dependency, file: str) -> List[Error]: + return [] + + def __check_keyvalue(self, c: KeyValue, file: str) -> List[Error]: + errors: List[Error] = [] + c.name = c.name.strip().lower() + + for checker in self.checkers: + checker.code = self.code + errors += checker.check(c, file) + + return errors + + def check_attribute(self, a: Attribute, file: str) -> list[Error]: + return self.__check_keyvalue(a, file) + + def check_variable(self, v: Variable, file: str) -> list[Error]: + return self.__check_keyvalue(v, file) + + def check_comment(self, c: Comment, file: str) -> List[Error]: + errors: List[Error] = [] + return errors + + def check_condition(self, c: ConditionalStatement, file: str) -> List[Error]: + errors = super().check_condition(c, file) + return errors + + def check_unitblock(self, u: UnitBlock, file: str) -> List[Error]: + errors = super().check_unitblock(u, file) + errors += self.non_off_img.check(u, file) + + return errors + + +# NOTE: in the end of the file to avoid circular import +# Imports all the classes defined in the __init__.py file +from glitch.analysis.terraform import * +from glitch.analysis.security import * diff --git a/glitch/analysis/terraform/access_control.py b/glitch/analysis/terraform/access_control.py index c52bf871..46efa1a2 100644 --- a/glitch/analysis/terraform/access_control.py +++ b/glitch/analysis/terraform/access_control.py @@ -2,8 +2,9 @@ from typing import List from glitch.analysis.terraform.smell_checker import TerraformSmellChecker from glitch.analysis.rules import Error -from glitch.analysis.security import SecurityVisitor -from glitch.repr.inter import AtomicUnit, Attribute, CodeElement, KeyValue +from glitch.analysis.security.visitor import SecurityVisitor +from glitch.analysis.checkers.string_checker import StringChecker +from glitch.repr.inter import * class TerraformAccessControl(TerraformSmellChecker): @@ -21,10 +22,16 @@ def _check_attribute( pattern = re.compile(rf"{expr}") allow_expr = '"effect":' + "\\s*" + '"allow"' allow_pattern = re.compile(rf"{allow_expr}") - if ( - isinstance(attribute.value, str) - and re.search(pattern, attribute.value) - and re.search(allow_pattern, attribute.value) + + pattern_checker = StringChecker( + lambda x: re.search(pattern, x.lower()) is not None + ) + allow_checker = StringChecker( + lambda x: re.search(allow_pattern, x.lower()) is not None + ) + + if pattern_checker.check(attribute.value) and allow_checker.check( + attribute.value ): return [ Error( @@ -32,85 +39,87 @@ def _check_attribute( ) ] + star_checker = StringChecker(lambda x: x == "*") if ( - re.search(r"actions\[\d+\]", attribute.name) + attribute.name == "actions" and parent_name == "permissions" - and atomic_unit.type == "resource.azurerm_role_definition" - and attribute.value == "*" + and atomic_unit.type == "azurerm_role_definition" + and star_checker.check(attribute.value) ): return [Error("sec_access_control", attribute, file, repr(attribute))] elif ( - ( - re.search(r"members\[\d+\]", attribute.name) - and atomic_unit.type == "resource.google_storage_bucket_iam_binding" - ) - or ( - attribute.name == "member" - and atomic_unit.type == "resource.google_storage_bucket_iam_member" + attribute.name == "member" + and atomic_unit.type == "google_storage_bucket_iam_member" + ): + value_str = ( + attribute.value.value.lower() + if isinstance(attribute.value, String) + else str(attribute.value).lower() ) - ) and ( - attribute.value == "allusers" or attribute.value == "allauthenticatedusers" + if value_str in ["allusers", "allauthenticatedusers"]: + return [Error("sec_access_control", attribute, file, repr(attribute))] + elif ( + attribute.name == "members" + and atomic_unit.type == "google_storage_bucket_iam_binding" + and isinstance(attribute.value, Array) ): - return [Error("sec_access_control", attribute, file, repr(attribute))] + for item in attribute.value.value: + if isinstance(item, String) and item.value.lower() in [ + "allusers", + "allauthenticatedusers", + ]: + return [ + Error("sec_access_control", attribute, file, repr(attribute)) + ] elif ( attribute.name == "email" and parent_name == "service_account" - and atomic_unit.type == "resource.google_compute_instance" - and isinstance(attribute.value, str) - and re.search(r".-compute@developer.gserviceaccount.com", attribute.value) + and atomic_unit.type == "google_compute_instance" + and isinstance(attribute.value, String) + and re.search( + r".-compute@developer.gserviceaccount.com", attribute.value.value + ) ): return [Error("sec_access_control", attribute, file, repr(attribute))] - for config in SecurityVisitor.ACCESS_CONTROL_CONFIGS: - if ( - attribute.name == config["attribute"] - and atomic_unit.type in config["au_type"] - and parent_name in config["parents"] - and not attribute.has_variable - and isinstance(attribute.value, str) - and attribute.value.lower() not in config["values"] - and config["values"] != [""] - ): - return [Error("sec_access_control", attribute, file, repr(attribute))] - return [] def check(self, element: CodeElement, file: str) -> List[Error]: errors: List[Error] = [] if isinstance(element, AtomicUnit): - if element.type == "resource.aws_api_gateway_method": - http_method = self.check_required_attribute( - element.attributes, [""], "http_method" - ) + get_checker = StringChecker(lambda x: x.lower() == "get") + none_checker = StringChecker(lambda x: x.lower() == "none") + + if element.type == "aws_api_gateway_method": + http_method = self.check_required_attribute(element, [], "http_method") authorization = self.check_required_attribute( - element.attributes, [""], "authorization" + element, [], "authorization" ) - if ( - isinstance(http_method, KeyValue) - and isinstance(authorization, KeyValue) - and http_method.value is not None - and authorization.value is not None + if isinstance(http_method, (KeyValue, Attribute)) and isinstance( + authorization, (KeyValue, Attribute) ): - if ( - http_method.value.lower() == "get" - and authorization.value.lower() == "none" + if get_checker.check(http_method.value) and none_checker.check( + authorization.value ): api_key_required = self.check_required_attribute( - element.attributes, [""], "api_key_required" + element, [], "api_key_required" ) - if ( - isinstance(api_key_required, KeyValue) - and api_key_required.value is not None - and f"{api_key_required.value}".lower() != "true" - ): - errors.append( - Error( - "sec_access_control", - api_key_required, - file, - repr(api_key_required), + if isinstance(api_key_required, (KeyValue, Attribute)): + value = api_key_required.value + is_true = False + if isinstance(value, Boolean): + is_true = value.value + elif isinstance(value, String): + is_true = value.value.lower() == "true" + if not is_true: + errors.append( + Error( + "sec_access_control", + api_key_required, + file, + repr(api_key_required), + ) ) - ) elif not api_key_required: errors.append( Error( @@ -131,30 +140,41 @@ def check(self, element: CodeElement, file: str) -> List[Error]: f"Suggestion: check for a required attribute with name 'authorization'.", ) ) - elif element.type == "resource.github_repository": - visibility = self.check_required_attribute( - element.attributes, [""], "visibility" - ) - if isinstance(visibility, KeyValue) and isinstance( - visibility.value, str + elif element.type == "github_repository": + visibility = self.check_required_attribute(element, [], "visibility") + if isinstance(visibility, (KeyValue, Attribute)) and isinstance( + visibility.value, String ): - if visibility.value.lower() not in ["private", "internal"]: + if visibility.value.value.lower() not in ["private", "internal"]: errors.append( Error( "sec_access_control", visibility, file, repr(visibility) ) ) else: - private = self.check_required_attribute( - element.attributes, [""], "private" - ) - if isinstance(private, KeyValue) and isinstance(private.value, str): - if f"{private.value}".lower() != "true": - errors.append( - Error( - "sec_access_control", private, file, repr(private) + private = self.check_required_attribute(element, [], "private") + if isinstance(private, (KeyValue, Attribute)): + value = private.value + if isinstance(value, String): + if value.value.lower() != "true": + errors.append( + Error( + "sec_access_control", + private, + file, + repr(private), + ) + ) + elif isinstance(value, Boolean): + if not value.value: + errors.append( + Error( + "sec_access_control", + private, + file, + repr(private), + ) ) - ) else: errors.append( Error( @@ -165,7 +185,7 @@ def check(self, element: CodeElement, file: str) -> List[Error]: f"Suggestion: check for a required attribute with name 'visibility' or 'private'.", ) ) - elif element.type == "resource.google_sql_database_instance": + elif element.type == "google_sql_database_instance": errors += self.check_database_flags( element, file, @@ -173,16 +193,17 @@ def check(self, element: CodeElement, file: str) -> List[Error]: "cross db ownership chaining", "off", ) - elif element.type == "resource.aws_s3_bucket": - expr = "\\${aws_s3_bucket\\." + f"{element.name}\\." + # This does not work when the name is not a String :/ + elif element.type == "aws_s3_bucket" and isinstance(element.name, String): + expr = "aws_s3_bucket\\." + f"{element.name.value}\\." pattern = re.compile(rf"{expr}") if ( self.get_associated_au( file, - "resource.aws_s3_bucket_public_access_block", + "aws_s3_bucket_public_access_block", "bucket", pattern, - [""], + [], ) is None ): @@ -198,20 +219,51 @@ def check(self, element: CodeElement, file: str) -> List[Error]: ) for config in SecurityVisitor.ACCESS_CONTROL_CONFIGS: + values = [None] + if len(config["values"]) > 0: + values = config["values"] + + required_attribute = self.check_required_attribute( + element, + config["parents"], + config["attribute"], + ) if ( - config["required"] == "yes" - and element.type in config["au_type"] - and not self.check_required_attribute( - element.attributes, config["parents"], config["attribute"] - ) + element.type not in config["au_type"] + # If the attribute is not required and the attribute is not present, skip + # The default value is OK + or (required_attribute is None and config["required"] == "no") ): + continue + + satisfied = False + for value in values: + if ( + self.check_required_attribute( + element, config["parents"], config["attribute"], value=value + ) + is not None + ): + satisfied = True + + if not satisfied: + if required_attribute is not None: + element_with_error = required_attribute + else: + element_with_error = element + + full_name = ".".join(config["parents"] + [config["attribute"]]) + value = "" + if len(values) > 0: + value = f" with value in {values}" + errors.append( Error( "sec_access_control", - element, + element_with_error, file, - repr(element), - f"Suggestion: check for a required attribute with name '{config['msg']}'.", + repr(element_with_error), + f"Suggestion: check for a required attribute with name '{full_name}'{value}.", ) ) diff --git a/glitch/analysis/terraform/attached_resource.py b/glitch/analysis/terraform/attached_resource.py index fe12f36e..daf2c107 100644 --- a/glitch/analysis/terraform/attached_resource.py +++ b/glitch/analysis/terraform/attached_resource.py @@ -1,8 +1,15 @@ from typing import List from glitch.analysis.terraform.smell_checker import TerraformSmellChecker from glitch.analysis.rules import Error -from glitch.analysis.security import SecurityVisitor -from glitch.repr.inter import AtomicUnit, CodeElement, KeyValue, Attribute +from glitch.analysis.security.visitor import SecurityVisitor +from glitch.repr.inter import ( + AtomicUnit, + CodeElement, + KeyValue, + Attribute, + UnitBlock, + Array, +) class TerraformAttachedResource(TerraformSmellChecker): @@ -11,32 +18,47 @@ def check(self, element: CodeElement, file: str) -> List[Error]: if isinstance(element, AtomicUnit): + def check_value_for_resource( + value_code: str, resource_types: List[str] + ) -> bool: + value_lower = value_code.lower() + for resource_type in resource_types: + if value_lower.startswith( + "${" + f"{resource_type}." + ) or value_lower.startswith(f"{resource_type}."): + resource_name = value_lower.split(".")[1] + if self.get_au(file, resource_name, resource_type): + return True + return False + def check_attached_resource( - attributes: List[KeyValue] | List[Attribute], resource_types: List[str] + attributes: List[KeyValue] | List[Attribute], + statements: List[CodeElement], + resource_types: List[str], ) -> bool: for a in attributes: - if a.value != None: - for resource_type in resource_types: - if f"{a.value}".lower().startswith( - "${" + f"{resource_type}." - ) or f"{a.value}".lower().startswith(f"{resource_type}."): - resource_name = a.value.lower().split(".")[1] - if self.get_au( - file, resource_name, f"resource.{resource_type}" - ): + if hasattr(a.value, "code"): + if check_value_for_resource(a.value.code, resource_types): + return True + if isinstance(a.value, Array): + for item in a.value.value: + if hasattr(item, "code"): + if check_value_for_resource(item.code, resource_types): return True - elif a.value == None: - attached = check_attached_resource(a.keyvalues, resource_types) - if attached: + for stmt in statements: + if isinstance(stmt, UnitBlock): + if check_attached_resource( + stmt.attributes, stmt.statements, resource_types + ): return True return False - if element.type == "resource.aws_route53_record": - type_A = self.check_required_attribute( - element.attributes, [""], "type", "a" - ) + if element.type == "aws_route53_record": + type_A = self.check_required_attribute(element, [], "type", "a") if type_A and not check_attached_resource( - element.attributes, SecurityVisitor.POSSIBLE_ATTACHED_RESOURCES + element.attributes, + element.statements, + SecurityVisitor.POSSIBLE_ATTACHED_RESOURCES, ): errors.append( Error("sec_attached_resource", element, file, repr(element)) diff --git a/glitch/analysis/terraform/authentication.py b/glitch/analysis/terraform/authentication.py index c2839b4f..f5c604bc 100644 --- a/glitch/analysis/terraform/authentication.py +++ b/glitch/analysis/terraform/authentication.py @@ -2,8 +2,16 @@ from typing import List from glitch.analysis.terraform.smell_checker import TerraformSmellChecker from glitch.analysis.rules import Error -from glitch.analysis.security import SecurityVisitor -from glitch.repr.inter import AtomicUnit, Attribute, CodeElement, KeyValue +from glitch.analysis.security.visitor import SecurityVisitor +from glitch.analysis.checkers.var_checker import VariableChecker +from glitch.repr.inter import ( + AtomicUnit, + Attribute, + CodeElement, + KeyValue, + String, + Boolean, +) class TerraformAuthentication(TerraformSmellChecker): @@ -22,8 +30,11 @@ def _check_attribute( config["keyword"].lower() + "\\s*" + config["value"].lower() ) pattern = re.compile(rf"{expr}") - if isinstance(attribute.value, str) and not re.search( - pattern, attribute.value + value_str = None + if isinstance(attribute.value, String): + value_str = attribute.value.value + if value_str is not None and not re.search( + pattern, value_str.lower() ): return [ Error( @@ -38,13 +49,23 @@ def _check_attribute( if ( attribute.name == config["attribute"] and atomic_unit.type in config["au_type"] - and parent_name in config["parents"] - and not attribute.has_variable - and isinstance(attribute.value, str) - and attribute.value.lower() not in config["values"] - and config["values"] != [""] + and self._parent_matches(parent_name, config["parents"]) + and config["values"] != [] ): - return [Error("sec_authentication", attribute, file, repr(attribute))] + value_str = None + if isinstance(attribute.value, String): + value_str = attribute.value.value + elif isinstance(attribute.value, Boolean): + value_str = "true" if attribute.value.value else "false" + + if ( + value_str is not None + and not VariableChecker().check(attribute.value) + and value_str.lower() not in config["values"] + ): + return [ + Error("sec_authentication", attribute, file, repr(attribute)) + ] return [] @@ -52,7 +73,7 @@ def check(self, element: CodeElement, file: str) -> List[Error]: errors: List[Error] = [] if isinstance(element, AtomicUnit): - if element.type == "resource.google_sql_database_instance": + if element.type == "google_sql_database_instance": errors += self.check_database_flags( element, file, @@ -60,11 +81,16 @@ def check(self, element: CodeElement, file: str) -> List[Error]: "contained database authentication", "off", ) - elif element.type == "resource.aws_iam_group": - expr = "\\${aws_iam_group\\." + f"{element.name}\\." + elif element.type == "aws_iam_group": + name_str = ( + element.name.value + if isinstance(element.name, String) + else str(element.name) + ) + expr = "(\\$\\{)?aws_iam_group\\." + f"{name_str}\\." pattern = re.compile(rf"{expr}") if not self.get_associated_au( - file, "resource.aws_iam_group_policy", "group", pattern, [""] + file, "aws_iam_group_policy", "group", pattern, [] ): errors.append( Error( @@ -81,17 +107,23 @@ def check(self, element: CodeElement, file: str) -> List[Error]: if ( config["required"] == "yes" and element.type in config["au_type"] - and not self.check_required_attribute( - element.attributes, config["parents"], config["attribute"] + and self.check_required_attribute( + element, config["parents"], config["attribute"] ) + is None ): + msg = config.get("msg") + if msg is None: + parents = config["parents"] + attr = config["attribute"] + msg = ".".join(parents + [attr]) if parents else attr errors.append( Error( "sec_authentication", element, file, repr(element), - f"Suggestion: check for a required attribute with name '{config['msg']}'.", + f"Suggestion: check for a required attribute with name '{msg}'.", ) ) diff --git a/glitch/analysis/terraform/dns_policy.py b/glitch/analysis/terraform/dns_policy.py index f0474955..cac55f19 100644 --- a/glitch/analysis/terraform/dns_policy.py +++ b/glitch/analysis/terraform/dns_policy.py @@ -1,8 +1,9 @@ from typing import List from glitch.analysis.terraform.smell_checker import TerraformSmellChecker from glitch.analysis.rules import Error -from glitch.analysis.security import SecurityVisitor -from glitch.repr.inter import AtomicUnit, Attribute, CodeElement, KeyValue +from glitch.analysis.security.visitor import SecurityVisitor +from glitch.analysis.checkers.var_checker import VariableChecker +from glitch.repr.inter import AtomicUnit, Attribute, CodeElement, KeyValue, String class TerraformDnsWithoutDnssec(TerraformSmellChecker): @@ -18,9 +19,9 @@ def _check_attribute( attribute.name == config["attribute"] and atomic_unit.type in config["au_type"] and parent_name in config["parents"] - and not attribute.has_variable - and isinstance(attribute.value, str) - and attribute.value.lower() not in config["values"] + and not VariableChecker().check(attribute.value) + and isinstance(attribute.value, String) + and attribute.value.value.lower() not in config["values"] and config["values"] != [""] ): return [Error("sec_dnssec", attribute, file, repr(attribute))] @@ -34,7 +35,7 @@ def check(self, element: CodeElement, file: str) -> List[Error]: config["required"] == "yes" and element.type in config["au_type"] and not self.check_required_attribute( - element.attributes, config["parents"], config["attribute"] + element, config["parents"], config["attribute"] ) ): errors.append( diff --git a/glitch/analysis/terraform/firewall_misconfig.py b/glitch/analysis/terraform/firewall_misconfig.py index b1c065a1..b1a07214 100644 --- a/glitch/analysis/terraform/firewall_misconfig.py +++ b/glitch/analysis/terraform/firewall_misconfig.py @@ -1,11 +1,46 @@ from typing import List from glitch.analysis.terraform.smell_checker import TerraformSmellChecker from glitch.analysis.rules import Error -from glitch.analysis.security import SecurityVisitor -from glitch.repr.inter import AtomicUnit, Attribute, CodeElement, KeyValue +from glitch.analysis.security.visitor import SecurityVisitor +from glitch.analysis.checkers.var_checker import VariableChecker +from glitch.repr.inter import ( + AtomicUnit, + Attribute, + CodeElement, + KeyValue, + String, + Boolean, + UnitBlock, +) class TerraformFirewallMisconfig(TerraformSmellChecker): + def _find_block_recursive(self, element: AtomicUnit | UnitBlock, name: str) -> bool: + blocks = ( + element.statements + if isinstance(element, AtomicUnit) + else element.statements + element.unit_blocks + ) + for stmt in blocks: + if isinstance(stmt, UnitBlock): + if stmt.name == name: + return True + if self._find_block_recursive(stmt, name): + return True + if isinstance(element, UnitBlock): + for ub in element.unit_blocks: + if ub.name == name: + return True + if self._find_block_recursive(ub, name): + return True + return False + + def _is_parent_also_checked(self, parent: str, au_type: str) -> bool: + for config in SecurityVisitor.FIREWALL_CONFIGS: + if config["attribute"] == parent and au_type in config["au_type"]: + return True + return False + def _check_attribute( self, attribute: Attribute | KeyValue, @@ -17,13 +52,19 @@ def _check_attribute( if ( attribute.name == config["attribute"] and atomic_unit.type in config["au_type"] - and parent_name in config["parents"] - and config["values"] != [""] + and self._parent_matches(parent_name, config["parents"]) + and config["values"] != [] ): + value_str = None + if isinstance(attribute.value, String): + value_str = attribute.value.value + elif isinstance(attribute.value, Boolean): + value_str = "true" if attribute.value.value else "false" + if ( "any_not_empty" in config["values"] - and isinstance(attribute.value, str) - and attribute.value.lower() == "" + and value_str is not None + and value_str.strip() == "" ): return [ Error( @@ -32,9 +73,9 @@ def _check_attribute( ] elif ( "any_not_empty" not in config["values"] - and not attribute.has_variable - and isinstance(attribute.value, str) - and attribute.value.lower() not in config["values"] + and value_str is not None + and not VariableChecker().check(attribute.value) + and value_str.lower() not in config["values"] ): return [ Error( @@ -47,22 +88,39 @@ def check(self, element: CodeElement, file: str) -> List[Error]: errors: List[Error] = [] if isinstance(element, AtomicUnit): for config in SecurityVisitor.FIREWALL_CONFIGS: - if ( - config["required"] == "yes" - and element.type in config["au_type"] - and not self.check_required_attribute( - element.attributes, config["parents"], config["attribute"] - ) - ): - errors.append( - Error( - "sec_firewall_misconfig", - element, - file, - repr(element), - f"Suggestion: check for a required attribute with name '{config['msg']}'.", + if config["required"] == "yes" and element.type in config["au_type"]: + raw_parents: list[str] | list[list[str]] = config["parents"] + parent_list: List[str] = [] + if len(raw_parents) > 0: + first_parent: str | list[str] = raw_parents[0] + if isinstance(first_parent, list): + parent_list = first_parent + else: + parent_list = raw_parents # type: ignore[assignment] + + if ( + len(parent_list) > 0 + and self._is_parent_also_checked(parent_list[0], element.type) + and not self._find_block_recursive(element, parent_list[0]) + ): + continue + + attr_name = config["attribute"] + if ( + self.check_required_attribute( + element, config["parents"], attr_name + ) + is None + ): + errors.append( + Error( + "sec_firewall_misconfig", + element, + file, + repr(element), + f"Suggestion: check for a required attribute with name '{config.get('msg', attr_name)}'.", + ) ) - ) errors += self._check_attributes(element, file) diff --git a/glitch/analysis/terraform/http_without_tls.py b/glitch/analysis/terraform/http_without_tls.py index 0f2bcb46..1fff3684 100644 --- a/glitch/analysis/terraform/http_without_tls.py +++ b/glitch/analysis/terraform/http_without_tls.py @@ -1,8 +1,16 @@ from typing import List from glitch.analysis.terraform.smell_checker import TerraformSmellChecker from glitch.analysis.rules import Error -from glitch.analysis.security import SecurityVisitor -from glitch.repr.inter import AtomicUnit, Attribute, CodeElement, KeyValue +from glitch.analysis.security.visitor import SecurityVisitor +from glitch.analysis.checkers.var_checker import VariableChecker +from glitch.repr.inter import ( + AtomicUnit, + Attribute, + Boolean, + CodeElement, + KeyValue, + String, +) class TerraformHttpWithoutTls(TerraformSmellChecker): @@ -17,12 +25,24 @@ def _check_attribute( if ( attribute.name == config["attribute"] and atomic_unit.type in config["au_type"] - and parent_name in config["parents"] - and not attribute.has_variable - and isinstance(attribute.value, str) - and attribute.value.lower() not in config["values"] + and self._parent_matches(parent_name, config["parents"]) + and not VariableChecker().check(attribute.value) ): - return [Error("sec_https", attribute, file, repr(attribute))] + if ( + isinstance(attribute.value, Boolean) + and str(attribute.value.value).lower() not in config["values"] + ): + return [Error("sec_https", attribute, file, repr(attribute))] + elif ( + isinstance(attribute.value, String) + and attribute.value.value.lower() not in config["values"] + ): + return [Error("sec_https", attribute, file, repr(attribute))] + elif ( + isinstance(attribute.value, str) + and attribute.value.lower() not in config["values"] + ): + return [Error("sec_https", attribute, file, repr(attribute))] return [] @@ -30,14 +50,15 @@ def check(self, element: CodeElement, file: str) -> List[Error]: errors: List[Error] = [] if isinstance(element, AtomicUnit): if element.type == "data.http": - url = self.check_required_attribute(element.attributes, [""], "url") + url = self.check_required_attribute(element, [], "url") if ( isinstance(url, KeyValue) - and isinstance(url.value, str) - and "${" in url.value + and hasattr(url.value, "code") + and "${" in url.value.code ): - vars = url.value.split("${") - r = url.value.split("${")[1].split("}")[0] + url_code = url.value.code + vars = url_code.split("${") + r = url_code.split("${")[1].split("}")[0] for var in vars: if "data" in var or "resource" in var: r = var.split("}")[0] @@ -58,7 +79,7 @@ def check(self, element: CodeElement, file: str) -> List[Error]: config["required"] == "yes" and element.type in config["au_type"] and not self.check_required_attribute( - element.attributes, config["parents"], config["attribute"] + element, config["parents"], config["attribute"] ) ): errors.append( @@ -67,7 +88,7 @@ def check(self, element: CodeElement, file: str) -> List[Error]: element, file, repr(element), - f"Suggestion: check for a required attribute with name '{config['msg']}'.", + f"Suggestion: check for a required attribute with name '{config.get('msg', config['attribute'])}'.", ) ) diff --git a/glitch/analysis/terraform/integrity_policy.py b/glitch/analysis/terraform/integrity_policy.py index cdf658a4..69ad211c 100644 --- a/glitch/analysis/terraform/integrity_policy.py +++ b/glitch/analysis/terraform/integrity_policy.py @@ -1,8 +1,16 @@ from typing import List from glitch.analysis.terraform.smell_checker import TerraformSmellChecker from glitch.analysis.rules import Error -from glitch.analysis.security import SecurityVisitor -from glitch.repr.inter import AtomicUnit, Attribute, CodeElement, KeyValue +from glitch.analysis.security.visitor import SecurityVisitor +from glitch.analysis.checkers.var_checker import VariableChecker +from glitch.repr.inter import ( + AtomicUnit, + Attribute, + Boolean, + CodeElement, + KeyValue, + String, +) class TerraformIntegrityPolicy(TerraformSmellChecker): @@ -17,12 +25,26 @@ def _check_attribute( if ( attribute.name == policy["attribute"] and atomic_unit.type in policy["au_type"] - and parent_name in policy["parents"] - and not attribute.has_variable - and isinstance(attribute.value, str) - and attribute.value.lower() not in policy["values"] + and ( + parent_name in policy["parents"] + or (not policy["parents"] and not parent_name) + ) + and not VariableChecker().check(attribute.value) ): - return [Error("sec_integrity_policy", attribute, file, repr(attribute))] + if ( + isinstance(attribute.value, Boolean) + and str(attribute.value.value).lower() not in policy["values"] + ): + return [ + Error("sec_integrity_policy", attribute, file, repr(attribute)) + ] + elif ( + isinstance(attribute.value, String) + and attribute.value.value.lower() not in policy["values"] + ): + return [ + Error("sec_integrity_policy", attribute, file, repr(attribute)) + ] return [] @@ -34,7 +56,7 @@ def check(self, element: CodeElement, file: str) -> List[Error]: policy["required"] == "yes" and element.type in policy["au_type"] and not self.check_required_attribute( - element.attributes, policy["parents"], policy["attribute"] + element, policy["parents"], policy["attribute"] ) ): errors.append( diff --git a/glitch/analysis/terraform/key_management.py b/glitch/analysis/terraform/key_management.py index 3fa30bca..4dd979e2 100644 --- a/glitch/analysis/terraform/key_management.py +++ b/glitch/analysis/terraform/key_management.py @@ -2,8 +2,16 @@ from typing import List from glitch.analysis.terraform.smell_checker import TerraformSmellChecker from glitch.analysis.rules import Error -from glitch.analysis.security import SecurityVisitor -from glitch.repr.inter import AtomicUnit, Attribute, CodeElement, KeyValue +from glitch.analysis.security.visitor import SecurityVisitor +from glitch.analysis.checkers.var_checker import VariableChecker +from glitch.repr.inter import ( + AtomicUnit, + Attribute, + CodeElement, + KeyValue, + String, + Boolean, +) class TerraformKeyManagement(TerraformSmellChecker): @@ -18,22 +26,27 @@ def _check_attribute( if ( attribute.name == config["attribute"] and atomic_unit.type in config["au_type"] - and parent_name in config["parents"] - and config["values"] != [""] + and self._parent_matches(parent_name, config["parents"]) + and config["values"] != [] ): + value_str = None + if isinstance(attribute.value, String): + value_str = attribute.value.value + elif isinstance(attribute.value, Boolean): + value_str = "true" if attribute.value.value else "false" if ( "any_not_empty" in config["values"] - and isinstance(attribute.value, str) - and attribute.value.lower() == "" + and value_str is not None + and value_str.strip() == "" ): return [ Error("sec_key_management", attribute, file, repr(attribute)) ] elif ( "any_not_empty" not in config["values"] - and not attribute.has_variable - and isinstance(attribute.value, str) - and attribute.value.lower() not in config["values"] + and value_str is not None + and not VariableChecker().check(attribute.value) + and value_str.lower() not in config["values"] ): return [ Error("sec_key_management", attribute, file, repr(attribute)) @@ -41,14 +54,17 @@ def _check_attribute( if ( attribute.name == "rotation_period" - and atomic_unit.type == "resource.google_kms_crypto_key" + and atomic_unit.type == "google_kms_crypto_key" ): expr1 = r"\d+\.\d{0,9}s" expr2 = r"\d+s" - if isinstance(attribute.value, str) and ( - re.search(expr1, attribute.value) or re.search(expr2, attribute.value) + value_str = ( + attribute.value.value if isinstance(attribute.value, String) else None + ) + if value_str is not None and ( + re.search(expr1, value_str) or re.search(expr2, value_str) ): - if int(attribute.value.split("s")[0]) > 7776000: + if int(value_str.split("s")[0].split(".")[0]) > 7776000: return [ Error("sec_key_management", attribute, file, repr(attribute)) ] @@ -71,15 +87,20 @@ def _check_attribute( def check(self, element: CodeElement, file: str) -> List[Error]: errors: List[Error] = [] if isinstance(element, AtomicUnit): - if element.type == "resource.azurerm_storage_account": - expr = "\\${azurerm_storage_account\\." + f"{element.name}\\." + if element.type == "azurerm_storage_account": + name_str = ( + element.name.value + if isinstance(element.name, String) + else str(element.name) + ) + expr = "(\\$\\{)?azurerm_storage_account\\." + f"{name_str}\\." pattern = re.compile(rf"{expr}") if not self.get_associated_au( file, - "resource.azurerm_storage_account_customer_managed_key", + "azurerm_storage_account_customer_managed_key", "storage_account_id", pattern, - [""], + [], ): errors.append( Error( @@ -92,22 +113,23 @@ def check(self, element: CodeElement, file: str) -> List[Error]: ) ) for config in SecurityVisitor.KEY_MANAGEMENT: - if ( - config["required"] == "yes" - and element.type in config["au_type"] - and not self.check_required_attribute( - element.attributes, config["parents"], config["attribute"] - ) - ): - errors.append( - Error( - "sec_key_management", - element, - file, - repr(element), - f"Suggestion: check for a required attribute with name '{config['msg']}'.", + if config["required"] == "yes" and element.type in config["au_type"]: + attr_name = config["attribute"] + if ( + self.check_required_attribute( + element, config["parents"], attr_name + ) + is None + ): + errors.append( + Error( + "sec_key_management", + element, + file, + repr(element), + f"Suggestion: check for a required attribute with name '{config.get('msg', attr_name)}'.", + ) ) - ) errors += self._check_attributes(element, file) diff --git a/glitch/analysis/terraform/logging.py b/glitch/analysis/terraform/logging.py index 8d1d242c..2e64684d 100644 --- a/glitch/analysis/terraform/logging.py +++ b/glitch/analysis/terraform/logging.py @@ -3,8 +3,19 @@ from typing import List from glitch.analysis.terraform.smell_checker import TerraformSmellChecker from glitch.analysis.rules import Error -from glitch.analysis.security import SecurityVisitor -from glitch.repr.inter import AtomicUnit, Attribute, CodeElement, KeyValue +from glitch.analysis.security.visitor import SecurityVisitor +from glitch.analysis.checkers.var_checker import VariableChecker +from glitch.repr.inter import ( + Array, + AtomicUnit, + Attribute, + Boolean, + CodeElement, + Integer, + KeyValue, + String, + UnitBlock, +) class TerraformLogging(TerraformSmellChecker): @@ -17,27 +28,7 @@ def __check_log_attribute( all: bool = False, ) -> List[Error]: errors: List[Error] = [] - attribute = self.check_required_attribute( - element.attributes, [""], f"{attribute_name}[0]" - ) - - if all: - active = True - for v in values[:]: - attribute_checked, _ = self.iterate_required_attributes( - element.attributes, - attribute_name, - lambda x: isinstance(x.value, str) and x.value.lower() == v, - ) - if attribute_checked: - values.remove(v) - active = active and attribute_checked - else: - active, _ = self.iterate_required_attributes( - element.attributes, - attribute_name, - lambda x: isinstance(x.value, str) and x.value.lower() in values, - ) + attribute = self.check_required_attribute(element, [], attribute_name) if attribute is None: errors.append( @@ -49,18 +40,33 @@ def __check_log_attribute( f"Suggestion: check for a required attribute with name '{attribute_name}'.", ) ) - elif not active and not all: - errors.append(Error("sec_logging", attribute, file, repr(attribute))) - elif not active and all: - errors.append( - Error( - "sec_logging", - attribute, - file, - repr(attribute), - f"Suggestion: check for additional log type(s) {values}.", + return errors + + if not isinstance(attribute, Attribute) or not isinstance( + attribute.value, Array + ): + return errors + + array_values = [ + v.value.lower() if isinstance(v, String) else str(v).lower() + for v in attribute.value.value + ] + + if all: + missing = [v for v in values if v not in array_values] + if missing: + errors.append( + Error( + "sec_logging", + attribute, + file, + repr(attribute), + f"Suggestion: check for additional log type(s) {missing}.", + ) ) - ) + else: + if not any(v in values for v in array_values): + errors.append(Error("sec_logging", attribute, file, repr(attribute))) return errors @@ -68,13 +74,13 @@ def __check_azurerm_storage_container(self, element: AtomicUnit, file: str): errors: List[Error] = [] container_access_type = self.check_required_attribute( - element.attributes, [""], "container_access_type" + element, [], "container_access_type" ) if ( container_access_type and isinstance(container_access_type, Attribute) - and isinstance(container_access_type.value, str) - and container_access_type.value.lower() + and isinstance(container_access_type.value, String) + and container_access_type.value.value.lower() not in [ "blob", "private", @@ -90,14 +96,22 @@ def __check_azurerm_storage_container(self, element: AtomicUnit, file: str): ) storage_account_name = self.check_required_attribute( - element.attributes, [""], "storage_account_name" + element, [], "storage_account_name" ) + storage_name_value = None + if storage_account_name is not None and isinstance( + storage_account_name, Attribute + ): + if isinstance(storage_account_name.value, String): + storage_name_value = storage_account_name.value.value.lower() + elif hasattr(storage_account_name.value, "code"): + storage_name_value = storage_account_name.value.code.lower() + if not ( - storage_account_name is not None - and isinstance(storage_account_name, Attribute) - and isinstance(storage_account_name.value, str) - and storage_account_name.value.lower().startswith( - "${azurerm_storage_account." + storage_name_value is not None + and ( + storage_name_value.startswith("${azurerm_storage_account.") + or storage_name_value.startswith("azurerm_storage_account.") ) ): errors.append( @@ -112,8 +126,14 @@ def __check_azurerm_storage_container(self, element: AtomicUnit, file: str): ) return errors - name = storage_account_name.value.lower().split(".")[1] - storage_account_au = self.get_au(file, name, "resource.azurerm_storage_account") + name = ( + storage_name_value.split(".")[1] + if storage_name_value.startswith("${") + else storage_name_value.split(".")[0] + ) + if storage_name_value.startswith("azurerm_storage_account."): + name = storage_name_value.split(".")[1] + storage_account_au = self.get_au(file, name, "azurerm_storage_account") if storage_account_au is None: errors.append( Error( @@ -127,14 +147,14 @@ def __check_azurerm_storage_container(self, element: AtomicUnit, file: str): ) return errors - expr = "\\${azurerm_storage_account\\." + f"{name}\\." + expr = "(\\$\\{)?azurerm_storage_account\\." + f"{name}\\." pattern = re.compile(rf"{expr}") assoc_au = self.get_associated_au( file, - "resource.azurerm_log_analytics_storage_insights", + "azurerm_log_analytics_storage_insights", "storage_account_id", pattern, - [""], + [], ) if assoc_au is None: errors.append( @@ -150,7 +170,7 @@ def __check_azurerm_storage_container(self, element: AtomicUnit, file: str): return errors blob_container_names = self.check_required_attribute( - assoc_au.attributes, [""], "blob_container_names[0]" + assoc_au, [], "blob_container_names" ) if blob_container_names is None: errors.append( @@ -164,18 +184,23 @@ def __check_azurerm_storage_container(self, element: AtomicUnit, file: str): ) return errors - contains_blob_name, _ = self.iterate_required_attributes( - assoc_au.attributes, "blob_container_names", lambda x: x.value # type: ignore - ) - if not contains_blob_name: - errors.append( - Error( - "sec_logging", - assoc_au.attributes[-1], - file, - repr(assoc_au.attributes[-1]), + if isinstance(blob_container_names, Attribute) and isinstance( + blob_container_names.value, Array + ): + has_valid_name = False + for item in blob_container_names.value.value: + if isinstance(item, String) and item.value.strip(): + has_valid_name = True + break + if not has_valid_name: + errors.append( + Error( + "sec_logging", + blob_container_names, + file, + repr(blob_container_names), + ) ) - ) return errors @@ -188,16 +213,28 @@ def _check_attribute( ) -> List[Error]: if ( attribute.name == "cloud_watch_logs_group_arn" - and atomic_unit.type == "resource.aws_cloudtrail" + and atomic_unit.type == "aws_cloudtrail" ): - if isinstance(attribute.value, str) and re.match( - r"^\${aws_cloudwatch_log_group\..", attribute.value + value_str = None + if isinstance(attribute.value, String): + value_str = attribute.value.value + elif hasattr(attribute.value, "code"): + value_str = attribute.value.code + + if value_str and re.match( + r"^(\$\{)?aws_cloudwatch_log_group\..", value_str ): - aws_cloudwatch_log_group_name = attribute.value.split(".")[1] + aws_cloudwatch_log_group_name = ( + value_str.split(".")[1] + if value_str.startswith("$") + else value_str.split(".")[0] + ) + if value_str.startswith("aws_cloudwatch_log_group."): + aws_cloudwatch_log_group_name = value_str.split(".")[1] if not self.get_au( file, aws_cloudwatch_log_group_name, - "resource.aws_cloudwatch_log_group", + "aws_cloudwatch_log_group", ): return [ Error( @@ -212,60 +249,57 @@ def _check_attribute( else: return [Error("sec_logging", attribute, file, repr(attribute))] elif ( - ( - attribute.name == "retention_in_days" - and parent_name == "" - and atomic_unit.type - in [ - "resource.azurerm_mssql_database_extended_auditing_policy", - "resource.azurerm_mssql_server_extended_auditing_policy", - ] - ) - or ( - attribute.name == "days" - and parent_name == "retention_policy" - and atomic_unit.type == "resource.azurerm_network_watcher_flow_log" - ) - ) and ( - isinstance(attribute.value, str) - and ( - not attribute.value.isnumeric() - or (attribute.value.isnumeric() and int(attribute.value) < 90) - ) + attribute.name == "retention_in_days" + and parent_name == "" + and atomic_unit.type + in [ + "azurerm_mssql_database_extended_auditing_policy", + "azurerm_mssql_server_extended_auditing_policy", + ] + ) or ( + attribute.name == "days" + and parent_name == "retention_policy" + and atomic_unit.type == "azurerm_network_watcher_flow_log" ): - return [Error("sec_logging", attribute, file, repr(attribute))] + if isinstance(attribute.value, Integer) and attribute.value.value < 90: + return [Error("sec_logging", attribute, file, repr(attribute))] + elif isinstance(attribute.value, String) and ( + not attribute.value.value.isnumeric() or int(attribute.value.value) < 90 + ): + return [Error("sec_logging", attribute, file, repr(attribute))] elif ( attribute.name == "days" and parent_name == "retention_policy" - and atomic_unit.type == "resource.azurerm_monitor_log_profile" - and ( - isinstance(attribute.value, str) - and ( - not attribute.value.isnumeric() - or (attribute.value.isnumeric() and int(attribute.value) < 365) - ) - ) + and atomic_unit.type == "azurerm_monitor_log_profile" ): - return [Error("sec_logging", attribute, file, repr(attribute))] + if isinstance(attribute.value, Integer) and attribute.value.value < 365: + return [Error("sec_logging", attribute, file, repr(attribute))] + elif isinstance(attribute.value, String) and ( + not attribute.value.value.isnumeric() + or int(attribute.value.value) < 365 + ): + return [Error("sec_logging", attribute, file, repr(attribute))] for config in SecurityVisitor.LOGGING: if ( attribute.name == config["attribute"] and atomic_unit.type in config["au_type"] - and parent_name in config["parents"] - and config["values"] != [""] + and self._parent_matches(parent_name, config["parents"]) + and config["values"] != [] ): - if ( - "any_not_empty" in config["values"] - and isinstance(attribute.value, str) - and attribute.value.lower() == "" - ): + if isinstance(attribute.value, (String, Boolean)): + value_str = str(attribute.value.value).lower() + elif hasattr(attribute.value, "code"): + value_str = attribute.value.code.lower() + else: + continue + + if "any_not_empty" in config["values"] and value_str == "": return [Error("sec_logging", attribute, file, repr(attribute))] elif ( "any_not_empty" not in config["values"] - and not attribute.has_variable - and isinstance(attribute.value, str) - and attribute.value.lower() not in config["values"] + and not VariableChecker().check(attribute.value) + and value_str not in config["values"] ): return [Error("sec_logging", attribute, file, repr(attribute))] @@ -274,7 +308,7 @@ def _check_attribute( def check(self, element: CodeElement, file: str) -> List[Error]: errors: List[Error] = [] if isinstance(element, AtomicUnit): - if element.type == "resource.aws_eks_cluster": + if element.type == "aws_eks_cluster": errors.extend( self.__check_log_attribute( element, @@ -290,32 +324,28 @@ def check(self, element: CodeElement, file: str) -> List[Error]: all=True, ) ) - elif element.type == "resource.aws_msk_cluster": + elif element.type == "aws_msk_cluster": broker_logs = self.check_required_attribute( - element.attributes, ["logging_info"], "broker_logs" + element, ["logging_info"], "broker_logs" ) - if isinstance(broker_logs, KeyValue): + if isinstance(broker_logs, UnitBlock): active = False logs_type = ["cloudwatch_logs", "firehose", "s3"] - a_list: List[KeyValue] = [] - for type in logs_type: - log = self.check_required_attribute( - broker_logs.keyvalues, [""], type - ) - if isinstance(log, KeyValue): - enabled = self.check_required_attribute( - log.keyvalues, [""], "enabled" - ) - if ( - isinstance(enabled, KeyValue) - and f"{enabled.value}".lower() == "true" - ): - active = True - elif ( - isinstance(enabled, KeyValue) - and f"{enabled.value}".lower() != "true" - ): - a_list.append(enabled) + a_list: List[Attribute | KeyValue] = [] + for log_type in logs_type: + log = self.check_required_attribute(broker_logs, [], log_type) + if isinstance(log, UnitBlock): + enabled = self.check_required_attribute(log, [], "enabled") + if isinstance(enabled, (Attribute, KeyValue)): + enabled_val = ( + str(enabled.value.value).lower() + if isinstance(enabled.value, Boolean) + else str(enabled.value).lower() + ) + if enabled_val == "true": + active = True + else: + a_list.append(enabled) if not active and a_list == []: errors.append( Error( @@ -341,13 +371,13 @@ def check(self, element: CodeElement, file: str) -> List[Error]: + f"'logging_info.broker_logs.[cloudwatch_logs/firehose/s3].enabled'.", ) ) - elif element.type == "resource.aws_neptune_cluster": + elif element.type == "aws_neptune_cluster": errors.extend( self.__check_log_attribute( element, "enable_cloudwatch_logs_exports", file, ["audit"] ) ) - elif element.type == "resource.aws_docdb_cluster": + elif element.type == "aws_docdb_cluster": errors.extend( self.__check_log_attribute( element, @@ -356,15 +386,20 @@ def check(self, element: CodeElement, file: str) -> List[Error]: ["audit", "profiler"], ) ) - elif element.type == "resource.azurerm_mssql_server": - expr = "\\${azurerm_mssql_server\\." + f"{element.name}\\." + elif element.type == "azurerm_mssql_server": + name = ( + element.name.value + if isinstance(element.name, String) + else str(element.name) + ) + expr = "(\\$\\{)?azurerm_mssql_server\\." + f"{name}\\." pattern = re.compile(rf"{expr}") assoc_au = self.get_associated_au( file, - "resource.azurerm_mssql_server_extended_auditing_policy", + "azurerm_mssql_server_extended_auditing_policy", "server_id", pattern, - [""], + [], ) if not assoc_au: errors.append( @@ -377,15 +412,20 @@ def check(self, element: CodeElement, file: str) -> List[Error]: + f"associated to an 'azurerm_mssql_server' resource.", ) ) - elif element.type == "resource.azurerm_mssql_database": - expr = "\\${azurerm_mssql_database\\." + f"{element.name}\\." + elif element.type == "azurerm_mssql_database": + name = ( + element.name.value + if isinstance(element.name, String) + else str(element.name) + ) + expr = "(\\$\\{)?azurerm_mssql_database\\." + f"{name}\\." pattern = re.compile(rf"{expr}") assoc_au = self.get_associated_au( file, - "resource.azurerm_mssql_database_extended_auditing_policy", + "azurerm_mssql_database_extended_auditing_policy", "database_id", pattern, - [""], + [], ) if not assoc_au: errors.append( @@ -398,20 +438,22 @@ def check(self, element: CodeElement, file: str) -> List[Error]: + f"associated to an 'azurerm_mssql_database' resource.", ) ) - elif element.type == "resource.azurerm_postgresql_configuration": - name = self.check_required_attribute(element.attributes, [""], "name") - value = self.check_required_attribute(element.attributes, [""], "value") + elif element.type == "azurerm_postgresql_configuration": + name_attr = self.check_required_attribute(element, [], "name") + value_attr = self.check_required_attribute(element, [], "value") if ( - isinstance(name, KeyValue) - and isinstance(name.value, str) - and name.value.lower() + isinstance(name_attr, (Attribute, KeyValue)) + and isinstance(name_attr.value, String) + and name_attr.value.value.lower() in ["log_connections", "connection_throttling", "log_checkpoints"] - and isinstance(value, KeyValue) - and isinstance(value.value, str) - and value.value.lower() != "on" + and isinstance(value_attr, (Attribute, KeyValue)) + and isinstance(value_attr.value, String) + and value_attr.value.value.lower() != "on" ): - errors.append(Error("sec_logging", value, file, repr(value))) - elif element.type == "resource.azurerm_monitor_log_profile": + errors.append( + Error("sec_logging", value_attr, file, repr(value_attr)) + ) + elif element.type == "azurerm_monitor_log_profile": errors.extend( self.__check_log_attribute( element, @@ -421,7 +463,7 @@ def check(self, element: CodeElement, file: str) -> List[Error]: all=True, ) ) - elif element.type == "resource.google_sql_database_instance": + elif element.type == "google_sql_database_instance": for flag in SecurityVisitor.GOOGLE_SQL_DATABASE_LOG_FLAGS: required_flag = True if flag["required"] == "no": @@ -434,20 +476,20 @@ def check(self, element: CodeElement, file: str) -> List[Error]: flag["value"], required_flag, ) - elif element.type == "resource.azurerm_storage_container": + elif element.type == "azurerm_storage_container": errors += self.__check_azurerm_storage_container(element, file) - elif element.type == "resource.aws_ecs_cluster": + elif element.type == "aws_ecs_cluster": name = self.check_required_attribute( - element.attributes, ["setting"], "name", "containerinsights" + element, ["setting"], "name", "containerinsights" ) if name is not None: enabled = self.check_required_attribute( - element.attributes, ["setting"], "value" + element, ["setting"], "value" ) - if isinstance(enabled, KeyValue): + if isinstance(enabled, (Attribute, KeyValue)): if ( - isinstance(enabled.value, str) - and enabled.value.lower() != "enabled" + isinstance(enabled.value, String) + and enabled.value.value.lower() != "enabled" ): errors.append( Error("sec_logging", enabled, file, repr(enabled)) @@ -472,11 +514,16 @@ def check(self, element: CodeElement, file: str) -> List[Error]: "Suggestion: check for a required attribute with name 'setting.name' and value 'containerInsights'.", ) ) - elif element.type == "resource.aws_vpc": - expr = "\\${aws_vpc\\." + f"{element.name}\\." + elif element.type == "aws_vpc": + name = ( + element.name.value + if isinstance(element.name, String) + else str(element.name) + ) + expr = "(\\$\\{)?aws_vpc\\." + f"{name}\\." pattern = re.compile(rf"{expr}") assoc_au = self.get_associated_au( - file, "resource.aws_flow_log", "vpc_id", pattern, [""] + file, "aws_flow_log", "vpc_id", pattern, [] ) if not assoc_au: errors.append( @@ -490,21 +537,44 @@ def check(self, element: CodeElement, file: str) -> List[Error]: ) ) + has_dynamic_block = any( + isinstance(s, UnitBlock) and s.name == "dynamic" + for s in element.statements + ) + for config in SecurityVisitor.LOGGING: if ( config["required"] == "yes" and element.type in config["au_type"] - and not self.check_required_attribute( - element.attributes, config["parents"], config["attribute"] + and self.check_required_attribute( + element, config["parents"], config["attribute"] ) + is None ): + parents = config["parents"] + if ( + element.type == "azurerm_storage_account" + and config["attribute"] == "logging" + and len(parents) == 1 + ): + first_parent = self.check_required_attribute( + element, [], parents[0] + ) + if first_parent is None: + continue + if ( + has_dynamic_block + and config["values"] == [] + and len(parents) == 0 + ): + continue errors.append( Error( "sec_logging", element, file, repr(element), - f"Suggestion: check for a required attribute with name '{config['msg']}'.", + f"Suggestion: check for a required attribute with name '{config.get('msg', config['attribute'])}'.", ) ) diff --git a/glitch/analysis/terraform/missing_encryption.py b/glitch/analysis/terraform/missing_encryption.py index e0fcc46a..f7afe651 100644 --- a/glitch/analysis/terraform/missing_encryption.py +++ b/glitch/analysis/terraform/missing_encryption.py @@ -2,8 +2,9 @@ from typing import List from glitch.analysis.terraform.smell_checker import TerraformSmellChecker from glitch.analysis.rules import Error -from glitch.analysis.security import SecurityVisitor -from glitch.repr.inter import AtomicUnit, Attribute, CodeElement, KeyValue +from glitch.analysis.security.visitor import SecurityVisitor +from glitch.analysis.checkers.var_checker import VariableChecker +from glitch.repr.inter import * class TerraformMissingEncryption(TerraformSmellChecker): @@ -18,14 +19,17 @@ def _check_attribute( if ( attribute.name == config["attribute"] and atomic_unit.type in config["au_type"] - and parent_name in config["parents"] - and config["values"] != [""] + and self._parent_matches(parent_name, config["parents"]) + and config["values"] != [] ): - if ( - "any_not_empty" in config["values"] - and isinstance(attribute.value, str) - and attribute.value.lower() == "" - ): + if isinstance(attribute.value, (String, Boolean)): + value_str = str(attribute.value.value).lower() + elif hasattr(attribute.value, "code"): + value_str = attribute.value.code.lower() + else: + continue + + if "any_not_empty" in config["values"] and value_str == "": return [ Error( "sec_missing_encryption", attribute, file, repr(attribute) @@ -33,9 +37,8 @@ def _check_attribute( ] elif ( "any_not_empty" not in config["values"] - and not attribute.has_variable - and isinstance(attribute.value, str) - and attribute.value.lower() not in config["values"] + and not VariableChecker().check(attribute.value) + and value_str not in config["values"] ): return [ Error( @@ -43,7 +46,8 @@ def _check_attribute( ) ] for item in SecurityVisitor.CONFIGURATION_KEYWORDS: - if item.lower() == attribute.name: + if item.lower() == attribute.name and isinstance(attribute.value, String): + value_str = attribute.value.value.lower() for config in SecurityVisitor.ENCRYPT_CONFIG: if atomic_unit.type in config["au_type"]: expr = ( @@ -51,8 +55,7 @@ def _check_attribute( ) pattern = re.compile(rf"{expr}") if ( - isinstance(attribute.value, str) - and not re.search(pattern, attribute.value) + not re.search(pattern, value_str) and config["required"] == "yes" ): return [ @@ -64,8 +67,7 @@ def _check_attribute( ) ] elif ( - isinstance(attribute.value, str) - and re.search(pattern, attribute.value) + re.search(pattern, value_str) and config["required"] == "must_not_exist" ): return [ @@ -82,17 +84,19 @@ def _check_attribute( def check(self, element: CodeElement, file: str) -> List[Error]: errors: List[Error] = [] if isinstance(element, AtomicUnit): - if element.type == "resource.aws_s3_bucket": - expr = "\\${aws_s3_bucket\\." + f"{element.name}\\." + # This does not work when the name is not a String :/ + if element.type == "aws_s3_bucket" and isinstance(element.name, String): + expr = "aws_s3_bucket\\." + f"{element.name.value}\\." pattern = re.compile(rf"{expr}") + r = self.get_associated_au( file, - "resource.aws_s3_bucket_server_side_encryption_configuration", + "aws_s3_bucket_server_side_encryption_configuration", "bucket", pattern, - [""], + [], ) - if not r: + if r is None: errors.append( Error( "sec_missing_encryption", @@ -103,28 +107,27 @@ def check(self, element: CodeElement, file: str) -> List[Error]: + f"associated to an 'aws_s3_bucket' resource.", ) ) - elif element.type == "resource.aws_eks_cluster": + elif element.type == "aws_eks_cluster": resources = self.check_required_attribute( - element.attributes, ["encryption_config"], "resources[0]" + element, ["encryption_config"], "resources" ) - if isinstance(resources, KeyValue): - i = 0 - valid = False - while isinstance(resources, KeyValue): - a = resources - if ( - isinstance(resources.value, str) - and resources.value.lower() == "secrets" - ): - valid = True - break - i += 1 - resources = self.check_required_attribute( - element.attributes, ["encryption_config"], f"resources[{i}]" + if isinstance(resources, Attribute) and isinstance( + resources.value, Array + ): + has_secrets = any( + isinstance(v, String) and v.value.lower() == "secrets" + for v in resources.value.value + ) + if not has_secrets: + errors.append( + Error( + "sec_missing_encryption", + resources, + file, + repr(resources), + ) ) - if not valid: - errors.append(Error("sec_missing_encryption", a, file, repr(a))) # type: ignore - else: + elif resources is None: errors.append( Error( "sec_missing_encryption", @@ -135,17 +138,17 @@ def check(self, element: CodeElement, file: str) -> List[Error]: ) ) elif element.type in [ - "resource.aws_instance", - "resource.aws_launch_configuration", + "aws_instance", + "aws_launch_configuration", ]: ebs_block_device = self.check_required_attribute( - element.attributes, [""], "ebs_block_device" + element, [], "ebs_block_device" ) - if isinstance(ebs_block_device, KeyValue): + if isinstance(ebs_block_device, UnitBlock): encrypted = self.check_required_attribute( - ebs_block_device.keyvalues, [""], "encrypted" + ebs_block_device, [], "encrypted" ) - if not encrypted: + if encrypted is None: errors.append( Error( "sec_missing_encryption", @@ -155,17 +158,15 @@ def check(self, element: CodeElement, file: str) -> List[Error]: f"Suggestion: check for a required attribute with name 'ebs_block_device.encrypted'.", ) ) - elif element.type == "resource.aws_ecs_task_definition": - volume = self.check_required_attribute( - element.attributes, [""], "volume" - ) - if isinstance(volume, KeyValue): + elif element.type == "aws_ecs_task_definition": + volume = self.check_required_attribute(element, [], "volume") + if isinstance(volume, UnitBlock): efs_volume_config = self.check_required_attribute( - volume.keyvalues, [""], "efs_volume_configuration" + volume, [], "efs_volume_configuration" ) - if isinstance(efs_volume_config, KeyValue): + if isinstance(efs_volume_config, UnitBlock): transit_encryption = self.check_required_attribute( - efs_volume_config.keyvalues, [""], "transit_encryption" + efs_volume_config, [], "transit_encryption" ) if not transit_encryption: errors.append( @@ -179,21 +180,26 @@ def check(self, element: CodeElement, file: str) -> List[Error]: ) ) + reported_msgs: set[str] = set() for config in SecurityVisitor.MISSING_ENCRYPTION: + msg = config.get("msg", config["attribute"]) if ( config["required"] == "yes" and element.type in config["au_type"] - and not self.check_required_attribute( - element.attributes, config["parents"], config["attribute"] + and msg not in reported_msgs + and self.check_required_attribute( + element, config["parents"], config["attribute"] ) + is None ): + reported_msgs.add(msg) errors.append( Error( "sec_missing_encryption", element, file, repr(element), - f"Suggestion: check for a required attribute with name '{config['msg']}'.", + f"Suggestion: check for a required attribute with name '{msg}'.", ) ) diff --git a/glitch/analysis/terraform/naming.py b/glitch/analysis/terraform/naming.py index 508f4a2c..3e78ac32 100644 --- a/glitch/analysis/terraform/naming.py +++ b/glitch/analysis/terraform/naming.py @@ -2,8 +2,17 @@ from typing import List from glitch.analysis.terraform.smell_checker import TerraformSmellChecker from glitch.analysis.rules import Error -from glitch.analysis.security import SecurityVisitor -from glitch.repr.inter import AtomicUnit, Attribute, CodeElement, KeyValue +from glitch.analysis.security.visitor import SecurityVisitor +from glitch.analysis.checkers.var_checker import VariableChecker +from glitch.repr.inter import ( + AtomicUnit, + Attribute, + CodeElement, + KeyValue, + Hash, + String, + UnitBlock, +) class TerraformNaming(TerraformSmellChecker): @@ -14,12 +23,10 @@ def _check_attribute( parent_name: str, file: str, ) -> List[Error]: - if attribute.name == "name" and atomic_unit.type in [ - "resource.azurerm_storage_account" - ]: + if attribute.name == "name" and atomic_unit.type in ["azurerm_storage_account"]: pattern = r"^[a-z0-9]{3,24}$" - if isinstance(attribute.value, str) and not re.match( - pattern, attribute.value + if isinstance(attribute.value, String) and not re.match( + pattern, attribute.value.value ): return [Error("sec_naming", attribute, file, repr(attribute))] @@ -27,20 +34,16 @@ def _check_attribute( if ( attribute.name == config["attribute"] and atomic_unit.type in config["au_type"] - and parent_name in config["parents"] - and config["values"] != [""] + and self._parent_matches(parent_name, config["parents"]) + and config["values"] != [] + and isinstance(attribute.value, String) ): - if ( - "any_not_empty" in config["values"] - and isinstance(attribute.value, str) - and attribute.value.lower() == "" - ): + if "any_not_empty" in config["values"] and attribute.value.value == "": return [Error("sec_naming", attribute, file, repr(attribute))] elif ( "any_not_empty" not in config["values"] - and not attribute.has_variable - and isinstance(attribute.value, str) - and attribute.value.lower() not in config["values"] + and not VariableChecker().check(attribute.value) + and attribute.value.value.lower() not in config["values"] ): return [Error("sec_naming", attribute, file, repr(attribute))] @@ -49,15 +52,11 @@ def _check_attribute( def check(self, element: CodeElement, file: str) -> List[Error]: errors: List[Error] = [] if isinstance(element, AtomicUnit): - if element.type == "resource.aws_security_group": - ingress = self.check_required_attribute( - element.attributes, [""], "ingress" - ) - egress = self.check_required_attribute( - element.attributes, [""], "egress" - ) - if isinstance(ingress, KeyValue) and not self.check_required_attribute( - ingress.keyvalues, [""], "description" + if element.type == "aws_security_group": + ingress = self.check_required_attribute(element, [], "ingress") + egress = self.check_required_attribute(element, [], "egress") + if isinstance(ingress, UnitBlock) and not self.check_required_attribute( + ingress, [], "description" ): errors.append( Error( @@ -68,8 +67,8 @@ def check(self, element: CodeElement, file: str) -> List[Error]: f"Suggestion: check for a required attribute with name 'ingress.description'.", ) ) - if isinstance(egress, KeyValue) and not self.check_required_attribute( - egress.keyvalues, [""], "description" + if isinstance(egress, UnitBlock) and not self.check_required_attribute( + egress, [], "description" ): errors.append( Error( @@ -80,15 +79,15 @@ def check(self, element: CodeElement, file: str) -> List[Error]: f"Suggestion: check for a required attribute with name 'egress.description'.", ) ) - elif element.type == "resource.google_container_cluster": + elif element.type == "google_container_cluster": resource_labels = self.check_required_attribute( - element.attributes, [""], "resource_labels", None + element, [], "resource_labels", None ) - if ( - isinstance(resource_labels, KeyValue) - and resource_labels.value is None - ): - if resource_labels.keyvalues == []: + if isinstance(resource_labels, Attribute): + if ( + isinstance(resource_labels.value, Hash) + and len(resource_labels.value.value) == 0 + ): errors.append( Error( "sec_naming", @@ -98,7 +97,7 @@ def check(self, element: CodeElement, file: str) -> List[Error]: f"Suggestion: check empty 'resource_labels'.", ) ) - else: + elif resource_labels is None: errors.append( Error( "sec_naming", @@ -114,7 +113,7 @@ def check(self, element: CodeElement, file: str) -> List[Error]: config["required"] == "yes" and element.type in config["au_type"] and not self.check_required_attribute( - element.attributes, config["parents"], config["attribute"] + element, config["parents"], config["attribute"] ) ): errors.append( @@ -123,7 +122,7 @@ def check(self, element: CodeElement, file: str) -> List[Error]: element, file, repr(element), - f"Suggestion: check for a required attribute with name '{config['msg']}'.", + f"Suggestion: check for a required attribute with name '{config.get('msg', config['attribute'])}'.", ) ) diff --git a/glitch/analysis/terraform/network_policy.py b/glitch/analysis/terraform/network_policy.py index 83c001c0..1897f41e 100644 --- a/glitch/analysis/terraform/network_policy.py +++ b/glitch/analysis/terraform/network_policy.py @@ -2,8 +2,18 @@ from typing import List from glitch.analysis.terraform.smell_checker import TerraformSmellChecker from glitch.analysis.rules import Error -from glitch.analysis.security import SecurityVisitor -from glitch.repr.inter import AtomicUnit, Attribute, CodeElement, KeyValue +from glitch.analysis.security.visitor import SecurityVisitor +from glitch.repr.inter import ( + Array, + AtomicUnit, + Attribute, + Boolean, + CodeElement, + KeyValue, + String, + UnitBlock, +) +from glitch.analysis.checkers.var_checker import VariableChecker class TerraformNetworkSecurityRules(TerraformSmellChecker): @@ -18,87 +28,101 @@ def _check_attribute( if ( attribute.name == rule["attribute"] and atomic_unit.type in rule["au_type"] - and parent_name in rule["parents"] - and not attribute.has_variable - and attribute.value is not None - and attribute.value.lower() not in rule["values"] - and rule["values"] != [""] + and self._parent_matches(parent_name, rule["parents"]) + and not VariableChecker().check(attribute.value) + and rule["values"] != [] ): - return [ - Error( - "sec_network_security_rules", attribute, file, repr(attribute) - ) - ] + if isinstance(attribute.value, (String, Boolean)): + if str(attribute.value.value).lower() not in rule["values"]: + return [ + Error( + "sec_network_security_rules", + attribute, + file, + repr(attribute), + ) + ] + elif ( + hasattr(attribute.value, "code") + and attribute.value.code.lower() not in rule["values"] + ): + return [ + Error( + "sec_network_security_rules", + attribute, + file, + repr(attribute), + ) + ] return [] + def _has_str_value( + self, attr: Attribute | UnitBlock | KeyValue | None, value: str + ) -> bool: + return ( + isinstance(attr, (Attribute, KeyValue)) + and isinstance(attr.value, String) + and attr.value.value.lower() == value + ) + + def _has_str_value_in( + self, attr: Attribute | UnitBlock | KeyValue | None, values: list[str] + ) -> bool: + return ( + isinstance(attr, (Attribute, KeyValue)) + and isinstance(attr.value, String) + and attr.value.value.lower() in values + ) + + def _is_permissive_source( + self, attr: Attribute | UnitBlock | KeyValue | None + ) -> bool: + if not isinstance(attr, (Attribute, KeyValue)) or not isinstance( + attr.value, String + ): + return False + val = attr.value.value.lower() + return val in ["*", "/0", "internet", "any"] or bool(re.match(r"^0.0.0.0", val)) + def check(self, element: CodeElement, file: str) -> List[Error]: errors: List[Error] = [] if isinstance(element, AtomicUnit): - if element.type == "resource.azurerm_network_security_rule": - access = self.check_required_attribute( - element.attributes, [""], "access" - ) - if ( - isinstance(access, KeyValue) - and isinstance(access.value, str) - and access.value.lower() == "allow" - ): - protocol = self.check_required_attribute( - element.attributes, [""], "protocol" - ) - if ( - isinstance(protocol, KeyValue) - and isinstance(protocol.value, str) - and protocol.value.lower() == "udp" - ): + if element.type == "azurerm_network_security_rule": + access = self.check_required_attribute(element, [], "access") + if self._has_str_value(access, "allow"): + protocol = self.check_required_attribute(element, [], "protocol") + if self._has_str_value(protocol, "udp"): errors.append( Error( "sec_network_security_rules", access, file, repr(access) ) ) - elif ( - isinstance(protocol, KeyValue) - and isinstance(protocol.value, str) - and protocol.value.lower() == "tcp" - ): + elif self._has_str_value(protocol, "tcp"): dest_port_range = self.check_required_attribute( - element.attributes, [""], "destination_port_range" + element, [], "destination_port_range" ) - port = ( - isinstance(dest_port_range, KeyValue) - and isinstance(dest_port_range.value, str) - and dest_port_range.value.lower() - in [ - "22", - "3389", - "*", - ] - ) - port_ranges, _ = self.iterate_required_attributes( - element.attributes, - "destination_port_ranges", - lambda x: ( - isinstance(x.value, str) - and x.value.lower() in ["22", "3389", "*"] - ), + port = self._has_str_value_in( + dest_port_range, ["22", "3389", "*"] ) + port_ranges = False + for attr in element.attributes: + if attr.name == "destination_port_ranges" and isinstance( + attr.value, Array + ): + for item in attr.value.value: + if isinstance( + item, String + ) and item.value.lower() in ["22", "3389", "*"]: + port_ranges = True + break + if port or port_ranges: source_address_prefix = self.check_required_attribute( - element.attributes, [""], "source_address_prefix" + element, [], "source_address_prefix" ) - if ( - isinstance(source_address_prefix, KeyValue) - and isinstance(source_address_prefix.value, str) - and ( - source_address_prefix.value.lower() - in ["*", "/0", "internet", "any"] - or re.match( - r"^0.0.0.0", source_address_prefix.value.lower() - ) - ) - ): + if self._is_permissive_source(source_address_prefix): errors.append( Error( "sec_network_security_rules", @@ -107,62 +131,30 @@ def check(self, element: CodeElement, file: str) -> List[Error]: repr(source_address_prefix), ) ) - elif element.type == "resource.azurerm_network_security_group": + + elif element.type == "azurerm_network_security_group": access = self.check_required_attribute( - element.attributes, ["security_rule"], "access" + element, ["security_rule"], "access" ) - if ( - isinstance(access, KeyValue) - and isinstance(access.value, str) - and access.value.lower() == "allow" - ): + if self._has_str_value(access, "allow"): protocol = self.check_required_attribute( - element.attributes, ["security_rule"], "protocol" + element, ["security_rule"], "protocol" ) - if ( - isinstance(protocol, KeyValue) - and isinstance(protocol.value, str) - and protocol.value.lower() == "udp" - ): + if self._has_str_value(protocol, "udp"): errors.append( Error( "sec_network_security_rules", access, file, repr(access) ) ) - elif ( - isinstance(protocol, KeyValue) - and isinstance(protocol.value, str) - and protocol.value.lower() == "tcp" - ): + elif self._has_str_value(protocol, "tcp"): dest_port_range = self.check_required_attribute( - element.attributes, - ["security_rule"], - "destination_port_range", + element, ["security_rule"], "destination_port_range" ) - if ( - isinstance(dest_port_range, KeyValue) - and isinstance(dest_port_range.value, str) - and dest_port_range.value.lower() - in [ - "22", - "3389", - "*", - ] - ): + if self._has_str_value_in(dest_port_range, ["22", "3389", "*"]): source_address_prefix = self.check_required_attribute( - element.attributes, [""], "source_address_prefix" + element, ["security_rule"], "source_address_prefix" ) - if ( - isinstance(source_address_prefix, KeyValue) - and isinstance(source_address_prefix.value, str) - and ( - source_address_prefix.value.lower() - in ["*", "/0", "internet", "any"] - or re.match( - r"^0.0.0.0", source_address_prefix.value.lower() - ) - ) - ): + if self._is_permissive_source(source_address_prefix): errors.append( Error( "sec_network_security_rules", @@ -176,9 +168,10 @@ def check(self, element: CodeElement, file: str) -> List[Error]: if ( rule["required"] == "yes" and element.type in rule["au_type"] - and not self.check_required_attribute( - element.attributes, rule["parents"], rule["attribute"] + and self.check_required_attribute( + element, rule["parents"], rule["attribute"] ) + is None ): errors.append( Error( @@ -186,7 +179,7 @@ def check(self, element: CodeElement, file: str) -> List[Error]: element, file, repr(element), - f"Suggestion: check for a required attribute with name '{rule['msg']}'.", + f"Suggestion: check for a required attribute with name '{rule.get('msg', rule['attribute'])}'.", ) ) diff --git a/glitch/analysis/terraform/permission_iam_policies.py b/glitch/analysis/terraform/permission_iam_policies.py index fa7ed426..f67e470e 100644 --- a/glitch/analysis/terraform/permission_iam_policies.py +++ b/glitch/analysis/terraform/permission_iam_policies.py @@ -2,8 +2,10 @@ from typing import List from glitch.analysis.terraform.smell_checker import TerraformSmellChecker from glitch.analysis.rules import Error -from glitch.analysis.security import SecurityVisitor -from glitch.repr.inter import AtomicUnit, Attribute, CodeElement, KeyValue +from glitch.analysis.security.visitor import SecurityVisitor +from glitch.analysis.checkers.var_checker import VariableChecker +from glitch.analysis.checkers.string_checker import StringChecker +from glitch.repr.inter import AtomicUnit, Attribute, CodeElement, KeyValue, String class TerraformPermissionIAMPolicies(TerraformSmellChecker): @@ -15,59 +17,77 @@ def _check_attribute( file: str, ) -> List[Error]: if ( - (attribute.name == "member" or attribute.name.split("[")[0] == "members") - and atomic_unit.type in SecurityVisitor.GOOGLE_IAM_MEMBER - and isinstance(attribute.value, str) - and ( - re.search(r".-compute@developer.gserviceaccount.com", attribute.value) - or re.search(r".@appspot.gserviceaccount.com", attribute.value) - or re.search(r"user:", attribute.value) + attribute.name == "member" or attribute.name.split("[")[0] == "members" + ) and atomic_unit.type in SecurityVisitor.GOOGLE_IAM_MEMBER: + iam_checker = StringChecker( + lambda x: bool( + re.search(r".-compute@developer.gserviceaccount.com", x) + or re.search(r".@appspot.gserviceaccount.com", x) + or re.search(r"user:", x) + ) ) - ): - return [ - Error("sec_permission_iam_policies", attribute, file, repr(attribute)) - ] + if iam_checker.check(attribute.value): + return [ + Error( + "sec_permission_iam_policies", attribute, file, repr(attribute) + ) + ] for config in SecurityVisitor.PERMISSION_IAM_POLICIES: if ( attribute.name == config["attribute"] and atomic_unit.type in config["au_type"] - and parent_name in config["parents"] + and self._parent_matches(parent_name, config["parents"]) and config["values"] != [""] ): - if ( - config["logic"] == "equal" - and not attribute.has_variable - and isinstance(attribute.value, str) - and attribute.value.lower() not in config["values"] - ) or ( - config["logic"] == "diff" - and isinstance(attribute.value, str) - and attribute.value.lower() in config["values"] - ): - return [ - Error( - "sec_permission_iam_policies", - attribute, - file, - repr(attribute), - ) - ] + if config["logic"] == "equal": + checker = StringChecker( + lambda x, c=config: x.lower() not in c["values"] + ) + if not VariableChecker().check(attribute.value) and checker.check( + attribute.value + ): + return [ + Error( + "sec_permission_iam_policies", + attribute, + file, + repr(attribute), + ) + ] + elif config["logic"] == "diff": + checker = StringChecker( + lambda x, c=config: x.lower() in c["values"] + ) + if checker.check(attribute.value): + return [ + Error( + "sec_permission_iam_policies", + attribute, + file, + repr(attribute), + ) + ] return [] def check(self, element: CodeElement, file: str) -> List[Error]: errors: List[Error] = [] if isinstance(element, AtomicUnit): - if element.type == "resource.aws_iam_user": - expr = "\\${aws_iam_user\\." + f"{element.name}\\." - pattern = re.compile(rf"{expr}") + if element.type == "aws_iam_user": + name = ( + element.name.value + if isinstance(element.name, String) + else element.name + ) + expr = f"aws_iam_user\\.{name}\\." + pattern = re.compile(expr) assoc_au = self.get_associated_au( - file, "resource.aws_iam_user_policy", "user", pattern, [""] + file, "aws_iam_user_policy", "user", pattern, [] ) if assoc_au is not None: a = self.check_required_attribute( - assoc_au.attributes, [""], "user", None, pattern + assoc_au, [], "user", None, pattern ) errors.append( Error("sec_permission_iam_policies", a, file, repr(a)) diff --git a/glitch/analysis/terraform/public_ip.py b/glitch/analysis/terraform/public_ip.py index b249ddc6..45aefb0f 100644 --- a/glitch/analysis/terraform/public_ip.py +++ b/glitch/analysis/terraform/public_ip.py @@ -1,8 +1,16 @@ from typing import List from glitch.analysis.terraform.smell_checker import TerraformSmellChecker from glitch.analysis.rules import Error -from glitch.analysis.security import SecurityVisitor -from glitch.repr.inter import AtomicUnit, Attribute, CodeElement, KeyValue +from glitch.analysis.security.visitor import SecurityVisitor +from glitch.analysis.checkers.var_checker import VariableChecker +from glitch.repr.inter import ( + AtomicUnit, + Attribute, + CodeElement, + KeyValue, + Boolean, + String, +) class TerraformPublicIp(TerraformSmellChecker): @@ -17,13 +25,23 @@ def _check_attribute( if ( attribute.name == config["attribute"] and atomic_unit.type in config["au_type"] - and parent_name in config["parents"] - and not attribute.has_variable - and attribute.value is not None - and attribute.value.lower() not in config["values"] + and ( + parent_name in config["parents"] + or (not config["parents"] and not parent_name) + ) + and not VariableChecker().check(attribute.value) and config["values"] != [""] ): - return [Error("sec_public_ip", attribute, file, repr(attribute))] + if ( + isinstance(attribute.value, Boolean) + and str(attribute.value.value).lower() not in config["values"] + ): + return [Error("sec_public_ip", attribute, file, repr(attribute))] + elif ( + isinstance(attribute.value, String) + and attribute.value.value.lower() not in config["values"] + ): + return [Error("sec_public_ip", attribute, file, repr(attribute))] return [] @@ -35,7 +53,7 @@ def check(self, element: CodeElement, file: str) -> List[Error]: config["required"] == "yes" and element.type in config["au_type"] and not self.check_required_attribute( - element.attributes, config["parents"], config["attribute"] + element, config["parents"], config["attribute"] ) ): errors.append( @@ -52,7 +70,7 @@ def check(self, element: CodeElement, file: str) -> List[Error]: and element.type in config["au_type"] ): a = self.check_required_attribute( - element.attributes, config["parents"], config["attribute"] + element, config["parents"], config["attribute"] ) if a is not None: errors.append(Error("sec_public_ip", a, file, repr(a))) diff --git a/glitch/analysis/terraform/replication.py b/glitch/analysis/terraform/replication.py index cf858270..6e005715 100644 --- a/glitch/analysis/terraform/replication.py +++ b/glitch/analysis/terraform/replication.py @@ -2,8 +2,9 @@ from typing import List from glitch.analysis.terraform.smell_checker import TerraformSmellChecker from glitch.analysis.rules import Error -from glitch.analysis.security import SecurityVisitor -from glitch.repr.inter import AtomicUnit, Attribute, CodeElement, KeyValue +from glitch.analysis.security.visitor import SecurityVisitor +from glitch.repr.inter import AtomicUnit, Attribute, CodeElement, KeyValue, String +from glitch.analysis.checkers.var_checker import VariableChecker class TerraformReplication(TerraformSmellChecker): @@ -14,15 +15,16 @@ def _check_attribute( parent_name: str, file: str, ) -> List[Error]: + var_checker = VariableChecker() for config in SecurityVisitor.REPLICATION: if ( attribute.name == config["attribute"] and atomic_unit.type in config["au_type"] and parent_name in config["parents"] and config["values"] != [""] - and not attribute.has_variable - and isinstance(attribute.value, str) - and attribute.value.lower() not in config["values"] + and not var_checker.check(attribute.value) + and isinstance(attribute.value, String) + and attribute.value.value.lower() not in config["values"] ): return [Error("sec_replication", attribute, file, repr(attribute))] @@ -56,9 +58,10 @@ def check(self, element: CodeElement, file: str) -> List[Error]: if ( config["required"] == "yes" and element.type in config["au_type"] - and not self.check_required_attribute( - element.attributes, config["parents"], config["attribute"] + and self.check_required_attribute( + element, config["parents"], config["attribute"] ) + is None ): errors.append( Error( diff --git a/glitch/analysis/terraform/sensitive_iam_action.py b/glitch/analysis/terraform/sensitive_iam_action.py index 964fec3e..5ef6c1de 100644 --- a/glitch/analysis/terraform/sensitive_iam_action.py +++ b/glitch/analysis/terraform/sensitive_iam_action.py @@ -1,8 +1,15 @@ import json -from typing import List, Dict +from typing import Any, List from glitch.analysis.terraform.smell_checker import TerraformSmellChecker from glitch.analysis.rules import Error -from glitch.repr.inter import AtomicUnit, CodeElement, KeyValue +from glitch.repr.inter import ( + Array, + AtomicUnit, + CodeElement, + KeyValue, + String, + UnitBlock, +) class TerraformSensitiveIAMAction(TerraformSmellChecker): @@ -20,56 +27,63 @@ def convert_string_to_dict(input_string: str): if not isinstance(element, AtomicUnit): return errors - if element.type != "data.aws_iam_policy_document": - return errors - - statements = self.check_required_attribute( - element.attributes, [""], "statement", return_all=True - ) - if isinstance(statements, list): + if element.type == "data.aws_iam_policy_document": + statements = self.get_attributes(element, [], "statement") for statement in statements: - allow = self.check_required_attribute( - statement.keyvalues, [""], "effect" - ) - if ( - isinstance(allow, KeyValue) - and isinstance(allow.value, str) - and allow.value.lower() == "allow" - ) or (not allow): - sensitive_action, action = self.iterate_required_attributes( - statement.keyvalues, - "actions", - lambda x: isinstance(x.value, str) and "*" in x.value.lower(), - ) - if sensitive_action: - errors.append( - Error( - "sec_sensitive_iam_action", action, file, repr(action) - ) - ) + if isinstance(statement, UnitBlock): + effect_value = None + for attr in statement.attributes: + if attr.name == "effect": + if isinstance(attr.value, String): + effect_value = attr.value.value.lower() + elif isinstance(attr.value, str): + effect_value = attr.value.lower() + break - wildcarded_resource, resource = self.iterate_required_attributes( - statement.keyvalues, - "resources", - lambda x: isinstance(x.value, str) - and ((x.value.lower() in ["*"]) or (":*" in x.value.lower())), - ) - if wildcarded_resource: - errors.append( - Error( - "sec_sensitive_iam_action", - resource, - file, - repr(resource), - ) - ) + if effect_value == "allow" or effect_value is None: + for attr in statement.attributes: + if attr.name == "actions" and isinstance(attr.value, Array): + for item in attr.value.value: + item_val = ( + item.value if isinstance(item, String) else item + ) + if isinstance(item_val, str) and "*" in item_val: + errors.append( + Error( + "sec_sensitive_iam_action", + attr, + file, + repr(attr), + ) + ) + break + + if attr.name == "resources" and isinstance( + attr.value, Array + ): + for item in attr.value.value: + item_val = ( + item.value if isinstance(item, String) else item + ) + if isinstance(item_val, str) and ( + item_val == "*" or ":*" in item_val + ): + errors.append( + Error( + "sec_sensitive_iam_action", + attr, + file, + repr(attr), + ) + ) + break elif element.type in [ "resource.aws_iam_role_policy", "resource.aws_iam_policy", "resource.aws_iam_user_policy", "resource.aws_iam_group_policy", ]: - policy = self.check_required_attribute(element.attributes, [""], "policy") + policy = self.check_required_attribute(element, [""], "policy") if not isinstance(policy, KeyValue) or not isinstance(policy.value, str): return errors @@ -77,11 +91,12 @@ def convert_string_to_dict(input_string: str): if not (policy_dict and policy_dict["statement"]): return errors - policy_statements = policy_dict["statement"] - if isinstance(statements, dict): - policy_statements: List[Dict[str, str | List[str]]] = [statements] + policy_statements: List[Any] = policy_dict["statement"] + if isinstance(policy_statements, dict): + policy_statements = [policy_statements] for statement in policy_statements: + statement: Any if not ( statement["effect"] and statement["action"] diff --git a/glitch/analysis/terraform/smell_checker.py b/glitch/analysis/terraform/smell_checker.py index b453e7c5..e9560670 100644 --- a/glitch/analysis/terraform/smell_checker.py +++ b/glitch/analysis/terraform/smell_checker.py @@ -1,13 +1,28 @@ import os -import re from re import Pattern -from typing import Optional, List, Callable, Any +from typing import Optional, List, Callable from glitch.repr.inter import * from glitch.analysis.rules import Error, SmellChecker +from glitch.analysis.checkers.string_checker import StringChecker class TerraformSmellChecker(SmellChecker): + def _parent_matches( + self, parent_name: str, config_parents: list[list[str] | str] + ) -> bool: + if not config_parents and not parent_name: + return True + if not config_parents: + return False + for p in config_parents: + if isinstance(p, list): + if len(p) == 1 and p[0] == parent_name: + return True + elif p == parent_name: + return True + return False + def get_au( self, file: str, @@ -28,7 +43,19 @@ def get_au( return au elif isinstance(c, UnitBlock): for au in c.atomic_units: - if au.type == type and au.name == name: + if isinstance(au.name, String): + au_name = au.name.value + else: + au_name = au.name + if au.type == type and au_name == name: + return au + if ( + type.startswith("resource.") + and au.type == type[9:] + and au_name == name + ): + return au + if type.startswith("data.") and au.type == type and au_name == name: return au return None @@ -59,68 +86,143 @@ def get_associated_au( elif isinstance(code, UnitBlock): for au in code.atomic_units: if au.type == type and self.check_required_attribute( - au.attributes, attribute_parents, attribute_name, None, pattern + au, attribute_parents, attribute_name, None, pattern ): return au return None - def get_attributes_with_name_and_value( + def get_attributes( self, - attributes: List[KeyValue] | List[Attribute], + element: AtomicUnit | Attribute | UnitBlock, parents: List[str], name: str, - value: Optional[Any] = None, - pattern: Optional[Pattern[str]] = None, - ) -> List[KeyValue]: - aux: List[KeyValue] = [] - for a in attributes: - if a.name.split("dynamic.")[-1] == name and parents == [""]: - if ( - value and isinstance(a.value, str) and a.value.lower() == value - ) or ( - pattern - and isinstance(a.value, str) - and re.match(pattern, a.value.lower()) - ): - aux.append(a) - elif ( - value and isinstance(a.value, str) and a.value.lower() != value - ) or ( - pattern - and isinstance(a.value, str) - and not re.match(pattern, a.value.lower()) - ): - continue - elif not value and not pattern: - aux.append(a) - elif a.name.split("dynamic.")[-1] in parents: - aux += self.get_attributes_with_name_and_value( - a.keyvalues, [""], name, value, pattern - ) - elif a.keyvalues != []: - aux += self.get_attributes_with_name_and_value( - a.keyvalues, parents, name, value, pattern - ) - return aux + ) -> List[Attribute | UnitBlock | KeyValue]: + res: List[Attribute | UnitBlock | KeyValue] = [] + if isinstance(element, AtomicUnit): + for attr in element.attributes: + res.extend(self.get_attributes(attr, parents, name)) + for ub in element.statements: + if isinstance(ub, UnitBlock): + res.extend(self.get_attributes(ub, parents, name)) + elif ( + isinstance(element, Attribute) + and len(parents) > 0 + and element.name == parents[0] + ): + if isinstance(element.value, Hash): + for k, v in element.value.value.items(): + key_name = k.value if isinstance(k, VariableReference) else str(k) + if len(parents) == 1 and key_name == name: + elem_info = ElementInfo( + k.line, k.column, k.end_line, k.end_column, k.code + ) + res.append(KeyValue(key_name, v, elem_info)) + elif isinstance(element, UnitBlock) and element.type == UnitBlockType.block: + if len(parents) > 0: + matched = element.name == parents[0] + next_parents = parents[1:] if matched else parents + if matched: + for attribute in element.attributes: + res.extend(self.get_attributes(attribute, next_parents, name)) + for ub in element.statements + element.unit_blocks: + if isinstance(ub, UnitBlock): + res.extend(self.get_attributes(ub, next_parents, name)) + elif element.name == name: + res.append(element) + else: + for attribute in element.attributes: + res.extend(self.get_attributes(attribute, parents, name)) + for ub in element.statements + element.unit_blocks: + if isinstance(ub, UnitBlock): + res.extend(self.get_attributes(ub, parents, name)) + elif len(parents) == 0 and element.name == name: + res.append(element) - def check_required_attribute( + return res + + def get_attribute( self, - attributes: List[Attribute] | List[KeyValue], + element: AtomicUnit | Attribute | UnitBlock, parents: List[str], name: str, - value: Optional[Any] = None, + ) -> Attribute | UnitBlock | KeyValue | None: + attributes = self.get_attributes(element, parents, name) + return attributes[0] if len(attributes) > 0 else None + + def check_required_attribute( + self, + atomic_unit: AtomicUnit | UnitBlock, + parents: List[str] | List[List[str]], + name: str, + value: Optional[str] = None, pattern: Optional[Pattern[str]] = None, - return_all: bool = False, - ) -> Union[Optional[KeyValue], List[KeyValue]]: - attributes = self.get_attributes_with_name_and_value( - attributes, parents, name, value, pattern - ) - if attributes != []: - if return_all: - return attributes # type: ignore - return attributes[0] + ) -> Attribute | UnitBlock | KeyValue | None: + # Handle [0] suffix - check if array attribute has at least one element + if name.endswith("[0]"): + base_name = name[:-3] + element = self.check_required_attribute(atomic_unit, parents, base_name) + if ( + element is not None + and isinstance(element, Attribute) + and isinstance(element.value, Array) + and len(element.value.value) > 0 + ): + return element + return None - return None + element = None + # In the case we have a list, we consider that one of the + # parents list must be satisfied. This is particularly useful + # when attributes changes its location between Terraform versions. + has_parents_list = False + for parents_list in parents: + if isinstance(parents_list, list): + has_parents_list = True + element = self.get_attribute(atomic_unit, parents_list, name) + if element is not None: + break + if not has_parents_list: + element = self.get_attribute(atomic_unit, parents, name) # type: ignore + + if value is not None: + if value == "any_not_empty": + value_checker = StringChecker(lambda x: len(x.strip()) > 0) + else: + value_checker = StringChecker(lambda x: x.lower() == value.lower()) + + if element is not None and isinstance(element, Attribute): + if ( + value == "true" + and isinstance(element.value, Boolean) + and element.value.value + ): + return element + elif ( + value == "false" + and isinstance(element.value, Boolean) + and not element.value.value + ): + return element + elif value == "non_empty_list": + if ( + isinstance(element.value, Array) + and len(element.value.value) > 0 + ): + return element + elif value_checker.check(element.value): + return element + return None + elif ( + pattern is not None + and element is not None + and isinstance(element, Attribute) + ): + # HACK: using the code is sort of an hack (avoids dealing with Access) + if pattern.match(element.value.code) is not None: + return element + return None + else: + return element def check_database_flags( self, @@ -131,33 +233,42 @@ def check_database_flags( safe_value: str, required_flag: bool = True, ) -> List[Error]: - database_flags = self.get_attributes_with_name_and_value( - au.attributes, ["settings"], "database_flags" - ) + database_flags = self.get_attributes(au, ["settings"], "database_flags") found_flag = False errors: List[Error] = [] if database_flags != []: for flag in database_flags: + if isinstance(flag, Attribute) or flag.name is None: + continue + + # Fake AtomicUnit to use the check_required_attribute method + fake_au = AtomicUnit(Null(), "") + fake_au.statements = [flag] name = self.check_required_attribute( - flag.keyvalues, [""], "name", flag_name + fake_au, [flag.name], "name", flag_name ) + # Attribute not found but it is not required + if name is None and not required_flag: + continue + + # Attribute found if name is not None: found_flag = True - value = self.check_required_attribute(flag.keyvalues, [""], "value") - if ( - isinstance(value, KeyValue) - and isinstance(value.value, str) - and value.value.lower() != safe_value - ): - errors.append(Error(smell, value, file, repr(value))) - break - elif not value and required_flag: + value = self.check_required_attribute( + fake_au, [flag.name], "value", value=safe_value + ) + # But value is not correct + if value is None: + value_attr = self.check_required_attribute( + fake_au, [flag.name], "value" + ) + error_element = value_attr if value_attr is not None else flag errors.append( Error( smell, - flag, + error_element, file, - repr(flag), + repr(error_element), f"Suggestion: check for a required attribute with name 'value'.", ) ) @@ -180,14 +291,17 @@ def iterate_required_attributes( name: str, check: Callable[[KeyValue], bool], ): + fake_au = AtomicUnit(Null(), "") + fake_au.attributes = list(attributes) # type: ignore + i = 0 - attribute = self.check_required_attribute(attributes, [""], f"{name}[{i}]") + attribute = self.check_required_attribute(fake_au, [""], f"{name}[{i}]") while isinstance(attribute, KeyValue): if check(attribute): return True, attribute i += 1 - attribute = self.check_required_attribute(attributes, [""], f"{name}[{i}]") + attribute = self.check_required_attribute(fake_au, [""], f"{name}[{i}]") return False, None @@ -202,21 +316,46 @@ def _check_attribute( def __check_attribute( self, - attribute: Attribute | KeyValue, + element: Attribute | UnitBlock, atomic_unit: AtomicUnit, parent_name: str, file: str, ) -> List[Error]: errors: List[Error] = [] - errors += self._check_attribute(attribute, atomic_unit, parent_name, file) - for attr_child in attribute.keyvalues: - errors += self.__check_attribute( - attr_child, atomic_unit, attribute.name, file - ) + if isinstance(element, Attribute): + errors += self._check_attribute(element, atomic_unit, parent_name, file) + if isinstance(element.value, Hash): + for k, v in element.value.value.items(): + key_name = k.value if isinstance(k, VariableReference) else str(k) + elem_info = ElementInfo( + k.line, k.column, k.end_line, k.end_column, k.code + ) + hash_attr = KeyValue(key_name, v, elem_info) + errors += self._check_attribute( + hash_attr, atomic_unit, element.name, file + ) + elif element.type == UnitBlockType.block: + for attr in element.attributes: + errors += self._check_attribute(attr, atomic_unit, element.name, file) # type: ignore + for statement in element.statements: + if ( + isinstance(statement, UnitBlock) + and statement.type == UnitBlockType.block + ): + errors += self.__check_attribute(statement, atomic_unit, element.name, file) # type: ignore + for ub in element.unit_blocks: + if ub.type == UnitBlockType.block: + errors += self.__check_attribute(ub, atomic_unit, element.name, file) # type: ignore return errors - def _check_attributes(self, atomic_unit: AtomicUnit, file: str) -> List[Error]: + def _check_attributes(self, element: AtomicUnit, file: str) -> List[Error]: errors: List[Error] = [] - for attribute in atomic_unit.attributes: - errors += self.__check_attribute(attribute, atomic_unit, "", file) + for attribute in element.attributes: + errors += self.__check_attribute(attribute, element, "", file) + for statement in element.statements: + if ( + isinstance(statement, UnitBlock) + and statement.type == UnitBlockType.block + ): + errors += self.__check_attribute(statement, element, "", file) return errors diff --git a/glitch/analysis/terraform/ssl_tls_policy.py b/glitch/analysis/terraform/ssl_tls_policy.py index 97a7b230..31b5e555 100644 --- a/glitch/analysis/terraform/ssl_tls_policy.py +++ b/glitch/analysis/terraform/ssl_tls_policy.py @@ -1,8 +1,17 @@ from typing import List from glitch.analysis.terraform.smell_checker import TerraformSmellChecker from glitch.analysis.rules import Error -from glitch.analysis.security import SecurityVisitor -from glitch.repr.inter import AtomicUnit, Attribute, KeyValue, CodeElement +from glitch.analysis.security.visitor import SecurityVisitor +from glitch.analysis.checkers.var_checker import VariableChecker +from glitch.repr.inter import ( + AtomicUnit, + Attribute, + Boolean, + KeyValue, + CodeElement, + String, + Value, +) class TerraformSslTlsPolicy(TerraformSmellChecker): @@ -17,12 +26,30 @@ def _check_attribute( if ( attribute.name == policy["attribute"] and atomic_unit.type in policy["au_type"] - and parent_name in policy["parents"] - and not attribute.has_variable - and attribute.value is not None - and attribute.value.lower() not in policy["values"] + and self._parent_matches(parent_name, policy["parents"]) + and not VariableChecker().check(attribute.value) ): - return [Error("sec_ssl_tls_policy", attribute, file, repr(attribute))] + if ( + isinstance(attribute.value, Boolean) + and str(attribute.value.value).lower() not in policy["values"] + ): + return [ + Error("sec_ssl_tls_policy", attribute, file, repr(attribute)) + ] + elif ( + isinstance(attribute.value, String) + and attribute.value.value.lower() not in policy["values"] + ): + return [ + Error("sec_ssl_tls_policy", attribute, file, repr(attribute)) + ] + elif ( + isinstance(attribute.value, str) + and attribute.value.lower() not in policy["values"] + ): + return [ + Error("sec_ssl_tls_policy", attribute, file, repr(attribute)) + ] return [] @@ -30,19 +57,18 @@ def check(self, element: CodeElement, file: str) -> List[Error]: errors: List[Error] = [] if isinstance(element, AtomicUnit): if element.type in [ - "resource.aws_alb_listener", - "resource.aws_lb_listener", + "aws_alb_listener", + "aws_lb_listener", ]: - protocol = self.check_required_attribute( - element.attributes, [""], "protocol" - ) + protocol = self.check_required_attribute(element, [], "protocol") if ( isinstance(protocol, KeyValue) - and isinstance(protocol.value, str) - and protocol.value.lower() in ["https", "tls"] + and isinstance(protocol.value, Value) + and isinstance(protocol.value.value, str) + and protocol.value.value.lower() in ["https", "tls"] ): ssl_policy = self.check_required_attribute( - element.attributes, [""], "ssl_policy" + element, [], "ssl_policy" ) if not ssl_policy: errors.append( @@ -60,16 +86,17 @@ def check(self, element: CodeElement, file: str) -> List[Error]: policy["required"] == "yes" and element.type in policy["au_type"] and not self.check_required_attribute( - element.attributes, policy["parents"], policy["attribute"] + element, policy["parents"], policy["attribute"] ) ): + attribute = policy["attribute"] errors.append( Error( "sec_ssl_tls_policy", element, file, repr(element), - f"Suggestion: check for a required attribute with name '{policy['msg']}'.", + f"Suggestion: check for a required attribute with name '{attribute}'.", ) ) diff --git a/glitch/analysis/terraform/threats_detection.py b/glitch/analysis/terraform/threats_detection.py index 2ae509f4..99e1bc07 100644 --- a/glitch/analysis/terraform/threats_detection.py +++ b/glitch/analysis/terraform/threats_detection.py @@ -1,8 +1,16 @@ from typing import List from glitch.analysis.terraform.smell_checker import TerraformSmellChecker from glitch.analysis.rules import Error -from glitch.analysis.security import SecurityVisitor -from glitch.repr.inter import AtomicUnit, Attribute, KeyValue, CodeElement +from glitch.analysis.security.visitor import SecurityVisitor +from glitch.analysis.checkers.var_checker import VariableChecker +from glitch.repr.inter import ( + AtomicUnit, + Attribute, + KeyValue, + CodeElement, + String, + Boolean, +) class TerraformThreatsDetection(TerraformSmellChecker): @@ -17,13 +25,19 @@ def _check_attribute( if ( attribute.name == config["attribute"] and atomic_unit.type in config["au_type"] - and parent_name in config["parents"] - and config["values"] != [""] + and self._parent_matches(parent_name, config["parents"]) + and config["values"] != [] ): + value_str = None + if isinstance(attribute.value, String): + value_str = attribute.value.value.lower() + elif isinstance(attribute.value, Boolean): + value_str = "true" if attribute.value.value else "false" + if ( "any_not_empty" in config["values"] - and isinstance(attribute.value, str) - and attribute.value.lower() == "" + and value_str is not None + and value_str == "" ): return [ Error( @@ -35,9 +49,9 @@ def _check_attribute( ] elif ( "any_not_empty" not in config["values"] - and not attribute.has_variable - and isinstance(attribute.value, str) - and attribute.value.lower() not in config["values"] + and value_str is not None + and not VariableChecker().check(attribute.value) + and value_str not in config["values"] ): return [ Error( @@ -54,28 +68,31 @@ def check(self, element: CodeElement, file: str) -> List[Error]: errors: List[Error] = [] if isinstance(element, AtomicUnit): for config in SecurityVisitor.MISSING_THREATS_DETECTION_ALERTS: - if ( - config["required"] == "yes" - and element.type in config["au_type"] - and not self.check_required_attribute( - element.attributes, config["parents"], config["attribute"] - ) - ): - errors.append( - Error( - "sec_threats_detection_alerts", - element, - file, - repr(element), - f"Suggestion: check for a required attribute with name '{config['msg']}'.", + if config["required"] == "yes" and element.type in config["au_type"]: + attr_name = config["attribute"] + if ( + self.check_required_attribute( + element, config["parents"], attr_name + ) + is None + ): + msg = config.get("msg", attr_name) + errors.append( + Error( + "sec_threats_detection_alerts", + element, + file, + repr(element), + f"Suggestion: check for a required attribute with name '{msg}'.", + ) ) - ) elif ( config["required"] == "must_not_exist" and element.type in config["au_type"] ): + attr_name = config["attribute"] a = self.check_required_attribute( - element.attributes, config["parents"], config["attribute"] + element, config["parents"], attr_name ) if a is not None: errors.append( diff --git a/glitch/analysis/terraform/versioning.py b/glitch/analysis/terraform/versioning.py index 5d640d10..a794e099 100644 --- a/glitch/analysis/terraform/versioning.py +++ b/glitch/analysis/terraform/versioning.py @@ -1,8 +1,10 @@ from typing import List from glitch.analysis.terraform.smell_checker import TerraformSmellChecker from glitch.analysis.rules import Error -from glitch.analysis.security import SecurityVisitor -from glitch.repr.inter import AtomicUnit, Attribute, KeyValue, CodeElement +from glitch.analysis.security.visitor import SecurityVisitor +from glitch.repr.inter import AtomicUnit, Attribute, KeyValue, CodeElement, Boolean +from glitch.analysis.checkers.var_checker import VariableChecker +from glitch.analysis.checkers.string_checker import StringChecker class TerraformVersioning(TerraformSmellChecker): @@ -13,17 +15,25 @@ def _check_attribute( parent_name: str, file: str, ) -> List[Error]: + var_checker = VariableChecker() for config in SecurityVisitor.VERSIONING: + string_checker = StringChecker(lambda x: x.lower() not in config["values"]) if ( attribute.name == config["attribute"] and atomic_unit.type in config["au_type"] and parent_name in config["parents"] and config["values"] != [""] - and not attribute.has_variable - and isinstance(attribute.value, str) - and attribute.value.lower() not in config["values"] + and not var_checker.check(attribute.value) ): - return [Error("sec_versioning", attribute, file, repr(attribute))] + if ( + isinstance(attribute.value, Boolean) + and str(attribute.value.value).lower() not in config["values"] + ): + return [Error("sec_versioning", attribute, file, repr(attribute))] + elif isinstance(attribute.value, str) and string_checker.check( + attribute.value + ): + return [Error("sec_versioning", attribute, file, repr(attribute))] return [] @@ -34,9 +44,10 @@ def check(self, element: CodeElement, file: str) -> List[Error]: if ( config["required"] == "yes" and element.type in config["au_type"] - and not self.check_required_attribute( - element.attributes, config["parents"], config["attribute"] + and self.check_required_attribute( + element, config["parents"], config["attribute"] ) + is None ): errors.append( Error( diff --git a/glitch/analysis/terraform/weak_password_key_policy.py b/glitch/analysis/terraform/weak_password_key_policy.py index 052d92a1..08190dd9 100644 --- a/glitch/analysis/terraform/weak_password_key_policy.py +++ b/glitch/analysis/terraform/weak_password_key_policy.py @@ -1,8 +1,17 @@ from typing import List from glitch.analysis.terraform.smell_checker import TerraformSmellChecker from glitch.analysis.rules import Error -from glitch.analysis.security import SecurityVisitor -from glitch.repr.inter import AtomicUnit, Attribute, KeyValue, CodeElement +from glitch.analysis.security.visitor import SecurityVisitor +from glitch.analysis.checkers.var_checker import VariableChecker +from glitch.repr.inter import ( + AtomicUnit, + Attribute, + KeyValue, + CodeElement, + Integer, + Boolean, + String, +) class TerraformWeakPasswordKeyPolicy(TerraformSmellChecker): @@ -17,14 +26,17 @@ def _check_attribute( if ( attribute.name == policy["attribute"] and atomic_unit.type in policy["au_type"] - and parent_name in policy["parents"] + and ( + parent_name in policy["parents"] + or (not policy["parents"] and not parent_name) + ) and policy["values"] != [""] ): if policy["logic"] == "equal": if ( "any_not_empty" in policy["values"] - and isinstance(attribute.value, str) - and attribute.value.lower() == "" + and isinstance(attribute.value, String) + and attribute.value.value == "" ): return [ Error( @@ -36,9 +48,9 @@ def _check_attribute( ] elif ( "any_not_empty" not in policy["values"] - and not attribute.has_variable - and isinstance(attribute.value, str) - and attribute.value.lower() not in policy["values"] + and not VariableChecker().check(attribute.value) + and isinstance(attribute.value, String) + and attribute.value.value.lower() not in policy["values"] ): return [ Error( @@ -48,15 +60,33 @@ def _check_attribute( repr(attribute), ) ] - elif ( - policy["logic"] == "gte" - and isinstance(attribute.value, str) - and not attribute.value.isnumeric() - ) or ( - policy["logic"] == "gte" - and isinstance(attribute.value, str) - and attribute.value.isnumeric() - and int(attribute.value) < int(policy["values"][0]) + elif ( + "any_not_empty" not in policy["values"] + and isinstance(attribute.value, Boolean) + and str(attribute.value.value).lower() not in policy["values"] + ): + return [ + Error( + "sec_weak_password_key_policy", + attribute, + file, + repr(attribute), + ) + ] + elif policy["logic"] == "gte" and ( + ( + isinstance(attribute.value, Integer) + and attribute.value.value < int(policy["values"][0]) + ) + or ( + isinstance(attribute.value, str) + and not attribute.value.isnumeric() + ) + or ( + isinstance(attribute.value, str) + and attribute.value.isnumeric() + and int(attribute.value) < int(policy["values"][0]) + ) ): return [ Error( @@ -66,15 +96,20 @@ def _check_attribute( repr(attribute), ) ] - elif ( - policy["logic"] == "lte" - and isinstance(attribute.value, str) - and not attribute.value.isnumeric() - ) or ( - policy["logic"] == "lte" - and isinstance(attribute.value, str) - and attribute.value.isnumeric() - and int(attribute.value) > int(policy["values"][0]) + elif policy["logic"] == "lte" and ( + ( + isinstance(attribute.value, Integer) + and attribute.value.value > int(policy["values"][0]) + ) + or ( + isinstance(attribute.value, str) + and not attribute.value.isnumeric() + ) + or ( + isinstance(attribute.value, str) + and attribute.value.isnumeric() + and int(attribute.value) > int(policy["values"][0]) + ) ): return [ Error( @@ -95,7 +130,7 @@ def check(self, element: CodeElement, file: str) -> List[Error]: policy["required"] == "yes" and element.type in policy["au_type"] and not self.check_required_attribute( - element.attributes, policy["parents"], policy["attribute"] + element, policy["parents"], policy["attribute"] ) ): errors.append( diff --git a/glitch/configs/default.ini b/glitch/configs/default.ini index f2efba19..fc3b1211 100644 --- a/glitch/configs/default.ini +++ b/glitch/configs/default.ini @@ -30,6 +30,8 @@ github_actions_resources = [] file_commands = ["file", "chmod", "mkdir"] shell_resources = ["shell", "execute", "exec"] ip_binding_commands = ["ip_bind_all"] +obsolete_commands = ["pack"] +null_values = ["nil", ""] [design] exec_atomic_units = ["exec"] diff --git a/glitch/configs/default.json b/glitch/configs/default.json new file mode 100644 index 00000000..961d6db2 --- /dev/null +++ b/glitch/configs/default.json @@ -0,0 +1,78 @@ +{ + "security": { + "suspicious_words": [ + "bug", "debug", "todo", "to-do", "to_do", "to be implemented", "fix", + "issue", "issues", "problem", "solve", "hack", "ticket", "later", + "incorrect", "fixme" + ], + "passwords": [ + "pass", "pwd", "password", "passwd", "passno", "pass-no", "pass_no" + ], + "users": [ + "root", "user", "uname", "username", "user-name", "user_name", + "owner-name", "owner_name", "admin", "login", "userid", "loginid" + ], + "profile": [ + "user-profile" + ], + "secrets": [ + "auth_token", "authetication_token", "auth-token", "authentication-token", + "secret", "uuid", "crypt", "certificate", "token", "ssh_key", "md5", + "rsa", "ssl_content", "ca_content", "ssl-content", "ca-content", + "ssh_key_content", "ssh-key-content", "ssh_key_public", "ssh-key-public", + "ssh_key_private", "ssh-key-private", "ssh_key_public_content", + "ssh_key_private_content", "ssh-key-public-content", "ssh-key-private-content" + ], + "misc_secrets": [ + "key", "cert" + ], + "roles": [], + "download_extensions": [ + "iso", "tar", "tar.gz", "tar.bzip2", "zip", + "rar", "gzip", "gzip2", "deb", "rpm", "sh", "run", "bin", "tgz" + ], + "ssh_dirs": [ + "source", "destination", "path", "directory", "src", "dest", "file" + ], + "admin": [ + "admin", "root" + ], + "checksum": [ + "gpg", "checksum" + ], + "weak_crypt": [ + "md5", "sha1", "arcfour" + ], + "weak_crypt_whitelist": [ + "checksum" + ], + "url_http_white_list": [ + "localhost", "127.0.0.1" + ], + "secrets_white_list": [], + "sensitive_data": [], + "secret_value_assign": [], + "github_actions_resources": [], + "file_commands": [ + "file", "chmod", "mkdir" + ], + "shell_resources": [ + "shell", "execute", "exec" + ], + "ip_binding_commands": [ + "ip_bind_all" + ], + "obsolete_commands": [ + "pack" + ], + "null_values": [ + "nil", "" + ] + }, + "design": { + "exec_atomic_units": [ + "exec" + ], + "default_variables": [] + } +} \ No newline at end of file diff --git a/glitch/configs/terraform.ini b/glitch/configs/terraform.ini index 7f148ba0..b5e57577 100644 --- a/glitch/configs/terraform.ini +++ b/glitch/configs/terraform.ini @@ -27,344 +27,337 @@ secret_value_assign = ["key_id=", "access_key=", "key=", "database_password=", " policy_keywords = ["policy"] policy_insecure_access_control = [{"keyword": "\"principal\":", "value": "\"\\*\""}, {"keyword": "\"action\":", "value": "\"\\*\""}] -policy_authentication = [{"au_type": ["resource.aws_iam_group_policy"], "keyword": "\"aws:multifactorauthpresent\":", +policy_authentication = [{"au_type": ["aws_iam_group_policy"], "keyword": "\"aws:multifactorauthpresent\":", "value": "\\[\"true\"\\]"}] configuration_keywords = ["configuration"] -encrypt_configuration = [{"au_type": ["resource.aws_emr_security_configuration"], +encrypt_configuration = [{"au_type": ["aws_emr_security_configuration"], "keyword": "\"enableatrestencryption\":", "value": "true", "required": "yes"}, - {"au_type": ["resource.aws_emr_security_configuration"], "keyword": "\"enableintransitencryption\":", + {"au_type": ["aws_emr_security_configuration"], "keyword": "\"enableintransitencryption\":", "value": "true", "required": "yes"}, - {"au_type": ["resource.aws_emr_security_configuration"], "keyword": "\"encryptionmode\":", + {"au_type": ["aws_emr_security_configuration"], "keyword": "\"encryptionmode\":", "value": "\"sse-kms\"", "required": "yes"}, - {"au_type": ["resource.aws_emr_security_configuration"], "keyword": "\"encryptionkeyprovidertype\":", + {"au_type": ["aws_emr_security_configuration"], "keyword": "\"encryptionkeyprovidertype\":", "value": "\"\"", "required": "must_not_exist"}, - {"au_type": ["resource.aws_emr_security_configuration"], "keyword": "\"awskmskey\":", + {"au_type": ["aws_emr_security_configuration"], "keyword": "\"awskmskey\":", "value": "\"\"", "required": "must_not_exist"}] secrets_white_list = ["cloudfront_default_certificate", "client_cert_enabled", "api_key_required", "issue_client_certificate", "kms_key_id", "kms_key_arn", "key_arn", "performance_insights_kms_key_id", "kms_master_key_id", "kms_key_self_link", "bypass", "enable_key_rotation", "storage_account_access_key_is_secondary", "key_pair", "default_kms_key_name"] -github_actions_resources = ["resource.github_actions_environment_secret", - "resource.github_actions_organization_secret", "resource.github_actions_secret"] +github_actions_resources = ["github_actions_environment_secret", + "github_actions_organization_secret", "github_actions_secret"] -integrity_policy = [{"au_type": ["resource.google_compute_instance"], "attribute": "enable_integrity_monitoring", +integrity_policy = [{"au_type": ["google_compute_instance"], "attribute": "enable_integrity_monitoring", "parents": ["shielded_instance_config"], "values": ["true"], "required": "no"}, - {"au_type": ["resource.google_compute_instance"], "attribute": "enable_vtpm", + {"au_type": ["google_compute_instance"], "attribute": "enable_vtpm", "parents": ["shielded_instance_config"], "values": ["true"], "required": "no"}, - {"au_type": ["resource.aws_ecr_repository"], "attribute": "image_tag_mutability", "parents": [""], + {"au_type": ["aws_ecr_repository"], "attribute": "image_tag_mutability", "parents": [], "values": ["immutable"], "required": "yes", "msg": "image_tag_mutability"}] -ensure_https = [{"au_type": ["resource.aws_cloudfront_distribution"], "attribute": "viewer_protocol_policy", - "parents": ["default_cache_behavior", "ordered_cache_behavior"], "values":["redirect-to-https", "https-only"], - "required": "yes", "msg": "cache_behavior.viewer_protocol_policy"}, - {"au_type": ["resource.aws_elasticsearch_domain"], "attribute": "enforce_https", "parents": ["domain_endpoint_options"], - "values": ["true"], "required": "yes", "msg": "domain_endpoint_options.enforce_https"}, - {"au_type": ["resource.aws_alb_listener", "resource.aws_lb_listener"], "attribute": "protocol", "parents": [""], - "values": ["https", "tls"], "required": "yes", "msg": "protocol"}, - {"au_type": ["resource.azurerm_app_service", "resource.azurerm_app_service_slot", "resource.azurerm_function_app", - "resource.azurerm_function_app_slot", "resource.azurerm_linux_web_app", "resource.azurerm_windows_web_app"], - "attribute": "https_only", "parents": [""], "values": ["true"], "required": "yes", "msg": "https_only"}, - {"au_type": ["resource.azurerm_storage_account"], "attribute": "enable_https_traffic_only", - "parents": [""], "values": ["true"], "required": "no"}, - {"au_type": ["resource.digitalocean_loadbalancer"], "attribute": "entry_protocol", +ensure_https = [{"au_type": ["aws_cloudfront_distribution"], "attribute": "viewer_protocol_policy", + "parents": [["default_cache_behavior"], ["ordered_cache_behavior"]], "values":["redirect-to-https", "https-only"], + "required": "yes"}, + {"au_type": ["aws_elasticsearch_domain"], "attribute": "enforce_https", "parents": ["domain_endpoint_options"], + "values": ["true"], "required": "yes"}, + {"au_type": ["aws_alb_listener", "aws_lb_listener"], "attribute": "protocol", "parents": [], + "values": ["https", "tls"], "required": "yes"}, + {"au_type": ["azurerm_app_service", "azurerm_app_service_slot", "azurerm_function_app", + "azurerm_function_app_slot", "azurerm_linux_web_app", "azurerm_windows_web_app"], + "attribute": "https_only", "parents": [], "values": ["true"], "required": "yes"}, + {"au_type": ["azurerm_storage_account"], "attribute": "enable_https_traffic_only", + "parents": [], "values": ["true"], "required": "no"}, + {"au_type": ["digitalocean_loadbalancer"], "attribute": "entry_protocol", "parents": ["forwarding_rule"], "values": ["https"], "required": "no"}] -ssl_tls_policy = [{"au_type": ["resource.aws_api_gateway_domain_name"], "attribute": "security_policy", - "parents": [""], "values": ["tls_1_2", "tls_1_3"], "required": "yes", "msg": "security_policy"}, - {"au_type": ["resource.aws_cloudfront_distribution"], "attribute": "minimum_protocol_version", +ssl_tls_policy = [{"au_type": ["aws_api_gateway_domain_name"], "attribute": "security_policy", + "parents": [], "values": ["tls_1_2", "tls_1_3"], "required": "yes"}, + {"au_type": ["aws_cloudfront_distribution"], "attribute": "minimum_protocol_version", "parents": ["viewer_certificate"], "values": ["tlsv1.2_2018", "tlsv1.2_2019", "tlsv1.2_2021"], "required": "yes", "msg": "viewer_certificate.minimum_protocol_version"}, - {"au_type": ["resource.aws_elasticsearch_domain"], "attribute": "tls_security_policy", + {"au_type": ["aws_elasticsearch_domain"], "attribute": "tls_security_policy", "parents": ["domain_endpoint_options"], "values": ["policy-min-tls-1-2-2019-07"], "required": "yes", "msg": "domain_endpoint_options.tls_security_policy"}, - {"au_type": ["resource.aws_alb_listener", "resource.aws_lb_listener"], "attribute": "ssl_policy", "parents": [""], + {"au_type": ["aws_alb_listener", "aws_lb_listener"], "attribute": "ssl_policy", "parents": [], "values": ["elbsecuritypolicy-tls-1-2-2017-01", "elbsecuritypolicy-tls-1-2-ext-2018-06"], "required": "no"}, - {"au_type": ["resource.azurerm_app_service", "resource.azurerm_app_service_slot", "resource.azurerm_function_app"], + {"au_type": ["azurerm_app_service", "azurerm_app_service_slot", "azurerm_function_app"], "parents": ["site_config"], "attribute": "min_tls_version", "values": ["1.2"], "required": "no"}, - {"au_type": ["resource.google_compute_ssl_policy"], "attribute": "min_tls_version", "parents": [""], - "values": ["tls_1_2"], "required": "yes", "msg": "min_tls_version"}, - {"au_type": ["resource.azurerm_postgresql_server", "resource.azurerm_mariadb_server", "resource.azurerm_mysql_server"], - "attribute": "ssl_enforcement_enabled", "parents": [""], "values": ["true"], "required": "yes", + {"au_type": ["google_compute_ssl_policy"], "attribute": "min_tls_version", "parents": [], + "values": ["tls_1_2"], "required": "yes"}, + {"au_type": ["azurerm_postgresql_server", "azurerm_mariadb_server", "azurerm_mysql_server"], + "attribute": "ssl_enforcement_enabled", "parents": [], "values": ["true"], "required": "yes", "msg": "ssl_enforcement_enabled"}, - {"au_type": ["resource.azurerm_mysql_server", "resource.azurerm_postgresql_server"], - "attribute": "ssl_minimal_tls_version_enforced", "parents": [""], "values": ["tls1_2"], "required": "no"}, - {"au_type": ["resource.azurerm_mssql_server"], "attribute": "minimum_tls_version", - "parents": [""], "values": ["1.2"], "required": "no"}, - {"au_type": ["resource.google_sql_database_instance"], "attribute": "ip_configuration", "parents": ["settings"], - "values": [""], "required": "yes", "msg": "settings.ip_configuration.require_ssl"}, - {"au_type": ["resource.google_sql_database_instance"], "attribute": "require_ssl", "parents": ["ip_configuration"], - "values": ["true"], "required": "yes", "msg": "settings.ip_configuration.require_ssl"}, - {"au_type": ["resource.azurerm_app_service"], "attribute": "client_cert_enabled", "parents": [""], - "values": ["true"], "required": "yes", "msg": "client_cert_enabled"}, - {"au_type": ["resource.azurerm_storage_account"], "attribute": "min_tls_version", "parents": [""], + {"au_type": ["azurerm_mysql_server", "azurerm_postgresql_server"], + "attribute": "ssl_minimal_tls_version_enforced", "parents": [], "values": ["tls1_2"], "required": "no"}, + {"au_type": ["azurerm_mssql_server"], "attribute": "minimum_tls_version", + "parents": [], "values": ["1.2"], "required": "no"}, + {"au_type": ["google_sql_database_instance"], "attribute": "require_ssl", "parents": ["settings", "ip_configuration"], + "values": ["true"], "required": "yes"}, + {"au_type": ["azurerm_app_service"], "attribute": "client_cert_enabled", "parents": [], + "values": ["true"], "required": "yes"}, + {"au_type": ["azurerm_storage_account"], "attribute": "min_tls_version", "parents": [], "values": ["tls1_2"], "required": "no"}] -ensure_dnssec = [{"au_type": ["resource.google_dns_managed_zone"], "attribute": "state", +ensure_dnssec = [{"au_type": ["google_dns_managed_zone"], "attribute": "state", "parents": ["dnssec_config"], "values": ["on"], "required": "yes", "msg": "dnssec_config.state"}] -use_public_ip = [{"au_type": ["resource.aws_launch_configuration", "resource.aws_instance"], - "attribute": "associate_public_ip_address", "parents": [""], "values": ["false"], "required": "no"}, - {"au_type": ["resource.aws_subnet"], "attribute": "map_public_ip_on_launch", "parents": [""], +use_public_ip = [{"au_type": ["aws_launch_configuration", "aws_instance"], + "attribute": "associate_public_ip_address", "parents": [], "values": ["false"], "required": "no"}, + {"au_type": ["aws_subnet"], "attribute": "map_public_ip_on_launch", "parents": [], "values": ["false"], "required": "no"}, - {"au_type": ["resource.google_compute_instance"], "attribute": "access_config", "parents": ["network_interface"], - "values": [""], "required": "must_not_exist"}, - {"au_type": ["resource.opc_compute_ip_address_reservation"], "attribute": "ip_address_pool", "parents": [""], + {"au_type": ["google_compute_instance"], "attribute": "access_config", "parents": ["network_interface"], + "values": [], "required": "must_not_exist"}, + {"au_type": ["opc_compute_ip_address_reservation"], "attribute": "ip_address_pool", "parents": [], "values": ["cloud-ippool"], "required": "no"}] -insecure_access_control = [{"au_type": ["resource.aws_eks_cluster"], "attribute": "endpoint_public_access", - "parents": ["vpc_config"], "values": ["false"], "required": "yes", "msg": "vpc_config.endpoint_public_access"}, - {"au_type": ["resource.aws_eks_cluster"], "attribute": "public_access_cidrs[0]", "parents": ["vpc_config"], - "values": [""], "required": "yes", "msg": "vpc_config.public_access_cidrs"}, - {"au_type": ["resource.aws_lambda_permission"], "attribute": "source_arn", "parents": [""], - "values": [""], "required": "yes", "msg": "source_arn"}, - {"au_type": ["resource.aws_db_instance", "resource.aws_rds_cluster_instance"], "attribute": "publicly_accessible", "parents": [""], +insecure_access_control = [{"au_type": ["aws_eks_cluster"], "attribute": "endpoint_public_access", + "parents": ["vpc_config"], "values": ["false"], "required": "yes"}, + {"au_type": ["aws_eks_cluster"], "attribute": "public_access_cidrs", "parents": ["vpc_config"], + "values": ["non_empty_list"], "required": "yes"}, + {"au_type": ["aws_lambda_permission"], "attribute": "source_arn", "parents": [], + "values": [], "required": "yes"}, + {"au_type": ["aws_db_instance", "aws_rds_cluster_instance"], "attribute": "publicly_accessible", "parents": [], "values": ["false"], "required": "no"}, - {"au_type": ["resource.aws_s3_bucket_public_access_block"], "attribute": "block_public_acls", "parents": [""], - "values": ["true"], "required": "yes", "msg": "block_public_acls"}, - {"au_type": ["resource.aws_s3_bucket_public_access_block"], "attribute": "block_public_policy", "parents": [""], - "values": ["true"], "required": "yes", "msg": "block_public_policy"}, - {"au_type": ["resource.aws_s3_bucket_public_access_block"], "attribute": "restrict_public_buckets", "parents": [""], - "values": ["true"], "required": "yes", "msg": "restrict_public_buckets"}, - {"au_type": ["resource.aws_s3_bucket_public_access_block"], "attribute": "ignore_public_acls", "parents": [""], - "values": ["true"], "required": "yes", "msg": "ignore_public_acls"}, - {"au_type": ["resource.aws_s3_bucket"], "attribute": "acl", "parents": [""], + {"au_type": ["aws_s3_bucket_public_access_block"], "attribute": "block_public_acls", "parents": [], + "values": ["true"], "required": "yes"}, + {"au_type": ["aws_s3_bucket_public_access_block"], "attribute": "block_public_policy", "parents": [], + "values": ["true"], "required": "yes"}, + {"au_type": ["aws_s3_bucket_public_access_block"], "attribute": "restrict_public_buckets", "parents": [], + "values": ["true"], "required": "yes"}, + {"au_type": ["aws_s3_bucket_public_access_block"], "attribute": "ignore_public_acls", "parents": [], + "values": ["true"], "required": "yes"}, + {"au_type": ["aws_s3_bucket"], "attribute": "acl", "parents": [], "values": ["private", "aws-exec-read", "log-delivery-write"], "required": "no"}, - {"au_type": ["resource.azurerm_kubernetes_cluster"], "attribute": "authorized_ip_ranges[0]", - "parents": ["api_server_access_profile"], "values": [""], "required": "yes", + {"au_type": ["azurerm_kubernetes_cluster"], "attribute": "authorized_ip_ranges", + "parents": ["api_server_access_profile"], "values": ["non_empty_list"], "required": "yes", "msg": "api_server_access_profile.authorized_ip_ranges"}, - {"au_type": ["resource.azurerm_postgresql_server", "resource.azurerm_mariadb_server", "resource.azurerm_mssql_server", - "resource.azurerm_mysql_server"], "attribute": "public_network_access_enabled", "parents": [""], - "values": ["false"], "required": "yes", "msg": "public_network_access_enabled"}, - {"au_type": ["resource.azurerm_data_factory"], "attribute": "public_network_enabled", "parents": [""], - "values": ["false"], "required": "yes", "msg": "public_network_enabled"}, - {"au_type": ["resource.digitalocean_spaces_bucket"], "attribute": "acl", "parents": [""], - "values": ["private"], "required": "yes", "msg": "acl"}, - {"au_type": ["resource.digitalocean_spaces_bucket_object"], "attribute": "acl", "parents": [""], + {"au_type": ["azurerm_postgresql_server", "azurerm_mariadb_server", "azurerm_mssql_server", + "azurerm_mysql_server"], "attribute": "public_network_access_enabled", "parents": [], + "values": ["false"], "required": "yes"}, + {"au_type": ["azurerm_data_factory"], "attribute": "public_network_enabled", "parents": [], + "values": ["false"], "required": "yes"}, + {"au_type": ["digitalocean_spaces_bucket"], "attribute": "acl", "parents": [], + "values": ["private"], "required": "yes"}, + {"au_type": ["digitalocean_spaces_bucket_object"], "attribute": "acl", "parents": [], "values": ["private"], "required": "no"}, - {"au_type": ["resource.google_bigquery_dataset"], "attribute": "special_group", "parents": ["access"], + {"au_type": ["google_bigquery_dataset"], "attribute": "special_group", "parents": ["access"], "values": ["projectowners", "projectreaders", "projectwriters"], "required": "no"}, - {"au_type": ["resource.google_container_cluster"], "attribute": "enable_private_nodes", + {"au_type": ["google_container_cluster"], "attribute": "enable_private_nodes", "parents": ["private_cluster_config"], "values": ["true"], "required": "yes", "msg": "private_cluster_config.enable_private_nodes"}, - {"au_type": ["resource.aws_mq_broker"], "attribute": "publicly_accessible", "parents": [""], "values": ["false"], + {"au_type": ["aws_mq_broker"], "attribute": "publicly_accessible", "parents": [], "values": ["false"], "required": "no"}, - {"au_type": ["resource.aws_athena_workgroup"], "attribute": "enforce_workgroup_configuration", + {"au_type": ["aws_athena_workgroup"], "attribute": "enforce_workgroup_configuration", "parents": ["configuration"], "values": ["true"], "required": "no"}, - {"au_type": ["resource.azurerm_kubernetes_cluster"], "attribute": "role_based_access_control_enabled", - "parents": [""], "values": ["true"], "required": "no"}, - {"au_type": ["resource.google_container_cluster"], "attribute": "enable_legacy_abac", - "parents": [""], "values": ["false"], "required": "no"}, - {"au_type": ["resource.google_storage_bucket"], "attribute": "uniform_bucket_level_access", - "parents": [""], "values": ["true"], "required": "yes", "msg": "uniform_bucket_level_access"}, - {"au_type": ["resource.google_compute_instance"], "attribute": "email", - "parents": ["service_account"], "values": [""], "required": "yes", "msg": "service_account.email"}, - {"au_type": ["resource.azurerm_storage_container"], "attribute": "container_access_type", "parents": [""], + {"au_type": ["azurerm_kubernetes_cluster"], "attribute": "role_based_access_control_enabled", + "parents": [], "values": ["true"], "required": "no"}, + {"au_type": ["google_container_cluster"], "attribute": "enable_legacy_abac", + "parents": [], "values": ["false"], "required": "no"}, + {"au_type": ["google_storage_bucket"], "attribute": "uniform_bucket_level_access", + "parents": [], "values": ["true"], "required": "yes"}, + {"au_type": ["google_compute_instance"], "attribute": "email", + "parents": ["service_account"], "values": [], "required": "yes"}, + {"au_type": ["azurerm_storage_container"], "attribute": "container_access_type", "parents": [], "values": ["private"], "required": "no"}] -authentication = [{"au_type": ["resource.azurerm_app_service", "resource.azurerm_function_app"], "attribute": "enabled", - "parents": ["auth_settings"], "values": ["true"], "required": "yes", "msg": "auth_settings.enabled"}, - {"au_type": ["resource.azurerm_linux_virtual_machine", "resource.azurerm_linux_virtual_machine_scale_set"], - "attribute": "disable_password_authentication", "parents": [""], "values": ["true"], "required": "no"}, - {"au_type": ["resource.azurerm_virtual_machine"], "attribute": "disable_password_authentication", +authentication = [{"au_type": ["azurerm_app_service", "azurerm_function_app"], "attribute": "enabled", + "parents": ["auth_settings"], "values": ["true"], "required": "yes"}, + {"au_type": ["azurerm_linux_virtual_machine", "azurerm_linux_virtual_machine_scale_set"], + "attribute": "disable_password_authentication", "parents": [], "values": ["true"], "required": "no"}, + {"au_type": ["azurerm_virtual_machine"], "attribute": "disable_password_authentication", "parents": ["os_profile_linux_config"], "values": ["true"], "required": "yes", "msg": "os_profile_linux_config.disable_password_authentication"}, - {"au_type": ["resource.google_container_cluster"], "attribute": "issue_client_certificate", + {"au_type": ["google_container_cluster"], "attribute": "issue_client_certificate", "parents": ["client_certificate_config"], "values": ["false"], "required": "no"}] -missing_encryption = [{"au_type": ["resource.aws_api_gateway_method_settings"], "attribute": "cache_data_encrypted", - "parents": ["settings"], "values": ["true"], "required": "yes", "msg": "settings.cache_data_encrypted"}, - {"au_type": ["resource.aws_athena_database"], "attribute": "encryption_option", "parents": ["encryption_configuration"], - "values": ["sse_kms", "sse_s3", "cse_kms"], "required": "yes", "msg": "encryption_configuration.encryption_option"}, - {"au_type": ["resource.aws_athena_workgroup"], "attribute": "result_configuration", "parents": ["configuration"], - "values": [""], "required": "yes", +missing_encryption = [{"au_type": ["aws_api_gateway_method_settings"], "attribute": "cache_data_encrypted", + "parents": ["settings"], "values": ["true"], "required": "yes"}, + {"au_type": ["aws_athena_database"], "attribute": "encryption_option", "parents": ["encryption_configuration"], + "values": ["sse_kms", "sse_s3", "cse_kms"], "required": "yes"}, + {"au_type": ["aws_athena_workgroup"], "attribute": "result_configuration", "parents": ["configuration"], + "values": [], "required": "yes", "msg": "configuration.result_configuration.encryption_configuration.encryption_option"}, - {"au_type": ["resource.aws_athena_workgroup"], "attribute": "encryption_configuration", - "parents": ["result_configuration"], "values": [""], "required": "yes", + {"au_type": ["aws_athena_workgroup"], "attribute": "encryption_configuration", + "parents": ["result_configuration"], "values": [], "required": "yes", "msg": "configuration.result_configuration.encryption_configuration.encryption_option"}, - {"au_type": ["resource.aws_athena_workgroup"], "attribute": "encryption_option", "parents": ["encryption_configuration"], + {"au_type": ["aws_athena_workgroup"], "attribute": "encryption_option", "parents": ["encryption_configuration"], "values": ["sse_kms", "sse_s3", "cse_kms"], "required": "yes", "msg": "configuration.result_configuration.encryption_configuration.encryption_option"}, - {"au_type": ["resource.aws_codebuild_project"], "attribute": "encryption_disabled", + {"au_type": ["aws_codebuild_project"], "attribute": "encryption_disabled", "parents": ["artifacts", "secondary_artifacts"], "values": ["false"], "required": "no"}, - {"au_type": ["resource.aws_docdb_cluster", "resource.aws_rds_cluster", "resource.aws_db_instance", - "resource.aws_db_cluster_snapshot", "resource.aws_rds_cluster_instance", "resource.aws_rds_global_cluster", - "resource.aws_neptune_cluster"], "attribute": "storage_encrypted", "parents": [""], "values": ["true"], - "required": "yes", "msg": "storage_encrypted"}, - {"au_type": ["resource.aws_dax_cluster", "resource.aws_dynamodb_table"], "attribute": "enabled", - "parents": ["server_side_encryption"], "values": ["true"], "required": "yes", "msg": "server_side_encryption.enabled"}, - {"au_type": ["resource.aws_ebs_volume", "resource.aws_efs_file_system"], "attribute": "encrypted", "parents": [""], - "values": ["true"], "required": "yes", "msg": "encrypted"}, - {"au_type": ["resource.aws_instance", "resource.aws_launch_configuration"], "attribute": "encrypted", - "parents": ["root_block_device"], "values": ["true"], "required": "yes", "msg": "root_block_device.encrypted"}, - {"au_type": ["resource.aws_instance", "resource.aws_launch_configuration"], "attribute": "encrypted", + {"au_type": ["aws_docdb_cluster", "aws_rds_cluster", "aws_db_instance", + "aws_db_cluster_snapshot", "aws_rds_cluster_instance", "aws_rds_global_cluster", + "aws_neptune_cluster"], "attribute": "storage_encrypted", "parents": [], "values": ["true"], + "required": "yes"}, + {"au_type": ["aws_dax_cluster", "aws_dynamodb_table"], "attribute": "enabled", + "parents": ["server_side_encryption"], "values": ["true"], "required": "yes"}, + {"au_type": ["aws_ebs_volume", "aws_efs_file_system"], "attribute": "encrypted", "parents": [], + "values": ["true"], "required": "yes"}, + {"au_type": ["aws_instance", "aws_launch_configuration"], "attribute": "encrypted", + "parents": ["root_block_device"], "values": ["true"], "required": "yes"}, + {"au_type": ["aws_instance", "aws_launch_configuration"], "attribute": "encrypted", "parents": ["ebs_block_device"], "values": ["true"], "required": "no"}, - {"au_type": ["resource.aws_ami"], "attribute": "encrypted", + {"au_type": ["aws_ami"], "attribute": "encrypted", "parents": ["ebs_block_device", "root_block_device"], "values": ["true"], "required": "no"}, - {"au_type": ["resource.aws_ecs_task_definition"], "attribute": "transit_encryption", + {"au_type": ["aws_ecs_task_definition"], "attribute": "transit_encryption", "parents": ["efs_volume_configuration"], "values": ["enabled"], "required": "no"}, - {"au_type": ["resource.aws_eks_cluster"], "attribute": "provider", "parents": ["encryption_config"], - "values": [""], "required": "yes", "msg": "encryption_config.provider.key_arn"}, - {"au_type": ["resource.aws_eks_cluster"], "attribute": "key_arn", "parents": ["provider"], "values": ["any_not_empty"], - "required": "yes", "msg": "encryption_config.provider.key_arn"}, - {"au_type": ["resource.aws_elasticsearch_domain"], "attribute": "enabled", "parents": ["encrypt_at_rest"], - "values": ["true"], "required": "yes", "msg": "encrypt_at_rest.enabled"}, - {"au_type": ["resource.aws_elasticache_replication_group"], "attribute": "at_rest_encryption_enabled", - "parents": [""], "values": ["true"], "required": "yes", "msg": "at_rest_encryption_enabled"}, - {"au_type": ["resource.aws_kinesis_stream"], "attribute": "encryption_type", - "parents": [""], "values": ["kms"], "required": "yes", "msg": "encryption_type"}, - {"au_type": ["resource.aws_msk_cluster"], "attribute": "encryption_in_transit", "parents": ["encryption_info"], - "values": [""], "required": "yes", "msg": "encryption_info.encryption_in_transit"}, - {"au_type": ["resource.aws_msk_cluster"], "attribute": "client_broker", + {"au_type": ["aws_eks_cluster"], "attribute": "key_arn", "parents": ["encryption_config", "provider"], "values": ["any_not_empty"], + "required": "yes"}, + {"au_type": ["aws_elasticsearch_domain"], "attribute": "enabled", "parents": ["encrypt_at_rest"], + "values": ["true"], "required": "yes"}, + {"au_type": ["aws_elasticache_replication_group"], "attribute": "at_rest_encryption_enabled", + "parents": [], "values": ["true"], "required": "yes"}, + {"au_type": ["aws_kinesis_stream"], "attribute": "encryption_type", + "parents": [], "values": ["kms"], "required": "yes"}, + {"au_type": ["aws_msk_cluster"], "attribute": "encryption_in_transit", "parents": ["encryption_info"], + "values": [], "required": "yes"}, + {"au_type": ["aws_msk_cluster"], "attribute": "client_broker", "parents": ["encryption_in_transit"], "values": ["tls"], "required": "no"}, - {"au_type": ["resource.aws_msk_cluster"], "attribute": "in_cluster", + {"au_type": ["aws_msk_cluster"], "attribute": "in_cluster", "parents": ["encryption_in_transit"], "values": ["true"], "required": "no"}, - {"au_type": ["resource.aws_redshift_cluster"], "attribute": "encrypted", - "parents": [""], "values": ["true"], "required": "yes", "msg": "encrypted"}, - {"au_type": ["resource.aws_workspaces_workspace"], "attribute": "root_volume_encryption_enabled", "parents": [""], - "values": ["true"], "required": "yes", "msg": "root_volume_encryption_enabled"}, - {"au_type": ["resource.aws_workspaces_workspace"], "attribute": "user_volume_encryption_enabled", "parents": [""], - "values": ["true"], "required": "yes", "msg": "user_volume_encryption_enabled"}, - {"au_type": ["resource.aws_elasticsearch_domain"], "attribute": "enabled", "parents": ["node_to_node_encryption"], - "values": ["true"], "required": "yes", "msg": "node_to_node_encryption.enabled"}, - {"au_type": ["resource.aws_elasticache_replication_group"], "attribute": "transit_encryption_enabled", - "parents": [""], "values": ["true"], "required": "yes", "msg": "transit_encryption_enabled"}, - {"au_type": ["resource.aws_ecr_repository"], "attribute": "encryption_type", "parents": ["encryption_configuration"], - "values": ["kms"], "required": "yes", "msg": "encryption_configuration.encryption_type"}, - {"au_type": ["resource.aws_s3_bucket_server_side_encryption_configuration"], - "attribute": "apply_server_side_encryption_by_default", "parents": ["rule"], - "values": [""], "required": "yes", "msg": "rule.apply_server_side_encryption_by_default.sse_algorithm"}, - {"au_type": ["resource.aws_s3_bucket_server_side_encryption_configuration"], - "attribute": "sse_algorithm", "parents": ["apply_server_side_encryption_by_default"], - "values": ["aes256", "aws:kms"], "required": "yes", "msg": "rule.apply_server_side_encryption_by_default.sse_algorithm"}] + {"au_type": ["aws_redshift_cluster"], "attribute": "encrypted", + "parents": [], "values": ["true"], "required": "yes"}, + {"au_type": ["aws_workspaces_workspace"], "attribute": "root_volume_encryption_enabled", "parents": [], + "values": ["true"], "required": "yes"}, + {"au_type": ["aws_workspaces_workspace"], "attribute": "user_volume_encryption_enabled", "parents": [], + "values": ["true"], "required": "yes"}, + {"au_type": ["aws_elasticsearch_domain"], "attribute": "enabled", "parents": ["node_to_node_encryption"], + "values": ["true"], "required": "yes"}, + {"au_type": ["aws_elasticache_replication_group"], "attribute": "transit_encryption_enabled", + "parents": [], "values": ["true"], "required": "yes"}, + {"au_type": ["aws_ecr_repository"], "attribute": "encryption_type", "parents": ["encryption_configuration"], + "values": ["kms"], "required": "yes"}, + {"au_type": ["aws_s3_bucket_server_side_encryption_configuration"], + "attribute": "sse_algorithm", + "parents": [["rule", "apply_server_side_encryption_by_default"], ["apply_server_side_encryption_by_default"]], + "values": ["aes256", "aws:kms"], "required": "yes"}] -firewall = [{"au_type": ["resource.aws_cloudfront_distribution"], "attribute": "web_acl_id", "parents": [""], - "values": ["any_not_empty"], "required": "yes", "msg": "web_acl_id"}, - {"au_type": ["resource.aws_alb", "resource.aws_lb", "resource.aws_elb"], "attribute": "internal", "parents": [""], "values": ["true"], - "required": "yes", "msg": "internal"}, - {"au_type": ["resource.aws_alb", "resource.aws_lb"], "attribute": "drop_invalid_header_fields", "parents": [""], - "values": ["true"], "required": "yes", "msg": "drop_invalid_header_fields"}, - {"au_type": ["resource.google_compute_instance"], "attribute": "can_ip_forward", "parents": [""], "values": ["false"], +firewall = [{"au_type": ["aws_cloudfront_distribution"], "attribute": "web_acl_id", "parents": [], + "values": ["any_not_empty"], "required": "yes"}, + {"au_type": ["aws_alb", "aws_lb", "aws_elb"], "attribute": "internal", "parents": [], "values": ["true"], + "required": "yes"}, + {"au_type": ["aws_alb", "aws_lb"], "attribute": "drop_invalid_header_fields", "parents": [], + "values": ["true"], "required": "yes"}, + {"au_type": ["google_compute_instance"], "attribute": "can_ip_forward", "parents": [], "values": ["false"], "required": "no"}, - {"au_type": ["resource.google_compute_firewall"], "attribute": "destination_ranges[0]", "parents": [""], "values": [""], - "required": "yes", "msg": "destination_ranges"}, - {"au_type": ["resource.google_compute_firewall"], "attribute": "source_ranges[0]", "parents": [""], "values": [""], - "required": "yes", "msg": "source_ranges"}, - {"au_type": ["resource.openstack_fw_rule_v1"], "attribute": "destination_ip_address", "parents": [""], "values": [""], - "required": "yes", "msg": "destination_ip_address"}, - {"au_type": ["resource.openstack_fw_rule_v1"], "attribute": "source_ip_address", "parents": [""], "values": [""], - "required": "yes", "msg": "source_ip_address"}, - {"au_type": ["resource.azurerm_key_vault"], "attribute": "default_action", "parents": ["network_acls"], - "values": ["deny"], "required": "yes", "msg": "network_acls.default_action"}, - {"au_type": ["resource.azurerm_key_vault"], "attribute": "bypass", "parents": ["network_acls"], - "values": ["azureservices"], "required": "yes", "msg": "network_acls.bypass"}, - {"au_type": ["resource.google_container_cluster"], "attribute": "cidr_blocks", "parents": ["master_authorized_networks_config"], - "values": [""], "required": "yes", "msg": "master_authorized_networks_config.cidr_blocks.cidr_block"}, - {"au_type": ["resource.google_container_cluster"], "attribute": "cidr_block", "parents": ["cidr_blocks"], - "values": [""], "required": "yes", "msg": "master_authorized_networks_config.cidr_blocks.cidr_block"}] + {"au_type": ["google_compute_firewall"], "attribute": "destination_ranges[0]", "parents": [], "values": [], + "required": "yes"}, + {"au_type": ["google_compute_firewall"], "attribute": "source_ranges[0]", "parents": [], "values": [], + "required": "yes"}, + {"au_type": ["openstack_fw_rule_v1"], "attribute": "destination_ip_address", "parents": [], "values": [], + "required": "yes"}, + {"au_type": ["openstack_fw_rule_v1"], "attribute": "source_ip_address", "parents": [], "values": [], + "required": "yes"}, + {"au_type": ["azurerm_key_vault"], "attribute": "default_action", "parents": ["network_acls"], + "values": ["deny"], "required": "yes"}, + {"au_type": ["azurerm_key_vault"], "attribute": "bypass", "parents": ["network_acls"], + "values": ["azureservices"], "required": "yes"}, + {"au_type": ["google_container_cluster"], "attribute": "cidr_blocks", "parents": ["master_authorized_networks_config"], + "values": [], "required": "yes"}, + {"au_type": ["google_container_cluster"], "attribute": "cidr_block", "parents": ["cidr_blocks"], + "values": [], "required": "yes"}] -missing_threats_detection_alerts = [{"au_type": ["resource.azurerm_mssql_server_security_alert_policy"], - "attribute": "disabled_alerts[0]", "parents": [""], "values": [""], "required": "must_not_exist"}, - {"au_type": ["resource.github_repository"], "attribute": "vulnerability_alerts", - "parents": [""], "values": ["true"], "required": "yes", "msg": "vulnerability_alerts"}, - {"au_type": ["resource.azurerm_mssql_server_security_alert_policy"], "attribute": "email_addresses[0]", - "parents": [""], "values": [""], "required": "yes", "msg": "email_addresses"}, - {"au_type": ["resource.azurerm_mssql_server_security_alert_policy"], "attribute": "email_account_admins", - "parents": [""], "values": ["true"], "required": "yes", "msg": "email_account_admins"}, - {"au_type": ["resource.azurerm_security_center_contact"], "attribute": "alert_notifications", - "parents": [""], "values": ["true"], "required": "yes", "msg": "alert_notifications"}, - {"au_type": ["resource.azurerm_security_center_contact"], "attribute": "alerts_to_admins", - "parents": [""], "values": ["true"], "required": "yes", "msg": "alerts_to_admins"}, - {"au_type": ["resource.azurerm_security_center_contact"], "attribute": "phone", - "parents": [""], "values": ["any_not_empty"], "required": "yes", "msg": "phone"}, - {"au_type": ["resource.aws_ecr_repository"], "attribute": "scan_on_push", "parents": ["image_scanning_configuration"], - "values": ["true"], "required": "yes", "msg": "image_scanning_configuration.scan_on_push"}] +missing_threats_detection_alerts = [{"au_type": ["azurerm_mssql_server_security_alert_policy"], + "attribute": "disabled_alerts[0]", "parents": [], "values": [], "required": "must_not_exist"}, + {"au_type": ["github_repository"], "attribute": "vulnerability_alerts", + "parents": [], "values": ["true"], "required": "yes"}, + {"au_type": ["azurerm_mssql_server_security_alert_policy"], "attribute": "email_addresses[0]", + "parents": [], "values": [], "required": "yes"}, + {"au_type": ["azurerm_mssql_server_security_alert_policy"], "attribute": "email_account_admins", + "parents": [], "values": ["true"], "required": "yes"}, + {"au_type": ["azurerm_security_center_contact"], "attribute": "alert_notifications", + "parents": [], "values": ["true"], "required": "yes"}, + {"au_type": ["azurerm_security_center_contact"], "attribute": "alerts_to_admins", + "parents": [], "values": ["true"], "required": "yes"}, + {"au_type": ["azurerm_security_center_contact"], "attribute": "phone", + "parents": [], "values": ["any_not_empty"], "required": "yes"}, + {"au_type": ["aws_ecr_repository"], "attribute": "scan_on_push", "parents": ["image_scanning_configuration"], + "values": ["true"], "required": "yes"}] -password_key_policy = [{"au_type": ["resource.aws_iam_account_password_policy"], "attribute": "password_reuse_prevention", - "parents": [""], "values": ["5"], "required": "yes", "msg": "password_reuse_prevention", "logic": "gte"}, - {"au_type": ["resource.aws_iam_account_password_policy"], "attribute": "require_lowercase_characters", - "parents": [""], "values": ["true"], "required": "yes", "msg": "require_lowercase_characters", "logic": "equal"}, - {"au_type": ["resource.aws_iam_account_password_policy"], "attribute": "require_numbers", - "parents": [""], "values": ["true"], "required": "yes", "msg": "require_numbers", "logic": "equal"}, - {"au_type": ["resource.aws_iam_account_password_policy"], "attribute": "require_symbols", - "parents": [""], "values": ["true"], "required": "yes", "msg": "require_symbols", "logic": "equal"}, - {"au_type": ["resource.aws_iam_account_password_policy"], "attribute": "require_uppercase_characters", - "parents": [""], "values": ["true"], "required": "yes", "msg": "require_uppercase_characters", "logic": "equal"}, - {"au_type": ["resource.aws_iam_account_password_policy"], "attribute": "max_password_age", - "parents": [""], "values": ["90"], "required": "yes", "msg": "max_password_age", "logic": "lte"}, - {"au_type": ["resource.aws_iam_account_password_policy"], "attribute": "minimum_password_length", - "parents": [""], "values": ["14"], "required": "yes", "msg": "minimum_password_length", "logic": "gte"}, - {"au_type": ["resource.azurerm_key_vault_secret"], "attribute": "expiration_date", - "parents": [""], "values": ["any_not_empty"], "required": "yes", "msg": "expiration_date", "logic": "equal"}, - {"au_type": ["resource.azurerm_key_vault"], "attribute": "purge_protection_enabled", - "parents": [""], "values": ["true"], "required": "yes", "msg": "purge_protection_enabled", "logic": "equal"}, - {"au_type": ["resource.azurerm_key_vault_key"], "attribute": "expiration_date", "parents": [""], +password_key_policy = [{"au_type": ["aws_iam_account_password_policy"], "attribute": "password_reuse_prevention", + "parents": [], "values": ["5"], "required": "yes", "msg": "password_reuse_prevention", "logic": "gte"}, + {"au_type": ["aws_iam_account_password_policy"], "attribute": "require_lowercase_characters", + "parents": [], "values": ["true"], "required": "yes", "msg": "require_lowercase_characters", "logic": "equal"}, + {"au_type": ["aws_iam_account_password_policy"], "attribute": "require_numbers", + "parents": [], "values": ["true"], "required": "yes", "msg": "require_numbers", "logic": "equal"}, + {"au_type": ["aws_iam_account_password_policy"], "attribute": "require_symbols", + "parents": [], "values": ["true"], "required": "yes", "msg": "require_symbols", "logic": "equal"}, + {"au_type": ["aws_iam_account_password_policy"], "attribute": "require_uppercase_characters", + "parents": [], "values": ["true"], "required": "yes", "msg": "require_uppercase_characters", "logic": "equal"}, + {"au_type": ["aws_iam_account_password_policy"], "attribute": "max_password_age", + "parents": [], "values": ["90"], "required": "yes", "msg": "max_password_age", "logic": "lte"}, + {"au_type": ["aws_iam_account_password_policy"], "attribute": "minimum_password_length", + "parents": [], "values": ["14"], "required": "yes", "msg": "minimum_password_length", "logic": "gte"}, + {"au_type": ["azurerm_key_vault_secret"], "attribute": "expiration_date", + "parents": [], "values": ["any_not_empty"], "required": "yes", "msg": "expiration_date", "logic": "equal"}, + {"au_type": ["azurerm_key_vault"], "attribute": "purge_protection_enabled", + "parents": [], "values": ["true"], "required": "yes", "msg": "purge_protection_enabled", "logic": "equal"}, + {"au_type": ["azurerm_key_vault_key"], "attribute": "expiration_date", "parents": [], "values": ["any_not_empty"], "required": "yes", "msg": "expiration_date", "logic": "equal"}] -key_management = [{"au_type": ["resource.aws_cloudwatch_log_group", "resource.aws_docdb_cluster", "resource.aws_ebs_volume", - "resource.aws_secretsmanager_secret", "resource.aws_kinesis_stream", "resource.aws_cloudtrail", "resource.aws_rds_cluster", - "resource.aws_db_instance", "resource.aws_redshift_cluster", "resource.aws_db_instance_automated_backups_replication", - "resource.aws_rds_cluster_activity_stream"], - "attribute": "kms_key_id", "parents": [""], "values": ["any_not_empty"], "required": "yes", "msg": "kms_key_id"}, - {"au_type": ["resource.aws_dynamodb_table"], "attribute": "kms_key_arn", "parents": ["server_side_encryption"], - "values": ["any_not_empty"], "required": "yes", "msg": "server_side_encryption.kms_key_arn"}, - {"au_type": ["resource.aws_neptune_cluster"], "attribute": "kms_key_arn", "parents": [""], - "values": ["any_not_empty"], "required": "yes", "msg": "kms_key_arn"}, - {"au_type": ["resource.aws_ecr_repository"], "attribute": "kms_key", "parents": ["encryption_configuration"], - "values": ["any_not_empty"], "required": "yes", "msg": "encryption_configuration.kms_key"}, - {"au_type": ["resource.aws_kms_key"], "attribute": "enable_key_rotation", "parents": [""], - "values": ["true"], "required": "yes", "msg": "enable_key_rotation"}, - {"au_type": ["resource.aws_rds_cluster_instance", "resource.aws_db_instance"], "attribute": "performance_insights_kms_key_id", - "parents": [""], "values": ["any_not_empty"], "required": "yes", "msg": "performance_insights_kms_key_id"}, - {"au_type": ["resource.google_kms_crypto_key"], "attribute": "rotation_period", "parents": [""], - "values": [""], "required": "yes", "msg": "rotation_period"}, - {"au_type": ["resource.google_compute_disk"], "attribute": "kms_key_self_link", "parents": ["disk_encryption_key"], - "values": ["any_not_empty"], "required": "yes", "msg": "disk_encryption_key.kms_key_self_link"}, - {"au_type": ["resource.google_compute_instance"], "attribute": "kms_key_self_link", "parents": ["boot_disk"], - "values": ["any_not_empty"], "required": "yes", "msg": "boot_disk.kms_key_self_link"}, - {"au_type": ["resource.google_compute_instance"], "attribute": "block-project-ssh-keys", "parents": ["metadata"], - "values": ["true"], "required": "yes", "msg": "metadata.block-project-ssh-keys"}, - {"au_type": ["resource.aws_s3_bucket_server_side_encryption_configuration"], - "attribute": "apply_server_side_encryption_by_default", "parents": ["rule"], - "values": [""], "required": "yes", "msg": "rule.apply_server_side_encryption_by_default.kms_master_key_id"}, - {"au_type": ["resource.aws_s3_bucket_server_side_encryption_configuration"], - "attribute": "kms_master_key_id", "parents": ["apply_server_side_encryption_by_default"], - "values": ["any_not_empty"], "required": "yes", "msg": "rule.apply_server_side_encryption_by_default.kms_master_key_id"}, - {"au_type": ["resource.aws_sns_topic", "resource.aws_sqs_queue"], "attribute": "kms_master_key_id", "parents": [""], - "values": ["any_not_empty"], "required": "yes", "msg": "kms_master_key_id"}, - {"au_type": ["resource.digitalocean_droplet"], "attribute": "ssh_keys[0]", "parents": [""], - "values": [""], "required": "yes", "msg": "ssh_keys"}, - {"au_type": ["resource.google_storage_bucket"], "attribute": "default_kms_key_name", "parents": ["encryption"], - "values": ["any_not_empty"], "required": "yes", "msg": "encryption.default_kms_key_name"}] +key_management = [{"au_type": ["aws_cloudwatch_log_group", "aws_docdb_cluster", "aws_ebs_volume", + "aws_secretsmanager_secret", "aws_kinesis_stream", "aws_cloudtrail", "aws_rds_cluster", + "aws_db_instance", "aws_redshift_cluster", "aws_db_instance_automated_backups_replication", + "aws_rds_cluster_activity_stream"], + "attribute": "kms_key_id", "parents": [], "values": ["any_not_empty"], "required": "yes"}, + {"au_type": ["aws_dynamodb_table"], "attribute": "kms_key_arn", "parents": ["server_side_encryption"], + "values": ["any_not_empty"], "required": "yes"}, + {"au_type": ["aws_neptune_cluster"], "attribute": "kms_key_arn", "parents": [], + "values": ["any_not_empty"], "required": "yes"}, + {"au_type": ["aws_ecr_repository"], "attribute": "kms_key", "parents": ["encryption_configuration"], + "values": ["any_not_empty"], "required": "yes"}, + {"au_type": ["aws_kms_key"], "attribute": "enable_key_rotation", "parents": [], + "values": ["true"], "required": "yes"}, + {"au_type": ["aws_rds_cluster_instance", "aws_db_instance"], "attribute": "performance_insights_kms_key_id", + "parents": [], "values": ["any_not_empty"], "required": "yes"}, + {"au_type": ["google_kms_crypto_key"], "attribute": "rotation_period", "parents": [], + "values": [], "required": "yes"}, + {"au_type": ["google_compute_disk"], "attribute": "kms_key_self_link", "parents": ["disk_encryption_key"], + "values": ["any_not_empty"], "required": "yes"}, + {"au_type": ["google_compute_instance"], "attribute": "kms_key_self_link", "parents": ["boot_disk"], + "values": ["any_not_empty"], "required": "yes"}, + {"au_type": ["google_compute_instance"], "attribute": "block-project-ssh-keys", "parents": ["metadata"], + "values": ["true"], "required": "yes"}, + {"au_type": ["aws_s3_bucket_server_side_encryption_configuration"], + "attribute": "kms_master_key_id", + "parents": [["rule", "apply_server_side_encryption_by_default"], + ["apply_server_side_encryption_by_default"]], + "values": ["any_not_empty"], "required": "yes"}, + {"au_type": ["aws_sns_topic", "aws_sqs_queue"], "attribute": "kms_master_key_id", "parents": [], + "values": ["any_not_empty"], "required": "yes"}, + {"au_type": ["digitalocean_droplet"], "attribute": "ssh_keys[0]", "parents": [], + "values": [], "required": "yes"}, + {"au_type": ["google_storage_bucket"], "attribute": "default_kms_key_name", "parents": ["encryption"], + "values": ["any_not_empty"], "required": "yes"}] -network_security_rules = [{"au_type": ["resource.azurerm_storage_account_network_rules"], "attribute": "default_action", - "parents": [""], "values": ["deny"], "required": "yes", "msg": "default_action"}, - {"au_type": ["resource.azurerm_storage_account"], "attribute": "default_action", - "parents": ["network_rules"], "values": ["deny"], "required": "yes", "msg": "network_rules.default_action"}, - {"au_type": ["resource.aws_network_acl_rule"], "attribute": "protocol", - "parents": [""], "values": ["tcp"], "required": "no"}, - {"au_type": ["resource.azurerm_kubernetes_cluster"], "attribute": "network_policy", "parents": ["network_profile"], - "values": ["calico", "azure"], "required": "yes", "msg": "network_profile.network_policy"}, - {"au_type": ["resource.azurerm_synapse_workspace"], "attribute": "managed_virtual_network_enabled", "parents": [""], - "values": ["true"], "required": "yes", "msg": "managed_virtual_network_enabled"}, - {"au_type": ["resource.google_compute_instance"], "attribute": "serial-port-enable", "parents": ["metadata"], +network_security_rules = [{"au_type": ["azurerm_storage_account_network_rules"], "attribute": "default_action", + "parents": [], "values": ["deny"], "required": "yes"}, + {"au_type": ["azurerm_storage_account"], "attribute": "default_action", + "parents": ["network_rules"], "values": ["deny"], "required": "yes"}, + {"au_type": ["aws_network_acl_rule"], "attribute": "protocol", + "parents": [], "values": ["tcp"], "required": "no"}, + {"au_type": ["azurerm_kubernetes_cluster"], "attribute": "network_policy", "parents": ["network_profile"], + "values": ["calico", "azure"], "required": "yes"}, + {"au_type": ["azurerm_synapse_workspace"], "attribute": "managed_virtual_network_enabled", "parents": [], + "values": ["true"], "required": "yes"}, + {"au_type": ["google_compute_instance"], "attribute": "serial-port-enable", "parents": ["metadata"], "values": ["false"], "required": "no"}, - {"au_type": ["resource.google_container_cluster"], "attribute": "ip_allocation_policy", "parents": [""], - "values": [""], "required": "yes", "msg": "ip_allocation_policy"}, - {"au_type": ["resource.google_container_cluster"], "attribute": "enabled", "parents": ["network_policy"], - "values": ["true"], "required": "yes", "msg": "network_policy.enabled"}, - {"au_type": ["resource.google_project"], "attribute": "auto_create_network", "parents": [""], - "values": ["false"], "required": "yes", "msg": "auto_create_network"}] + {"au_type": ["google_container_cluster"], "attribute": "ip_allocation_policy", "parents": [], + "values": [], "required": "yes"}, + {"au_type": ["google_container_cluster"], "attribute": "enabled", "parents": ["network_policy"], + "values": ["true"], "required": "yes"}, + {"au_type": ["google_project"], "attribute": "auto_create_network", "parents": [], + "values": ["false"], "required": "yes"}] -google_iam_member_resources = ["resource.google_project_iam_member", "resource.google_project_iam_binding", - "resource.google_organization_iam_member", "resource.google_organization_iam_binding", - "resource.google_folder_iam_member", "resource.google_folder_iam_binding"] +google_iam_member_resources = ["google_project_iam_member", "google_project_iam_binding", + "google_organization_iam_member", "google_organization_iam_binding", + "google_folder_iam_member", "google_folder_iam_binding"] -permission_iam_policies = [{"au_type": ["resource.google_project_iam_member", "resource.google_project_iam_binding", - "resource.google_organization_iam_member", "resource.google_organization_iam_binding", - "resource.google_folder_iam_member", "resource.google_folder_iam_binding"], - "attribute": "role", "parents": [""], "values": ["roles/owner", "roles/editor", "roles/iam.securityadmin", +permission_iam_policies = [{"au_type": ["google_project_iam_member", "google_project_iam_binding", + "google_organization_iam_member", "google_organization_iam_binding", + "google_folder_iam_member", "google_folder_iam_binding"], + "attribute": "role", "parents": [], "values": ["roles/owner", "roles/editor", "roles/iam.securityadmin", "roles/iam.serviceaccountadmin", "roles/iam.serviceaccountkeyadmin", "roles/iam.serviceaccountuser", "roles/iam.serviceaccounttokencreator", "roles/iam.workloadidentityuser", "roles/dataproc.editor", "roles/dataproc.admin", "roles/dataflow.developer", "roles/resourcemanager.folderadmin", @@ -372,61 +365,55 @@ permission_iam_policies = [{"au_type": ["resource.google_project_iam_member", "r "roles/cloudasset.viewer", "roles/cloudasset.owner", "roles/serverless.serviceagent", "roles/dataproc.serviceagent"], "required": "no", "logic": "diff"}] -logging = [{"au_type": ["resource.aws_cloudwatch_log_group"], "attribute": "retention_in_days", "parents": [""], +logging = [{"au_type": ["aws_cloudwatch_log_group"], "attribute": "retention_in_days", "parents": [], "values": ["1", "3", "5", "7", "14", "30", "60", "90", "120", "150", "180", "365", "400", "545", "731", "1827", "3653"], - "required": "yes", "msg": "retention_in_days"}, - {"au_type": ["resource.azurerm_storage_account"], "attribute": "logging", "parents": ["queue_properties"], - "values": [""], "required": "yes", "msg": "queue_properties.logging.delete"}, - {"au_type": ["resource.azurerm_storage_account"], "attribute": "logging", "parents": ["queue_properties"], - "values": [""], "required": "yes", "msg": "queue_properties.logging.read"}, - {"au_type": ["resource.azurerm_storage_account"], "attribute": "logging", "parents": ["queue_properties"], - "values": [""], "required": "yes", "msg": "queue_properties.logging.write"}, - {"au_type": ["resource.azurerm_storage_account"], "attribute": "delete", "parents": ["logging"], - "values": ["true"], "required": "yes", "msg": "queue_properties.logging.delete"}, - {"au_type": ["resource.azurerm_storage_account"], "attribute": "read", "parents": ["logging"], - "values": ["true"], "required": "yes", "msg": "queue_properties.logging.read"}, - {"au_type": ["resource.azurerm_storage_account"], "attribute": "write", "parents": ["logging"], - "values": ["true"], "required": "yes", "msg": "queue_properties.logging.write"}, - {"au_type": ["resource.aws_s3_bucket"], "attribute": "logging", "parents": [""], - "values": [""], "required": "yes", "msg": "logging"}, - {"au_type": ["resource.aws_apigatewayv2_stage", "resource.aws_api_gateway_stage"], "attribute": "destination_arn", + "required": "yes"}, + {"au_type": ["azurerm_storage_account"], "attribute": "logging", "parents": ["queue_properties"], + "values": [], "required": "yes"}, + {"au_type": ["azurerm_storage_account"], "attribute": "delete", "parents": ["queue_properties", "logging"], + "values": ["true"], "required": "yes"}, + {"au_type": ["azurerm_storage_account"], "attribute": "read", "parents": ["queue_properties", "logging"], + "values": ["true"], "required": "yes"}, + {"au_type": ["azurerm_storage_account"], "attribute": "write", "parents": ["queue_properties", "logging"], + "values": ["true"], "required": "yes"}, + {"au_type": ["aws_s3_bucket"], "attribute": "logging", "parents": [], + "values": [], "required": "yes"}, + {"au_type": ["aws_apigatewayv2_stage", "aws_api_gateway_stage"], "attribute": "destination_arn", "parents": ["access_log_settings"], "values": ["any_not_empty"], "required": "yes", "msg": "access_log_settings.destination_arn"}, - {"au_type": ["resource.aws_api_gateway_stage"], "attribute": "xray_tracing_enabled", "parents": [""], - "values": ["true"], "required": "yes", "msg": "xray_tracing_enabled"}, - {"au_type": ["resource.aws_cloudfront_distribution"], "attribute": "bucket", "parents": ["logging_config"], - "values": ["any_not_empty"], "required": "yes", "msg": "logging_config.bucket"}, - {"au_type": ["resource.aws_cloudtrail"], "attribute": "enable_log_file_validation", "parents": [""], - "values": ["true"], "required": "yes", "msg": "enable_log_file_validation"}, - {"au_type": ["resource.aws_cloudtrail"], "attribute": "cloud_watch_logs_group_arn", "parents": [""], - "values": [""], "required": "yes", "msg": "cloud_watch_logs_group_arn"}, - {"au_type": ["resource.aws_elasticsearch_domain"], "attribute": "log_type", "parents": ["log_publishing_options"], + {"au_type": ["aws_api_gateway_stage"], "attribute": "xray_tracing_enabled", "parents": [], + "values": ["true"], "required": "yes"}, + {"au_type": ["aws_cloudfront_distribution"], "attribute": "bucket", "parents": ["logging_config"], + "values": ["any_not_empty"], "required": "yes"}, + {"au_type": ["aws_cloudtrail"], "attribute": "enable_log_file_validation", "parents": [], + "values": ["true"], "required": "yes"}, + {"au_type": ["aws_cloudtrail"], "attribute": "cloud_watch_logs_group_arn", "parents": [], + "values": [], "required": "yes"}, + {"au_type": ["aws_elasticsearch_domain"], "attribute": "log_type", "parents": ["log_publishing_options"], "values": ["index_slow_logs", "search_slow_logs", "es_application_logs", "audit_logs"], "required": "yes", "msg": "log_publishing_options.log_type"}, - {"au_type": ["resource.aws_elasticsearch_domain"], "attribute": "enabled", "parents": ["log_publishing_options"], + {"au_type": ["aws_elasticsearch_domain"], "attribute": "enabled", "parents": ["log_publishing_options"], "values": ["true"], "required": "no"}, - {"au_type": ["resource.aws_lambda_function"], "attribute": "mode", "parents": ["tracing_config"], - "values": ["active"], "required": "yes", "msg": "tracing_config.mode"}, - {"au_type": ["resource.aws_mq_broker"], "attribute": "audit", "parents": ["logs"], - "values": ["true"], "required": "yes", "msg": "logs.audit"}, - {"au_type": ["resource.aws_mq_broker"], "attribute": "general", "parents": ["logs"], - "values": ["true"], "required": "yes", "msg": "logs.general"}, - {"au_type": ["resource.azurerm_kubernetes_cluster"], "attribute": "oms_agent", "parents": ["addon_profile"], - "values": [""], "required": "yes", "msg": "addon_profile.oms_agent.log_analytics_workspace_id"}, - {"au_type": ["resource.azurerm_kubernetes_cluster"], "attribute": "log_analytics_workspace_id", "parents": ["oms_agent"], - "values": ["any_not_empty"], "required": "yes", "msg": "addon_profile.oms_agent.log_analytics_workspace_id"}, - {"au_type": ["resource.azurerm_network_watcher_flow_log"], "attribute": "days", "parents": ["retention_policy"], - "values": [""], "required": "yes", "msg": "retention_policy.days"}, - {"au_type": ["resource.azurerm_monitor_log_profile"], "attribute": "days", "parents": ["retention_policy"], - "values": [""], "required": "yes", "msg": "retention_policy.days"}, - {"au_type": ["resource.google_compute_subnetwork"], "attribute": "log_config", "parents": [""], - "values": [""], "required": "yes", "msg": "log_config"}, - {"au_type": ["resource.google_container_cluster"], "attribute": "logging_service", "parents": [""], + {"au_type": ["aws_lambda_function"], "attribute": "mode", "parents": ["tracing_config"], + "values": ["active"], "required": "yes"}, + {"au_type": ["aws_mq_broker"], "attribute": "audit", "parents": ["logs"], + "values": ["true"], "required": "yes"}, + {"au_type": ["aws_mq_broker"], "attribute": "general", "parents": ["logs"], + "values": ["true"], "required": "yes"}, + {"au_type": ["azurerm_kubernetes_cluster"], "attribute": "log_analytics_workspace_id", "parents": [["addon_profile", "oms_agent"], ["oms_agent"]], + "values": ["any_not_empty"], "required": "yes"}, + {"au_type": ["azurerm_network_watcher_flow_log"], "attribute": "days", "parents": ["retention_policy"], + "values": [], "required": "yes"}, + {"au_type": ["azurerm_monitor_log_profile"], "attribute": "days", "parents": ["retention_policy"], + "values": [], "required": "yes"}, + {"au_type": ["google_compute_subnetwork"], "attribute": "log_config", "parents": [], + "values": [], "required": "yes"}, + {"au_type": ["google_container_cluster"], "attribute": "logging_service", "parents": [], "values": ["logging.googleapis.com/kubernetes"], "required": "no"}, - {"au_type": ["resource.google_container_cluster"], "attribute": "monitoring_service", "parents": [""], + {"au_type": ["google_container_cluster"], "attribute": "monitoring_service", "parents": [], "values": ["monitoring.googleapis.com/kubernetes"], "required": "no"}, - {"au_type": ["resource.aws_rds_cluster_instance", "resource.aws_db_instance"], "attribute": "performance_insights_enabled", - "parents": [""], "values": ["true"], "required": "yes", "msg": "performance_insights_enabled"}] + {"au_type": ["aws_rds_cluster_instance", "aws_db_instance"], "attribute": "performance_insights_enabled", + "parents": [], "values": ["true"], "required": "yes"}] google_sql_database_log_flags = [{"flag_name": "log_checkpoints", "value": "on", "required": "yes"}, {"flag_name": "log_connections", "value": "on", "required": "yes"}, @@ -441,17 +428,17 @@ possible_attached_resources_aws_route53 = ["aws_instance", "aws_eip", "aws_elb", "aws_globalaccelerator_accelerator", "aws_cloudfront_distribution", "aws_db_instance", "aws_apigatewayv2_domain_name", "aws_lightsail_instance"] -versioning = [{"au_type": ["resource.aws_s3_bucket", "resource.digitalocean_spaces_bucket"], "attribute": "enabled", +versioning = [{"au_type": ["aws_s3_bucket", "digitalocean_spaces_bucket"], "attribute": "enabled", "parents": ["versioning"], "values": ["true"], "required": "yes", "msg": "versioning.enabled"}] -naming = [{"au_type": ["resource.aws_security_group", "resource.aws_security_group_rule", - "resource.aws_elasticache_security_group", "resource.openstack_networking_secgroup_v2", - "resource.openstack_networking_secgroup_rule_v2"],"attribute": "description", "parents": [""], - "values": ["any_not_empty"], "required": "yes", "msg": "description"}, - {"au_type": ["resource.aws_security_group"], "attribute": "description", +naming = [{"au_type": ["aws_security_group", "aws_security_group_rule", + "aws_elasticache_security_group", "openstack_networking_secgroup_v2", + "openstack_networking_secgroup_rule_v2"],"attribute": "description", "parents": [], + "values": ["any_not_empty"], "required": "yes"}, + {"au_type": ["aws_security_group"], "attribute": "description", "parents": ["ingress", "egress"], "values": ["any_not_empty"], "required": "no"}] -replication = [{"au_type": ["resource.aws_s3_bucket_replication_configuration"], "attribute": "status", +replication = [{"au_type": ["aws_s3_bucket_replication_configuration"], "attribute": "status", "parents": ["rule"], "values": ["enabled"], "required": "yes", "msg": "rule.status"}] file_commands = ["file", "chmod", "mkdir"] diff --git a/glitch/exceptions.py b/glitch/exceptions.py index f7da3b3c..81a623e1 100644 --- a/glitch/exceptions.py +++ b/glitch/exceptions.py @@ -12,8 +12,6 @@ "CHEF_COULD_NOT_PARSE": "Chef - Could not parse file: {}", "GHA_COULD_NOT_PARSE": "Github Actions Workflow - Could not parse file: {}", "PUPPET_COULD_NOT_PARSE": "Puppet - Could not parse file: {}", - "DOCKER_NOT_IMPLEMENTED": "Docker - Could not parse: {}", - "DOCKER_UNKNOW_ERROR": "Docker - Unknown Error: {}", "SHELL_COULD_NOT_PARSE": "Shell Command - Could not parse: {}", "TERRAFORM_COULD_NOT_PARSE": "Terraform - Could not parse file: {}", } diff --git a/glitch/files/official_docker_images b/glitch/files/official_docker_images deleted file mode 100644 index d692e385..00000000 --- a/glitch/files/official_docker_images +++ /dev/null @@ -1,175 +0,0 @@ -alpine -busybox -nginx -ubuntu -python -redis -postgres -node -httpd -mongo -mysql -memcached -traefik -mariadb -docker -rabbitmq -hello-world -openjdk -golang -registry -wordpress -centos -debian -influxdb -consul -php -nextcloud -sonarqube -haproxy -ruby -amazonlinux -elasticsearch -tomcat -eclipse-mosquitto -maven -telegraf -vault -caddy -adminer -bash -ghost -kong -perl -neo4j -zookeeper -buildpack-deps -mongo-express -gradle -logstash -cassandra -couchdb -nats -chronograf -drupal -jenkins -kibana -java -solr -percona -teamspeak -sentry -matomo -fedora -composer -nats-streaming -adoptopenjdk -flink -couchbase -swarm -joomla -groovy -rethinkdb -rocket.chat -redmine -owncloud -rust -kapacitor -erlang -phpmyadmin -jruby -elixir -amazoncorretto -mediawiki -mono -pypy -jetty -clojure -arangodb -odoo -eclipse-temurin -xwiki -oraclelinux -znc -haxe -ros -hylang -websphere-liberty -django -sapmachine -gcc -archlinux -swift -tomee -piwik -yourls -rockylinux -iojs -crate -aerospike -photon -orientdb -julia -varnish -ibmjava -open-liberty -bonita -monica -neurodebian -opensuse -fluentd -rails -ubuntu-debootstrap -storm -r-base -irssi -haskell -backdrop -clearlinux -plone -notary -cirros -lightstreamer -geonetwork -nuxeo -postfixadmin -gazebo -php-zendserver -convertigo -friendica -hello-seattle -celery -spiped -swipl -fsharp -eggdrop -thrift -rapidoid -almalinux -docker-dev -rakudo-star -express-gateway -Kaazing Gateway -ibm-semeru-runtimes -ubuntu-upstart -silverpeas -mageia -hola-mundo -known -glassfish -dart -crux -euleros -jobber -sourcemage -clefos -alt -sl -hipache -hitch -scratch -satosa -emqx -api-firewall -cheers -dcl2020 \ No newline at end of file diff --git a/glitch/helpers.py b/glitch/helpers.py index 4dffe13e..b4e2d97d 100644 --- a/glitch/helpers.py +++ b/glitch/helpers.py @@ -1,6 +1,7 @@ -from typing import List, Tuple, Iterable +from typing import List, Tuple, Iterable, Dict from glitch.tech import Tech from glitch.analysis.rules import Error +import configparser def get_smell_types() -> Tuple[str, ...]: @@ -113,3 +114,38 @@ def compute_LPS_array(pat: str, M: int, lps: List[int]) -> None: else: lps[i] = 0 i += 1 + + +def ini_to_json_dict(config_path: str) -> Dict[str, Dict[str, List[str]]]: + config = configparser.ConfigParser() + config.read(config_path) + + result: Dict[str, Dict[str, List[str]]] = {} + + for section in config.sections(): + section_data = {} + + for key, value in config.items(section): + parsed = value.strip() + + # Case: empty list + if parsed == "": + section_data[key] = [] + continue + + # Case: list inside brackets: ["a", "b"] + if parsed.startswith("[") and parsed.endswith("]"): + inner = parsed[1:-1].strip() + if inner == "": + section_data[key] = [] + else: + section_data[key] = [ + item.strip().strip('"').strip("'") for item in inner.split(",") + ] + else: + # Case: plain value, keep as string + section_data[key] = parsed + + result[section] = section_data + + return result diff --git a/glitch/parsers/ansible.py b/glitch/parsers/ansible.py index a83fe96c..dd33f614 100644 --- a/glitch/parsers/ansible.py +++ b/glitch/parsers/ansible.py @@ -5,10 +5,8 @@ from ruamel.yaml.main import YAML from ruamel.yaml.nodes import ( Node, - ScalarNode, MappingNode, SequenceNode, - CollectionNode, ) from ruamel.yaml.tokens import Token from glitch.exceptions import EXCEPTIONS, throw_exception @@ -16,129 +14,133 @@ class AnsibleParser(YamlParser): - @staticmethod - def __parse_vars( - unit_block: UnitBlock, - cur_name: str, - node: Node, + __TASK_PARAMS = [ + "ansible.builtin.include", + "any_errors_fatal", + "ansible.legacy.include", + "args", + "async", + "become", + "become_exe", + "become_flags", + "become_method", + "become_user", + "changed_when", + "collections", + "connection", + "debugger", + "delegate_facts", + "deletage_to", + "diff", + "environment", + "failed_when", + "ignore_errors", + "ignore_unreachable", + "include", + "listen", + "local_action", + "loop", + "loop_control", + "module_defaults", + "notify", + "poll", + "port", + "register", + "remote_user", + "retries", + "run_once", + "tags", + "throttle", + "timeout", + "until", + "vars", + "when", + "with_dict" "with_fileglob", + "with_filetree", + "with_first_found", + "with_indexed_items", + "with_ini", + "with_inventory_hostnames", + "with_items", + "with_lines", + "with_random_choice", + "with_sequence", + "with_subelements", + "with_together", + ] + + def __init__(self) -> None: + super().__init__() + + def __create_variable( + self, + token: Token | Node, + val_node: Node, + value: Expr, + name: str, code: List[str], - child: bool = False, - ) -> List[Variable]: - def create_variable( - token: Token | Node, name: str, value: str | None, child: bool = False - ) -> Variable: - has_variable = ( - (("{{" in value) and ("}}" in value)) if value != None else False - ) - if value in ["null", "~"]: - value = "" - v = Variable(name, value, has_variable) - v.line = token.start_mark.line + 1 - v.column = token.start_mark.column + 1 - if value == None: - v.code = AnsibleParser._get_code(token, token, code) - else: - v.code = AnsibleParser._get_code(token, value, code) - v.code = "".join(code[token.start_mark.line : token.end_mark.line + 1]) - - variables.append(v) - if not child: - unit_block.add_variable(v) - return v + ) -> Variable: + if isinstance(value, Null): + v_code = self._get_code(token, token, code) + else: + v_code = self._get_code(token, val_node, code) + v_code = "".join(code[token.start_mark.line : token.end_mark.line + 1]) + info = ElementInfo( + token.start_mark.line + 1, + token.start_mark.column + 1, + val_node.end_mark.line + 1, + val_node.end_mark.column + 1, + v_code, + ) + return Variable(name, value, info) + def __parse_vars(self, node: MappingNode, code: List[str]) -> List[Variable]: variables: List[Variable] = [] - if isinstance(node, MappingNode): - if cur_name == "": - for key, v in node.value: - if hasattr(key, "value") and isinstance(key.value, str): - AnsibleParser.__parse_vars( - unit_block, key.value, v, code, child - ) - elif isinstance(key.value, MappingNode): - AnsibleParser.__parse_vars( - unit_block, cur_name, key.value[0][0], code, child # type: ignore - ) - else: - var = create_variable(node, cur_name, None, child) - for key, v in node.value: - if hasattr(key, "value") and isinstance(key.value, str): - var.keyvalues += AnsibleParser.__parse_vars( - unit_block, key.value, v, code, True - ) - elif isinstance(key.value, MappingNode): - var.keyvalues += AnsibleParser.__parse_vars( - unit_block, cur_name, key.value[0][0], code, True # type: ignore - ) - elif isinstance(node, ScalarNode): - create_variable(node, cur_name, str(node.value), child) - elif isinstance(node, SequenceNode): - value: List[Any] = [] - for i, val in enumerate(node.value): - if isinstance(val, CollectionNode): - variables += AnsibleParser.__parse_vars( - unit_block, f"{cur_name}[{i}]", val, code, child - ) - else: - value.append(val.value) - if len(value) > 0 and isinstance(node.value[-1], (Node, Token)): - create_variable(node.value[-1], cur_name, str(value), child) + for key, val in node.value: + name = key.value + value = self.get_value(val, code) + v = self.__create_variable(key, val, value, name, code) + variables.append(v) return variables - @staticmethod - def __parse_attribute( - cur_name: str, token: Token | Node, val: Any, code: List[str] - ) -> List[Attribute]: - def create_attribute(token: Token | Node, name: str, value: Any) -> Attribute: - has_variable = ( - (("{{" in value) and ("}}" in value)) if value != None else False - ) - if value in ["null", "~"]: - value = "" - a = Attribute(name, value, has_variable) - a.line = token.start_mark.line + 1 - a.column = token.start_mark.column + 1 - if val == None: - a.code = AnsibleParser._get_code(token, token, code) - else: - a.code = AnsibleParser._get_code(token, val, code) - attributes.append(a) - - return a - - attributes: List[Attribute] = [] - if isinstance(val, MappingNode): - attribute = create_attribute(token, cur_name, None) - aux_attributes: List[KeyValue] = [] - for aux, aux_val in val.value: - aux_attributes += AnsibleParser.__parse_attribute( - f"{aux.value}", aux, aux_val, code - ) - attribute.keyvalues = aux_attributes - elif isinstance(val, ScalarNode): - create_attribute(token, cur_name, str(val.value)) - elif isinstance(val, SequenceNode): - value: List[Any] = [] - for i, v in enumerate(val.value): - if not isinstance(v, ScalarNode): - attributes += AnsibleParser.__parse_attribute( - f"{cur_name}[{i}]", v, v, code - ) - else: - value.append(v.value) + def __create_attribute( + self, + token: Token | Node, + name: str, + value: Expr, + val_node: Node, + code: List[str], + ) -> Attribute: + if isinstance(value, Null): + a_code = self._get_code(token, token, code) + else: + a_code = self._get_code(token, val_node, code) + + info = ElementInfo( + token.start_mark.line + 1, + token.start_mark.column + 1, + val_node.end_mark.line + 1, + val_node.end_mark.column + 1, + a_code, + ) - if len(value) > 0: - create_attribute(token, cur_name, str(value)) + return Attribute(name, value, info) - return attributes + def __parse_attribute( + self, name: str, token: Token | Node, val: Any, code: List[str] + ) -> Attribute: + v = self.get_value(val, code) + return self.__create_attribute(token, name, v, val, code) - @staticmethod - def __parse_tasks(unit_block: UnitBlock, tasks: Node, code: List[str]) -> None: + def __parse_tasks( + self, unit_block: UnitBlock, tasks: Node, code: List[str] + ) -> None: for task in tasks.value: atomic_units: List[AtomicUnit] = [] attributes: List[Attribute] = [] - type, name, line = "", "", 0 + type, name, line = "", Null(), 0 is_block = False for key, val in task.value: @@ -153,31 +155,30 @@ def __parse_tasks(unit_block: UnitBlock, tasks: Node, code: List[str]) -> None: if key.value in ["block", "always", "rescue"]: is_block = True size = len(unit_block.atomic_units) - AnsibleParser.__parse_tasks(unit_block, val, code) + self.__parse_tasks(unit_block, val, code) created = len(unit_block.atomic_units) - size atomic_units = unit_block.atomic_units[-created:] elif key.value == "name": - name = val.value + name = self.get_value(val, code) elif key.value != "name": - if type == "": + if type == "" and key.value not in AnsibleParser.__TASK_PARAMS: type = key.value line = task.start_mark.line + 1 - names: List[str] = [n.strip() for n in name.split(",")] - for name in names: - if name == "": - continue + if isinstance(name, Array): + for n in name.value: + atomic_units.append(AtomicUnit(n, type)) + else: atomic_units.append(AtomicUnit(name, type)) - if isinstance(val, MappingNode): + if isinstance(val, MappingNode) and key.value == type: for atr, atr_val in val.value: - if atr.value != "name": - attributes += AnsibleParser.__parse_attribute( - atr.value, atr, atr_val, code - ) + attributes.append( + self.__parse_attribute(atr.value, atr, atr_val, code) + ) else: - attributes += AnsibleParser.__parse_attribute( - key.value, key, val, code + attributes.append( + self.__parse_attribute(key.value, key, val, code) ) if is_block: @@ -198,7 +199,7 @@ def __parse_tasks(unit_block: UnitBlock, tasks: Node, code: List[str]) -> None: # Tasks without name if len(atomic_units) == 0 and type != "": - au = AtomicUnit("", type) + au = AtomicUnit(Null(), type) au.attributes = attributes au.line = line if len(au.attributes) > 0: @@ -231,17 +232,19 @@ def __parse_playbook( if key.value == "name" and play.name == "": play.name = value.value elif key.value == "vars": - AnsibleParser.__parse_vars(play, "", value, code) + vars = self.__parse_vars(value, code) + for v in vars: + play.add_variable(v) elif key.value in ["tasks", "pre_tasks", "post_tasks", "handlers"]: - AnsibleParser.__parse_tasks(play, value, code) + self.__parse_tasks(play, value, code) else: - play.attributes += AnsibleParser.__parse_attribute( - key.value, key, value, code + play.attributes.append( + self.__parse_attribute(key.value, key, value, code) ) unit_block.add_unit_block(play) - for comment in AnsibleParser._get_comments(parsed_file, file): + for comment in self._get_comments(parsed_file, file): c = Comment(comment[1]) c.line = comment[0] c.code = code[c.line - 1] @@ -267,8 +270,8 @@ def __parse_tasks_file( if parsed_file is None: return unit_block - AnsibleParser.__parse_tasks(unit_block, parsed_file, code) - for comment in AnsibleParser._get_comments(parsed_file, file): + self.__parse_tasks(unit_block, parsed_file, code) + for comment in self._get_comments(parsed_file, file): c = Comment(comment[1]) c.line = comment[0] c.code = code[c.line - 1] @@ -294,8 +297,11 @@ def __parse_vars_file( if parsed_file is None: return unit_block - AnsibleParser.__parse_vars(unit_block, "", parsed_file, code) - for comment in AnsibleParser._get_comments(parsed_file, file): + assert isinstance(parsed_file, MappingNode) + vars = self.__parse_vars(parsed_file, code) + for v in vars: + unit_block.add_variable(v) + for comment in self._get_comments(parsed_file, file): c = Comment(comment[1]) c.line = comment[0] c.code = code[c.line - 1] diff --git a/glitch/parsers/chef.py b/glitch/parsers/chef.py index f305c737..9d72384c 100644 --- a/glitch/parsers/chef.py +++ b/glitch/parsers/chef.py @@ -5,7 +5,7 @@ import glitch.parsers.parser as p from string import Template -from pkg_resources import resource_filename +from importlib.resources import files from typing import Any, List, Tuple, Callable from glitch.repr.inter import * from glitch.parsers.ripper_parser import parser_yacc @@ -15,7 +15,36 @@ ChefValue = Tuple[str, str] | str | int | bool | List["ChefValue"] +def set_loc_from_info(code_element: CodeElement, info: ElementInfo) -> None: + code_element.line = info.line + code_element.column = info.column + code_element.end_line = info.end_line + code_element.end_column = info.end_column + code_element.code = info.code + + class ChefParser(p.Parser): + """ + https://kddnewton.com/ripper-docs/events + """ + + __ADD = [ + "args_add_star", + "args_add", + "qwords_add", + "qsymbols_add", + "words_add", + "symbols_add", + "word_add", + "stmts_add", + "mrhs_add", + "mrhs_add_star", + "mrhs_new_from_args", + "string_add", + "xstring_add", + "regexp_add", + ] + class Node: def __init__(self, id: str, args: List[Any]) -> None: self.id: str = id @@ -24,12 +53,30 @@ def __init__(self, id: str, args: List[Any]) -> None: def __repr__(self) -> str: return str(self.id) + def as_dict(self) -> Dict[str, Any]: + def parse_args(args: List[Any]) -> List[Any]: + res: List[Any] = [] + for arg in args: + if isinstance(arg, ChefParser.Node): + res.append(arg.as_dict()) + elif isinstance(arg, list): + res.append(parse_args(arg)) # type: ignore + else: + res.append(arg) + return res + + return {"id": self.id, "args": parse_args(self.args)} + def __iter__(self): return iter(self.args) def __reversed__(self): return reversed(self.args) + def __init__(self) -> None: + super().__init__() + self._inside_atomic_unit = False + @staticmethod def _check_id(ast: Any, ids: List[Any]) -> bool: return isinstance(ast, ChefParser.Node) and ast.id in ids @@ -66,17 +113,41 @@ def is_bounds(l: Any) -> bool: start_line, start_column = sys.maxsize, sys.maxsize end_line, end_column = 0, 0 - bounded_structures = [ - "brace_block", - "arg_paren", - "string_literal", - "string_embexpr", - "aref", - "array", - "args_add_block", - ] + bounded_structures: Dict[str, Tuple[List[str], List[str]]] = { + "brace_block": (["{"], ["}"]), + "arg_paren": (["("], [")"]), + "paren": (["("], [")"]), + "string_embexpr": (["{"], ["}"]), + "aref": ([""], ["]"]), + "aref_field": ([""], ["]"]), + # In words, qwords, qsymbols and symbols, an array can be bounded + # by ) or } + "array": (["[", "{", "("], ["]", "}", ")"]), + "hash": (["{"], ["}"]), + "xstring_literal": (["`"], ["`"]), + "defined": ([""], [")"]), + } + + if ChefParser._check_node(ast, ["@tstring_content"], 2): + lines = bytes(ast.args[0], "utf-8").decode("unicode_escape") + lines = lines.split("\n") + lines = [l + "\n" for l in lines[:-1]] + [lines[-1]] + start_line, start_column = ast.args[1][0], ast.args[1][1] + + on_new_line = lines[-1] == "" + if on_new_line: + lines.pop() + end_line = start_line + len(lines) + else: + end_line = start_line + len(lines) - 1 - if ( + if on_new_line: + end_column = -1 + elif len(lines) == 1: + end_column = start_column + len(lines[-1]) - 1 + else: + end_column = len(lines[-1]) - 1 + elif ( isinstance(ast, ChefParser.Node) and len(ast.args) > 0 and is_bounds(ast.args[-1]) @@ -86,7 +157,8 @@ def is_bounds(l: Any) -> bool: # of the node (variable name, string...) end_line, end_column = ( ast.args[-1][0], - ast.args[-1][1] + len(ast.args[-2]) - 1, + # The parser returns \n as two different characters + ast.args[-1][1] + len(ast.args[-2].replace("\\n", "\n")) - 1, ) # With identifiers we need to consider the : behind them @@ -95,58 +167,90 @@ def is_bounds(l: Any) -> bool: and source[start_line - 1][start_column - 1] == ":" ): start_column -= 1 - elif ChefParser._check_id(ast, ["@tstring_content"]): - end_line += ast.args[0].count("\\n") - elif isinstance(ast, (list, ChefParser.Node)): for arg in ast: # type: ignore bound = ChefParser._get_content_bounds(arg, source) if bound[0] < start_line: start_line = bound[0] - if bound[1] < start_column: + start_column = bound[1] + if bound[0] == start_line and bound[1] < start_column: start_column = bound[1] if bound[2] > end_line: end_line = bound[2] - if bound[3] > end_column: + end_column = bound[3] + if bound[2] == end_line and bound[3] > end_column: end_column = bound[3] # We have to consider extra characters which correspond # to enclosing characters of these structures if start_line != sys.maxsize and ChefParser._check_id( - ast, bounded_structures + ast, list(bounded_structures.keys()) ): - r_brackets = ["}", ")", "]", '"', "'"] + l_bracket = bounded_structures[ast.id][0] # type: ignore + r_bracket = bounded_structures[ast.id][1] # type: ignore + + next_lines = source[end_line - 1 :] + next_lines[0] = next_lines[0][end_column + 1 :] + next_lines = "".join(next_lines) # Add spaces/brackets in front of last token - for i, c in enumerate(source[end_line - 1][end_column + 1 :]): - if c in r_brackets: - end_column += i + 1 - break - elif not c.isspace(): + for c in next_lines: + if c in r_bracket: + end_column += 1 break + elif c == "\n": + end_line += 1 + end_column = -1 + else: + end_column += 1 - l_brackets = ["{", "(", "[", '"', "'"] + previous_lines = source[:start_line] + previous_lines[-1] = previous_lines[-1][:start_column] + previous_lines = "".join(previous_lines)[::-1] # Add spaces/brackets behind first token - for i, c in enumerate(source[start_line - 1][:start_column][::-1]): - if c in l_brackets: - start_column -= i + 1 + for c in previous_lines: + if l_bracket == [""]: break - elif not c.isspace(): + elif c in l_bracket: + start_column -= 1 break + elif c == "\n": + start_line -= 1 + start_column = len(source[start_line - 1]) - 1 + else: + start_column -= 1 - if ( - ChefParser._check_id(ast, ["string_embexpr"]) - and source[start_line - 1][start_column] == "{" - and source[start_line - 1][start_column - 1] == "#" - ): + if ChefParser._check_id(ast, ["string_embexpr"]): start_column -= 1 - # The original AST does not have the start column # of these refs. We need to consider the :: - elif ChefParser._check_id(ast, ["top_const_ref"]): + elif ChefParser._check_id(ast, ["top_const_ref", "top_const_field"]): start_column -= 2 + # String literal is a special case since it can be unbounded in heredocs + elif ( + ChefParser._check_id(ast, ["string_literal"]) + and start_line != sys.maxsize + and source[start_line - 1][start_column - 1] in ["'", '"'] + and source[end_line - 1][end_column + 1] in ["'", '"'] + ): + start_column -= 1 + end_column += 1 return (start_line, start_column, end_line, end_column) + @staticmethod + def _get_info(ast: Any, source: List[str]) -> ElementInfo: + bounds = ChefParser._get_content_bounds(ast, source) + return ElementInfo( + bounds[0], + bounds[1] + 1, + bounds[2], + # We are considering a start at 0 and the index to be inclusive + # so we need to add 2 to the end column since + # we want it to start at 1 and be exclusive + bounds[3] + 2, + ChefParser._get_content(ast, source), + ) + @staticmethod def _get_content(ast: Any, source: List[str]) -> str: empty_structures = {"string_literal": "", "hash": "{}", "array": "[]"} @@ -165,7 +269,7 @@ def _get_content(ast: Any, source: List[str]) -> str: if bounds[0] == sys.maxsize: return res - for l in range(bounds[0] - 1, bounds[2]): + for l in range(bounds[0] - 1, min(bounds[2], len(source))): if bounds[0] - 1 == bounds[2] - 1: res += source[l][bounds[1] : bounds[3] + 1] elif l == bounds[2] - 1: @@ -178,21 +282,411 @@ def _get_content(ast: Any, source: List[str]) -> str: if (ast.id == "method_add_block") and (ast.args[1].id == "do_block"): res += "\nend" - res = res.strip() - if res.startswith(('"', "'")) and res.endswith(('"', "'")): + if res.startswith(('"', "'", "`")) and res.endswith(('"', "'", "`")): res = res[1:-1] return remove_unmatched_brackets(res) - @staticmethod - def _get_source(ast: Any, source: List[str]) -> str: - bounds = ChefParser._get_content_bounds(ast, source) - return "".join(source[bounds[0] - 1 : bounds[2]]) + def __get_add_args(self, ast: Node, source: List[str]) -> List[Expr]: + current = ast + curr_args: List[Expr] = [] + + while ChefParser._check_id(current, ChefParser.__ADD): + if len(current.args) == 2: + value = self._get_value(current.args[1], source) + if value is not None: + curr_args.append(value) + current = current.args[0] + + curr_args = list(reversed(curr_args)) + return curr_args + + def __parse_assoc_hash(self, ast: Node, source: List[str]) -> Dict[Expr, Expr]: + hash_args: Dict[Expr, Expr] = {} + for arg in ast.args[0]: + if ChefParser._check_id(arg, ["assoc_new"]): + key = self._get_value(arg.args[0], source) + value = self._get_value(arg.args[1], source) + assert key is not None + assert value is not None + hash_args[key] = value + elif ChefParser._check_id(arg, ["assoc_splat"]): + # FIXME: not supported + key = self._get_value(arg.args[0], source) + assert key is not None + hash_args[key] = key + return hash_args + + def __parse_binary(self, ast: Node, source: List[str], info: ElementInfo) -> Expr: + left = self._get_value(ast.args[0], source) + right = self._get_value(ast.args[2], source) + assert left is not None + assert right is not None + op = ast.args[1][1] + + match op: + case "+": + return Sum(info, left, right) + case "-": + return Subtract(info, left, right) + case "*": + return Multiply(info, left, right) + case "/": + return Divide(info, left, right) + case "%": + return Modulo(info, left, right) + case "**": + return Power(info, left, right) + case "==": + return Equal(info, left, right) + case "!=": + return NotEqual(info, left, right) + case ">": + return GreaterThan(info, left, right) + case "<": + return LessThan(info, left, right) + case ">=": + return GreaterThanOrEqual(info, left, right) + case "<=": + return LessThanOrEqual(info, left, right) + case "<=>": + # TODO + return Null() + case "===": + return Equal(info, left, right) + case "&": + return BitwiseAnd(info, left, right) + case "|": + return BitwiseOr(info, left, right) + case "^": + return BitwiseXor(info, left, right) + case "<<": + return LeftShift(info, left, right) + case ">>": + return RightShift(info, left, right) + case "and" | '"&&"': + return And(info, left, right) + case "or" | '"||"': + return Or(info, left, right) + case "=~" | "!~": + # TODO: Unsupported operation + return Null() + case _: + raise ValueError(f"Unknown binary operator {op}") + + def __parse_unary(self, ast: Node, source: List[str], info: ElementInfo) -> Expr: + arg = self._get_value(ast.args[1], source) + assert arg is not None + op = ast.args[0][1] + + match op: + case "!" | "not": + return Not(info, arg) + case "-@": + return Minus(info, arg) + case _: + raise ValueError(f"Unknown unary operator {op}") + + def __parse_if_expr(self, ast: Any, source: List[str]) -> Expr | None: + predicate = self._get_value(ast.args[0], source) + assert predicate is not None + true_part = self._get_value(ast.args[1], source) + assert true_part is not None + true_statement = ConditionalStatement( + predicate, ConditionalStatement.ConditionType.IF, is_top=True + ) + true_statement.add_statement(true_part) + if len(ast.args) == 3: + false_part = self._get_value(ast.args[2], source) + assert false_part is not None + false_statement = ConditionalStatement( + Null(), ConditionalStatement.ConditionType.IF, is_default=True + ) + true_statement.else_statement = false_statement + false_statement.add_statement(false_part) + return true_statement + + def _get_value(self, ast: Any, source: List[str]) -> Expr | None: + NEW = [ + "args_new", + "qwords_new", + "qsymbols_new", + "words_new", + "symbols_new", + "word_new", + "stmts_new", + ] + + content = ChefParser._get_content(ast, source) + info = ChefParser._get_info(ast, source) + + if ChefParser._check_id( + ast, + [ + "dyna_symbol", + "@tstring_content", + "@label", + ], + ): + return String(content, info) + elif ChefParser._check_id(ast, ["string_content"]): + return String("", info) + elif ChefParser._check_node( + ast, ["string_literal", "xstring_literal"], 1 + ) or ChefParser._check_node(ast, ["regexp_literal"], 2): + value = self._get_value(ast.args[0], source) + assert value is not None + set_loc_from_info(value, info) + return value + elif ChefParser._check_id(ast, ["@int"]): + if "x" in content: + return Integer(int(content, 16), info) + if "o" in content: + return Integer(int(content, 8), info) + return Integer(int(content), info) + elif ChefParser._check_id(ast, ["@imaginary"]): + return Complex(complex(content.replace("i", "j")), info) + elif ChefParser._check_id(ast, ["@float"]): + return Float(float(content), info) + elif ChefParser._check_id(ast, ["@rational"]): + return Float(float(content[:-1]), info) + elif ChefParser._check_id(ast, ["binary"]): + return self.__parse_binary(ast, source, info) + elif ChefParser._check_node(ast, ["string_concat"], 2): + left = self._get_value(ast.args[0], source) + right = self._get_value(ast.args[1], source) + assert left is not None + assert right is not None + return Sum(info, left, right) + elif ChefParser._check_id(ast, ["unary"]): + return self.__parse_unary(ast, source, info) + elif ChefParser._check_node(ast, ["rescue_mod"], 2): + value = self._get_value(ast.args[0], source) + default = self._get_value(ast.args[1], source) + assert isinstance(value, Expr) + assert isinstance(default, Expr) + cond = NotEqual(info, value, Null()) + true = ConditionalStatement( + cond, ConditionalStatement.ConditionType.IF, is_top=True + ) + true.add_statement(value) + false = ConditionalStatement( + Null(), ConditionalStatement.ConditionType.IF, is_default=True + ) + false.add_statement(default) + true.else_statement = false + return true + elif ChefParser._check_id(ast, ["dot2", "dot3"]): + left = self._get_value(ast.args[0], source) + right = self._get_value(ast.args[1], source) + assert left is not None + assert right is not None + return FunctionCall("range", [left, right], info) + elif ChefParser._check_id(ast, ["array"]): + if len(ast.args) == 0: + return Array([], info) + value = self._get_value(ast.args[0], source) + assert value is not None + if isinstance(value, AddArgs): + return Array(value.value, info) + else: + return Array([value], info) + elif ChefParser._check_node(ast, ["ifop"], 3) or ChefParser._check_node( + ast, ["if_mod", "unless_mod"], 2 + ): + expr = self.__parse_if_expr(ast, source) + assert isinstance(expr, ConditionalStatement) + if ChefParser._check_id(ast, ["unless_mod"]): + expr.condition = Not(info, expr.condition) + return expr + elif ChefParser._check_id(ast, ["bare_assoc_hash"]): + hash_args: Dict[Expr, Expr] = self.__parse_assoc_hash(ast, source) + return Hash(hash_args, info) + elif ChefParser._check_id(ast, ["hash"]): + if ast.args == []: + return Hash({}, info) + elif len(ast.args) == 1 and ChefParser._check_node( + ast.args[0], ["assoclist_from_args"], 1 + ): + return Hash(self.__parse_assoc_hash(ast.args[0], source), info) + elif ChefParser._check_id( + ast, + [ + "vcall", + "symbol_literal", + "@ident", + "fcall", + "string_embexpr", + "var_field", + "var_ref", + "string_dvar", + "const_path_ref", + "top_const_ref", + "top_const_field", + "@backref", + "@const", + "field", + ], + ): + if ( + hasattr(ast, "args") + and len(ast.args) > 0 + and getattr(ast.args[0], "id", None) == "@kw" + and ChefParser._get_content(ast.args[0], source) == "nil" + ): + return Null(info) + + if content.startswith("#{") and content.endswith("}"): + content = content[2:-1] + info = ElementInfo( + info.line, + info.column + 2, + info.end_line, + info.end_column - 1, + content, + ) + return VariableReference(content, info) + elif ChefParser._check_id(ast, ["case"]): + c = ChefParser.CaseChecker(source, ast, self) + c.check_all() + return c.condition + elif ChefParser._check_id(ast, ["if"]): + c = ChefParser.IfChecker(source, ast, self) + c.check_all() + return c.condition + elif ChefParser._check_id(ast, ["defined", "next", "super", "zsuper"]): + if len(ast.args) != 0: + value = self._get_value(ast.args[0], source) + assert value is not None + value = [value] + else: + value = [] + return FunctionCall(ast.id, value, info) + elif ChefParser._check_id(ast, ["aref", "aref_field"]) and len(ast.args) in [ + 1, + 2, + ]: + ref = self._get_value(ast.args[0], source) + assert ref is not None + if len(ast.args) == 1: + index = Null(info) + else: + index = self._get_value(ast.args[1], source) + assert index is not None + return Access(info, ref, index) + elif ChefParser._check_node(ast, ["call"], 3): + receiver = self._get_value(ast.args[0], source) + assert receiver is not None + method = self._get_value(ast.args[2], source) + assert isinstance(method, VariableReference) + return MethodCall(receiver, method.value, [], info) + elif ChefParser._check_node(ast, ["command_call"], 4): + receiver = self._get_value(ast.args[0], source) + assert receiver is not None + method = self._get_value(ast.args[2], source) + assert isinstance(method, VariableReference) + + args = self._get_value(ast.args[3], source) + assert args is not None + if not isinstance(args, AddArgs): + args = [args] + else: + args = args.value + + return MethodCall(receiver, method.value, args, info) + elif ChefParser._check_id(ast, ["args_add_block"]): + value = self._get_value(ast.args[0], source) + assert value is not None + return value + elif ChefParser._check_node( + ast, ["word_add", "string_add", "regexp_add", "xstring_add"], 2 + ): + args = self.__get_add_args(ast, source) + if len(args) == 1: + return args[0] + sum_args = Sum(info, args[0], args[1]) + for i in range(2, len(args)): + sum_args = Sum(info, sum_args, args[i]) + return sum_args + elif ChefParser._check_node(ast, ChefParser.__ADD, 2): + args = self.__get_add_args(ast, source) + if len(args) == 1: + return args[0] + return AddArgs(args, info) + elif ChefParser._check_node(ast, ["arg_paren"], 0): + return Null(info) + elif ChefParser._check_node(ast, ["arg_paren", "arg_ambiguous"], 1): + return self._get_value(ast.args[0], source) + elif ChefParser._check_node(ast, ["paren"], 1): + value = self._get_value(ast.args[0], source) + assert value is not None + if isinstance(value, AddArgs) and len(value.value) == 1: + return value.value[0] + return value + elif ChefParser._check_id(ast, ["@period"]) or ast == []: + return None + elif ChefParser._check_node(ast, ["method_add_arg", "command"], 2): + method = self._get_value(ast.args[0], source) + args = self._get_value(ast.args[1], source) + assert args is not None + if not isinstance(args, AddArgs): + args = [args] + else: + args = args.value + + if isinstance(method, VariableReference): + return FunctionCall(method.value, args, info) + + assert isinstance(method, MethodCall) + method.args = args + + return method + elif ChefParser._check_node(ast, ["assign"], 2): + left = self._get_value(ast.args[0], source) + right = self._get_value(ast.args[1], source) + assert left is not None + assert right is not None + return Assign(info, left, right) + elif ( + ChefParser._check_id(ast, ["method_add_block"]) + and len(ast.args) == 2 + and ChefParser._check_id(ast.args[1], ["do_block", "brace_block"]) + ): + receiver = self._get_value(ast.args[0], source) + assert receiver is not None + block = self._get_value(ast.args[1], source) + assert isinstance(block, BlockExpr) + return MethodCall(receiver, "", [block], info) + elif ChefParser._check_id(ast, ["do_block", "brace_block"]): + block = BlockExpr(info) + self._transverse_ast(ast.args, block, source) + return block + elif ChefParser._check_id( + ast, + [ + "args_forward", + "until_mod", + "while_mod", + "hshptn", + "begin", + "yield", + "lambda", + "yield0", + "sclass", + ], + ): + # FIXME: Not supported + return Null(info) + elif ChefParser._check_id(ast, NEW): + return Null(info) + + raise ValueError(f"Unknown ast type {ast.id}: {content} {info} {len(ast.args)}") class Checker: - def __init__(self, source: List[str]) -> None: + def __init__(self, source: List[str], parser: "ChefParser") -> None: self.tests_ast_stack: List[Tuple[List[Callable[[Any], bool]], Any]] = [] self.source = source + self.parser = parser def check(self) -> bool: tests, ast = self.pop() @@ -216,12 +710,23 @@ def pop(self): class ResourceChecker(Checker): def __init__( - self, atomic_unit: AtomicUnit, source: List[str], ast: Any + self, + atomic_unit: AtomicUnit, + source: List[str], + ast: Any, + parser: "ChefParser", ) -> None: - super().__init__(source) + super().__init__(source, parser) self.push([self.is_block_resource, self.is_inline_resource], ast) self.atomic_unit = atomic_unit + def get_statements(self, ast: Any) -> None: + assert self.parser is not None + if ChefParser._check_id(ast, ["stmts_add"]): + self.parser._inside_atomic_unit = True + self.parser._transverse_ast(ast, self.atomic_unit, self.source) + self.parser._inside_atomic_unit = False + def is_block_resource(self, ast: Any) -> bool: if ( ChefParser._check_node(ast, ["method_add_block"], 2) @@ -230,10 +735,8 @@ def is_block_resource(self, ast: Any) -> bool: ): self.push([self.is_resource_body], ast.args[1]) self.push([self.is_resource_def], ast.args[0]) - self.atomic_unit.code = ChefParser._get_content(ast, self.source) - self.atomic_unit.line = ChefParser._get_content_bounds( - ast, self.source - )[0] + info = ChefParser._get_info(ast, self.source) + set_loc_from_info(self.atomic_unit, info) return True return False @@ -251,10 +754,8 @@ def is_inline_resource(self, ast: Any) -> bool: ast.args[1], ) self.push([self.is_resource_type], ast.args[0]) - self.atomic_unit.code = ChefParser._get_content(ast, self.source) - self.atomic_unit.line = ChefParser._get_content_bounds( - ast, self.source - )[0] + info = ChefParser._get_info(ast, self.source) + set_loc_from_info(self.atomic_unit, info) return True return False @@ -286,24 +787,24 @@ def is_resource_type(self, ast: "ChefParser.Node") -> bool: return False def is_resource_name(self, ast: "ChefParser.Node") -> bool: - if isinstance(ast.args[0][0], ChefParser.Node) and ast.args[1] is False: - resource_id = ast.args[0][0] - self.atomic_unit.name = ChefParser._get_content( - resource_id, self.source - ) + if ChefParser._check_id(ast, ["args_add_block"]) and ast.args[1] is False: + resource_id = ast.args[0] + name = self.parser._get_value(resource_id, self.source) + assert name is not None + self.atomic_unit.name = name return True return False def is_inline_resource_name(self, ast: "ChefParser.Node") -> bool: if ( - ChefParser._check_node(ast.args[0][0], ["method_add_block"], 2) + ChefParser._check_node(ast.args[0].args[0], ["method_add_block"], 2) and ast.args[1] is False ): - resource_id = ast.args[0][0].args[0] - self.atomic_unit.name = ChefParser._get_content( - resource_id, self.source - ) - self.push([self.is_attribute], ast.args[0][0].args[1]) + resource_id = ast.args[0].args[0].args[0] + name = self.parser._get_value(resource_id, self.source) + assert name is not None + self.atomic_unit.name = name + self.push([self.is_attribute], ast.args[0].args[0].args[1]) return True return False @@ -315,12 +816,12 @@ def is_resource_body(self, ast: "ChefParser.Node") -> bool: def is_resource_body_without_attributes(self, ast: "ChefParser.Node") -> bool: if ( - ChefParser._check_id(ast.args[0][0], ["string_literal"]) + ChefParser._check_id(ast.args[0].args[0], ["string_literal"]) and ast.args[1] is False ): - self.atomic_unit.name = ChefParser._get_content( - ast.args[0][0], self.source - ) + name = self.parser._get_value(ast.args[0].args[0], self.source) + assert name is not None + self.atomic_unit.name = name return True return False @@ -330,119 +831,136 @@ def is_attribute(self, ast: Any) -> bool: ) and ChefParser._check_id(ast.args[0], ["call"]): self.push([self.is_attribute], ast.args[0].args[0]) elif ( - ChefParser._check_id(ast, ["command", "method_add_arg"]) + ChefParser._check_id( + ast, ["command", "method_add_arg", "method_add_block"] + ) and ast.args[1] != [] - ) or ( - ChefParser._check_id(ast, ["method_add_block"]) - and ChefParser._check_id(ast.args[0], ["method_add_arg"]) - and ChefParser._check_id(ast.args[1], ["brace_block", "do_block"]) ): - has_variable = ChefParser._check_has_variable(ast.args[1]) - value = ChefParser._get_content(ast.args[1], self.source) - if value == "nil": - value = "" - has_variable = False + value = self.parser._get_value(ast.args[1], self.source) + assert value is not None + info = ChefParser._get_info(ast, self.source) + a = Attribute( ChefParser._get_content(ast.args[0], self.source), value, - has_variable, + info, ) - a.line = ChefParser._get_content_bounds(ast, self.source)[0] - a.column = ChefParser._get_content_bounds(ast, self.source)[1] - a.code = ChefParser._get_source(ast, self.source) self.atomic_unit.add_attribute(a) - elif isinstance(ast, (ChefParser.Node, list)): + elif isinstance(ast, ChefParser.Node): + self.get_statements(ast) + elif isinstance(ast, list): for arg in reversed(ast): # type: ignore self.push([self.is_attribute], arg) return True class VariableChecker(Checker): - def __init__(self, source: List[str], ast: Any) -> None: - super().__init__(source) + def __init__(self, source: List[str], ast: Any, parser: "ChefParser") -> None: + super().__init__(source, parser) self.variables: List[Variable] = [] self.push([self.is_variable], ast) def is_variable(self, ast: Any) -> bool: def create_variable( - key: Any, name: str, value: str | None, has_variable: bool + key: Any, + value_ast: Any, + name: str, + value: Expr, ): - variable = Variable(name, value, has_variable) - variable.line = ChefParser._get_content_bounds(key, self.source)[0] - variable.column = ChefParser._get_content_bounds(key, self.source)[1] - variable.code = ChefParser._get_source(ast, self.source) + bounds = ChefParser._get_content_bounds(key, self.source) + bounds_value = ChefParser._get_content_bounds(value_ast, self.source) + + code = ChefParser._get_content(key, self.source) + variable = Variable( + name, + value, + ElementInfo( + bounds[0], + bounds[1] + 1, + bounds_value[2], + # We are considering a start at 0 and the index to be inclusive + # so we need to add 2 to the end column since + # we want it to start at 1 and be exclusive + bounds_value[3] + 2, + code, + ), + ) return variable - def parse_variable( - parent: KeyValue | None, - ast: Any, - key: Any, - current_name: str, - value_ast: Any, - ) -> None: - if ChefParser._check_node( - value_ast, ["hash"], 1 - ) and ChefParser._check_id(value_ast.args[0], ["assoclist_from_args"]): - variable = create_variable(key, current_name, None, False) - if parent == None: - self.variables.append(variable) - else: - parent.keyvalues.append(variable) - parent = variable - for assoc in value_ast.args[0].args[0]: - parse_variable( - parent, - ast, - assoc.args[0], - ChefParser._get_content(assoc.args[0], self.source), - assoc.args[1], - ) - else: - value = ChefParser._get_content(value_ast, self.source) - has_variable = ChefParser._check_has_variable(value_ast) - if value == "nil": - value = "" - has_variable = False + def handle_assign_node(name_ast: Any, value_ast: Any): + name = ChefParser._get_content(name_ast, self.source) + value = self.parser._get_value(value_ast, self.source) + assert value is not None + variable = create_variable(name_ast, ast, name, value) + self.variables.append(variable) - variable = create_variable(key, current_name, value, has_variable) + if ChefParser._check_node(ast, ["assign", "massign"], 2): + handle_assign_node(ast.args[0], ast.args[1]) + return True + elif ChefParser._check_node(ast, ["opassign"], 3): + handle_assign_node(ast.args[0], ast.args[2]) + name = self.parser._get_value(ast.args[0], self.source) + assert name is not None + op = ast.args[1].args[0] + + def binary(type: Callable[[ElementInfo, Expr, Expr], Expr]) -> Expr: + return type( + ElementInfo.from_code_element(self.variables[-1]), + name, + self.variables[-1].value, + ) - if parent == None: - self.variables.append(variable) - else: - parent.keyvalues.append(variable) - - if ChefParser._check_node(ast, ["assign"], 2): - name = "" - names = ChefParser._get_content(ast.args[0], self.source).split("[") - parent = None - for i, n in enumerate(names): - if n.endswith("]"): - n = n[:-1] - if (n.startswith("'") and n.endswith("'")) or ( - n.startswith('"') and n.endswith('"') - ): - name = n[1:-1] - elif n.startswith(":"): - name = n[1:] - else: - name = n + match op: + case "||=": + value = self.parser._get_value(ast.args[0], self.source) + assert isinstance(value, Expr) + cond = NotEqual( + ElementInfo.from_code_element(self.variables[-1]), + value, + Null(), + ) + true = ConditionalStatement( + cond, ConditionalStatement.ConditionType.IF, is_top=True + ) + true.add_statement(value) + false = ConditionalStatement( + Null(), + ConditionalStatement.ConditionType.IF, + is_default=True, + ) + false.add_statement(name) + true.else_statement = false + self.variables[-1].value = true + case "+=": + self.variables[-1].value = binary(Sum) + case "-=": + self.variables[-1].value = binary(Subtract) + case "*=": + self.variables[-1].value = binary(Multiply) + case "/=": + self.variables[-1].value = binary(Divide) + case "%=": + self.variables[-1].value = binary(Modulo) + case "**=": + self.variables[-1].value = binary(Power) + case "|=": + self.variables[-1].value = binary(Sum) + case "&=": + self.variables[-1].value = binary(BitwiseAnd) + case "&&=": + self.variables[-1].value = binary(And) + case "<<=": + self.variables[-1].value = binary(LeftShift) + case _: + raise ValueError(f"Unknown operator {op}") - if i == len(names) - 1: - parse_variable(parent, ast, ast.args[0], name, ast.args[1]) - else: - variable = create_variable(ast.args[0], name, None, False) - if i == 0: - self.variables.append(variable) - elif parent is not None: - parent.keyvalues.append(variable) - parent = variable return True return False class IncludeChecker(Checker): - def __init__(self, source: List[str], ast: Any) -> None: - super().__init__(source) + def __init__(self, source: List[str], ast: Any, parser: "ChefParser") -> None: + super().__init__(source, parser) self.push([self.is_include], ast) self.code = "" @@ -454,7 +972,7 @@ def is_include(self, ast: Any) -> bool: ): self.push([self.is_include_name], ast.args[1]) self.push([self.is_include_type], ast.args[0]) - self.code = ChefParser._get_source(ast, self.source) + self.code = ChefParser._get_content(ast, self.source) return True return False @@ -469,78 +987,238 @@ def is_include_type(self, ast: "ChefParser.Node") -> bool: def is_include_name(self, ast: "ChefParser.Node") -> bool: if ( - ChefParser._check_id(ast.args[0][0], ["string_literal"]) + ChefParser._check_id(ast.args[0].args[0], ["string_literal"]) and ast.args[1] is False ): - d = Dependency(ChefParser._get_content(ast.args[0][0], self.source)) - d.line = ChefParser._get_content_bounds(ast, self.source)[0] - d.code = self.code + d = Dependency([ChefParser._get_content(ast.args[0][0], self.source)]) + info = ChefParser._get_info(ast, self.source) + info.code = self.code + set_loc_from_info(d, info) self.include = d return True return False - # FIXME only identifying case statement - class ConditionChecker(Checker): - def __init__(self, source: List[str], ast: Any) -> None: - super().__init__(source) + class CaseChecker(Checker): + def __init__(self, source: List[str], ast: Any, parser: "ChefParser") -> None: + super().__init__(source, parser) + self.current_condition = None + self.condition = None self.push([self.is_case], ast) + def get_statements(self, ast: Any, cond: ConditionalStatement) -> None: + for arg in ast.args: + if ChefParser._check_id(arg, ["stmts_add"]): + self.parser._transverse_ast(arg.args, cond, self.source) + def is_case(self, ast: Any) -> bool: if ChefParser._check_node(ast, ["case"], 2): - self.case_head = ChefParser._get_content(ast.args[0], self.source) - self.condition = None + case_head = self.parser._get_value(ast.args[0], self.source) + assert isinstance(case_head, Expr) + self.case_head: Expr = case_head + self.current_condition = None self.push([self.is_case_condition], ast.args[1]) return True elif ChefParser._check_node(ast, ["case"], 1): - self.case_head = "" - self.condition = None + self.case_head = Null() + self.current_condition = None self.push([self.is_case_condition], ast.args[0]) return True return False def is_case_condition(self, ast: Any) -> bool: - if ChefParser._check_node(ast, ["when"], 3) or ChefParser._check_node( - ast, ["when"], 2 + if ChefParser._check_node(ast, ["when", "in"], 3) or ChefParser._check_node( + ast, ["when", "in"], 2 ): + value = self.parser._get_value(ast.args[0], self.source) + assert isinstance(value, Expr) + + equals_info = ElementInfo.from_code_element(self.case_head) if self.condition is None: self.condition = ConditionalStatement( - self.case_head - + " == " - + ChefParser._get_content(ast.args[0][0], self.source), + Equal(equals_info, self.case_head, value), ConditionalStatement.ConditionType.SWITCH, + is_top=True, ) - self.condition.code = ChefParser._get_source(ast, self.source) - self.condition.line = ChefParser._get_content_bounds( - ast, self.source - )[0] + info = ChefParser._get_info(ast, self.source) + set_loc_from_info(self.condition, info) self.current_condition = self.condition - else: + elif self.current_condition is not None: self.current_condition.else_statement = ConditionalStatement( - self.case_head - + " == " - + ChefParser._get_content(ast.args[0][0], self.source), + Equal(equals_info, self.case_head, value), ConditionalStatement.ConditionType.SWITCH, ) self.current_condition = self.current_condition.else_statement - self.current_condition.code = ChefParser._get_source( - ast, self.source - ) - self.current_condition.line = ChefParser._get_content_bounds( - ast, self.source - )[0] + info = ChefParser._get_info(ast, self.source) + set_loc_from_info(self.current_condition, info) + if self.current_condition is not None: + self.get_statements(ast, self.current_condition) if len(ast.args) == 3: self.push([self.is_case_condition], ast.args[2]) return True - elif ChefParser._check_node(ast, ["else"], 1): + elif ( + ChefParser._check_node(ast, ["else"], 1) + and self.current_condition is not None + ): self.current_condition.else_statement = ConditionalStatement( - "", ConditionalStatement.ConditionType.SWITCH, is_default=True + Null(), ConditionalStatement.ConditionType.SWITCH, is_default=True ) - self.current_condition.else_statement.code = ChefParser._get_source( - ast, self.source + info = ChefParser._get_info(ast, self.source) + set_loc_from_info(self.current_condition.else_statement, info) + self.get_statements(ast, self.current_condition.else_statement) + return True + return False + + class IfChecker(Checker): + def __init__(self, source: List[str], ast: Any, parser: "ChefParser") -> None: + super().__init__(source, parser) + self.current_condition = None + self.condition = None + self.push([self.is_if], ast) + + def get_statements(self, ast: Any, cond: ConditionalStatement) -> None: + assert self.condition is not None + for arg in ast.args: + if ChefParser._check_id(arg, ["stmts_add"]): + self.parser._transverse_ast(arg.args, cond, self.source) + + def is_if(self, ast: Any) -> bool: + if ChefParser._check_node( + ast, ["if", "unless"], 2 + ) or ChefParser._check_node(ast, ["if", "unless"], 3): + condition = self.parser._get_value(ast.args[0], self.source) + assert isinstance(condition, Expr) + if ChefParser._check_id(ast, ["unless"]): + condition = Not(ElementInfo.from_code_element(condition), condition) + self.condition = ConditionalStatement( + condition, ConditionalStatement.ConditionType.IF, is_top=True ) - self.current_condition.else_statement.line = ( - ChefParser._get_content_bounds(ast, self.source)[0] + info = ChefParser._get_info(ast, self.source) + set_loc_from_info(self.condition, info) + self.current_condition = self.condition + if ChefParser._check_node(ast, ["if", "unless"], 3): + self.push([self.is_if_condition], ast.args[2]) + self.get_statements(ast, self.condition) + return True + return False + + def is_if_condition(self, ast: Any) -> bool: + if ( + ChefParser._check_node(ast, ["elsif"], 2) + or ChefParser._check_node(ast, ["elsif"], 3) + ) and self.current_condition is not None: + condition = self.parser._get_value(ast.args[0], self.source) + assert isinstance(condition, Expr) + new_condition = ConditionalStatement( + condition, ConditionalStatement.ConditionType.IF + ) + info = ChefParser._get_info(ast, self.source) + set_loc_from_info(new_condition, info) + self.get_statements(ast, new_condition) + self.current_condition.else_statement = new_condition + self.current_condition = new_condition + if ChefParser._check_node(ast, ["elsif"], 3): + self.push([self.is_if_condition], ast.args[2]) + return True + elif ( + ChefParser._check_node(ast, ["else"], 1) + and self.current_condition is not None + ): + new_condition = ConditionalStatement( + Null(), ConditionalStatement.ConditionType.IF, is_default=True ) + info = ChefParser._get_info(ast, self.source) + set_loc_from_info(new_condition, info) + self.get_statements(ast, new_condition) + self.current_condition.else_statement = new_condition + return True + return False + + class ClassChecker(Checker): + def __init__(self, source: List[str], ast: Any, parser: "ChefParser") -> None: + super().__init__(source, parser) + self.push([self.is_class], ast) + self.unit_block = None + + def is_class(self, ast: Any) -> bool: + if ChefParser._check_node( + ast, ["class", "sclass"], 2 + ) or ChefParser._check_node(ast, ["class"], 3): + name = ChefParser._get_content(ast.args[0], self.source) + self.class_name = name + self.unit_block = UnitBlock(name, UnitBlockType.definition) + if len(ast.args) == 2: + self.push([self.is_class_body], ast.args[1]) + else: + self.push([self.is_class_body], ast.args[2]) + return True + return False + + def is_class_body(self, ast: Any) -> bool: + if ChefParser._check_id(ast, ["bodystmt"]): + assert self.unit_block is not None + self.parser._transverse_ast(ast, self.unit_block, self.source) + return True + return False + + class DefChecker(Checker): + def __init__(self, source: List[str], ast: Any, parser: "ChefParser") -> None: + super().__init__(source, parser) + self.push([self.is_def], ast) + self.unit_block = None + + def is_def(self, ast: Any) -> bool: + if ChefParser._check_node(ast, ["def"], 2) or ChefParser._check_node( + ast, ["def"], 3 + ): + name = ChefParser._get_content(ast.args[0], self.source) + self.unit_block = UnitBlock(name, UnitBlockType.definition) + if len(ast.args) == 2: + self.push([self.is_def_body], ast.args[1]) + else: + self.push([self.is_def_body], ast.args[2]) + return True + elif ChefParser._check_node(ast, ["defs"], 4) or ChefParser._check_node( + ast, ["defs"], 5 + ): + target = ChefParser._get_content(ast.args[0], self.source) + op = ChefParser._get_content(ast.args[1], self.source) + method = ChefParser._get_content(ast.args[2], self.source) + name = target + op + method + self.unit_block = UnitBlock(name, UnitBlockType.definition) + if len(ast.args) == 4: + self.push([self.is_def_body], ast.args[3]) + else: + self.push([self.is_def_body], ast.args[4]) + return True + + return False + + def is_def_body(self, ast: Any) -> bool: + if ChefParser._check_id(ast, ["bodystmt"]): + assert self.unit_block is not None + self.parser._transverse_ast(ast, self.unit_block, self.source) + return True + return False + + class ModuleChecker(Checker): + def __init__(self, source: List[str], ast: Any, parser: "ChefParser") -> None: + super().__init__(source, parser) + self.push([self.is_module], ast) + self.unit_block = None + + def is_module(self, ast: Any) -> bool: + if ChefParser._check_node(ast, ["module"], 2): + name = ChefParser._get_content(ast.args[0], self.source) + self.unit_block = UnitBlock(name, UnitBlockType.block) + self.push([self.is_module_body], ast.args[1]) + return True + + return False + + def is_module_body(self, ast: Any) -> bool: + if ChefParser._check_id(ast, ["bodystmt"]): + assert self.unit_block is not None + self.parser._transverse_ast(ast, self.unit_block, self.source) return True return False @@ -568,73 +1246,106 @@ def __create_ast(l: List[ChefValue | "ChefParser.Node"]) -> "ChefParser.Node": return ChefParser.Node(l[0][1], args) # type: ignore - @staticmethod - def __transverse_ast(ast: Any, unit_block: UnitBlock, source: List[str]) -> None: - def get_var(parent_name: str, vars: List[Variable]): - for var in vars: - if var.name == parent_name: - return var - return None - - def add_variable_to_unit_block( - variable: Variable, unit_block_vars: List[Variable] - ) -> None: - var_name = variable.name - var = get_var(var_name, unit_block_vars) - if var and var.value == None and variable.value == None: - for v in variable.keyvalues: - add_variable_to_unit_block(v, var.keyvalues) # type: ignore - else: - unit_block_vars.append(variable) - + def _transverse_ast( + self, + ast: Any, + st: UnitBlock | ConditionalStatement | BlockExpr | AtomicUnit, + source: List[str], + ) -> None: if isinstance(ast, list): for arg in ast: # type: ignore if isinstance(arg, (ChefParser.Node, list)): - ChefParser.__transverse_ast(arg, unit_block, source) + self._transverse_ast(arg, st, source) else: resource_checker = ChefParser.ResourceChecker( - AtomicUnit("", ""), source, ast + AtomicUnit(Null(), ""), source, ast, self ) if resource_checker.check_all(): - unit_block.add_atomic_unit(resource_checker.atomic_unit) + if isinstance(st, UnitBlock): + st.add_atomic_unit(resource_checker.atomic_unit) + else: + st.add_statement(resource_checker.atomic_unit) return - variable_checker = ChefParser.VariableChecker(source, ast) + # When statements are inside atomic units + # this allows us to still correctly create attributes + if ( + self._inside_atomic_unit + and not ChefParser._check_id(ast, ["stmts_add"]) + and resource_checker.is_attribute(ast) + and len(resource_checker.atomic_unit.attributes) > 0 + ): + if isinstance(st, AtomicUnit): + st.add_attribute(resource_checker.atomic_unit.attributes[-1]) + else: + st.add_statement(resource_checker.atomic_unit.attributes[-1]) + return + + variable_checker = ChefParser.VariableChecker(source, ast, self) if variable_checker.check_all(): for variable in variable_checker.variables: - add_variable_to_unit_block(variable, unit_block.variables) - # variables might have resources associated to it - ChefParser.__transverse_ast(ast.args[1], unit_block, source) + if isinstance(st, UnitBlock): + st.add_variable(variable) + else: + st.add_statement(variable) return - include_checker = ChefParser.IncludeChecker(source, ast) + include_checker = ChefParser.IncludeChecker(source, ast, self) if include_checker.check_all(): - unit_block.add_dependency(include_checker.include) + if isinstance(st, UnitBlock): + st.add_dependency(include_checker.include) + else: + st.add_statement(include_checker.include) return - if_checker = ChefParser.ConditionChecker(source, ast) - if if_checker.check_all(): - if if_checker.condition is not None: - unit_block.add_statement(if_checker.condition) - # Check blocks inside - ChefParser.__transverse_ast( - ast.args[len(ast.args) - 1], unit_block, source - ) - return + if_checkers = [ + ChefParser.CaseChecker(source, ast, self), + ChefParser.IfChecker(source, ast, self), + ] + for if_checker in if_checkers: + if if_checker.check_all(): + if if_checker.condition is not None: + st.add_statement(if_checker.condition) + return + + block_checkers = [ + ChefParser.ClassChecker(source, ast, self), + ChefParser.DefChecker(source, ast, self), + ChefParser.ModuleChecker(source, ast, self), + ] + for checker in block_checkers: + if checker.check_all(): + assert checker.unit_block is not None + if isinstance(st, UnitBlock): + st.add_unit_block(checker.unit_block) + else: + st.add_statement(checker.unit_block) + return + + try: + value = self._get_value(ast, source) + if ( + value is not None + and ast.id not in ["stmts_add", "stmts_new"] + and not isinstance(value, Null) + ): + st.add_statement(value) + return + except ValueError: + pass for arg in ast.args: if isinstance(arg, (ChefParser.Node, list)): - ChefParser.__transverse_ast(arg, unit_block, source) + self._transverse_ast(arg, st, source) - @staticmethod - def __parse_recipe(path: str, file: str) -> UnitBlock: + def __parse_recipe(self, path: str, file: str) -> UnitBlock | None: with open(os.path.join(path, file)) as f: - ripper_path = resource_filename( - "glitch.parsers", "resources/comments.rb.template" + ripper_content = ( + files("glitch.parsers") + .joinpath("resources/comments.rb.template") + .read_text() ) - ripper = open(ripper_path, "r") - ripper_script = Template(ripper.read()) - ripper.close() + ripper_script = Template(ripper_content) ripper_script = ripper_script.substitute( {"path": '"' + os.path.join(path, file) + '"'} ) @@ -651,7 +1362,7 @@ def __parse_recipe(path: str, file: str) -> UnitBlock: throw_exception( EXCEPTIONS["CHEF_COULD_NOT_PARSE"], os.path.join(path, file) ) - return unit_block + return None with tempfile.NamedTemporaryFile(mode="w+") as tmp: tmp.write(ripper_script) @@ -662,35 +1373,39 @@ def __parse_recipe(path: str, file: str) -> UnitBlock: script_ast = p.read() p.close() comments, _ = parser_yacc(script_ast) - if comments is not None: - comments.reverse() + comments.reverse() for comment, line in comments: c = Comment(re.sub(r"\\n$", "", comment)) - c.code = source[line - 1] - c.line = line + comment_code = source[line - 1] + info = ElementInfo( + line, 1, line, len(comment_code), comment_code + ) + set_loc_from_info(c, info) unit_block.add_comment(c) except: throw_exception( EXCEPTIONS["CHEF_COULD_NOT_PARSE"], os.path.join(path, file) ) + return None try: p = os.popen( "ruby -r ripper -e 'file = \ File.open(\"" + os.path.join(path, file) - + "\")\npp Ripper.sexp(file)'" + + "\")\npp Ripper.sexp_raw(file)'" ) script_ast = p.read() p.close() _, program = parser_yacc(script_ast) - ast = ChefParser.__create_ast(program) - ChefParser.__transverse_ast(ast, unit_block, source) + ast = ChefParser.__create_ast(program) # type: ignore + self._transverse_ast(ast, unit_block, source) except: throw_exception( EXCEPTIONS["CHEF_COULD_NOT_PARSE"], os.path.join(path, file) ) + return None return unit_block @@ -701,7 +1416,9 @@ def parse_folder(path: str) -> None: f for f in os.listdir(path) if os.path.isfile(os.path.join(path, f)) ] for file in files: - res.add_block(self.__parse_recipe(path, file)) + recipe = self.__parse_recipe(path, file) + if recipe is not None: + res.add_block(recipe) res: Module = Module(os.path.basename(os.path.normpath(path)), path) super().parse_file_structure(res.folder, path) @@ -715,7 +1432,7 @@ def parse_folder(path: str) -> None: return res - def parse_file(self, path: str, type: UnitBlockType) -> UnitBlock: + def parse_file(self, path: str, type: UnitBlockType) -> UnitBlock | None: return self.__parse_recipe(os.path.dirname(path), os.path.basename(path)) def parse_folder(self, path: str) -> Project: diff --git a/glitch/parsers/docker.py b/glitch/parsers/docker.py deleted file mode 100644 index 75789573..00000000 --- a/glitch/parsers/docker.py +++ /dev/null @@ -1,433 +0,0 @@ -import ast -import os.path -import re -from dataclasses import dataclass, field -from typing import List, Dict, Tuple, Optional, Union - -import bashlex # type: ignore -from dockerfile_parse import DockerfileParser - -import glitch.parsers.parser as p -from glitch.exceptions import throw_exception, EXCEPTIONS -from glitch.repr.inter import * - - -@dataclass -class DFPStructure: - content: str - endline: int - instruction: str - startline: int - value: str - raw_content: str - - -class DockerParser(p.Parser): - def parse_file(self, path: str, type: UnitBlockType) -> UnitBlock: - try: - with open(path) as f: - file_lines = list(f) - f.seek(0) - dfp = DockerfileParser() - dfp.content = f.read() - structure = [ - DFPStructure( - raw_content="".join( - file_lines[s["startline"] : s["endline"] + 1] - ), - **s, - ) - for s in dfp.structure - ] - - stage_indexes = [ - i for i, e in enumerate(structure) if e.instruction == "FROM" - ] - if len(stage_indexes) > 1: - main_block = UnitBlock(os.path.basename(path), type) - main_block.code = dfp.content - stages = self.__get_stages(stage_indexes, structure) - for i, (name, s) in enumerate(stages): - unit_block = self.__parse_stage( - name, path, UnitBlockType.block, s - ) - unit_block.line = structure[stage_indexes[i]].startline + 1 - unit_block.code = "".join([struct.content for struct in s]) - main_block.add_unit_block(unit_block) - else: - self.__add_user_tag(structure) - main_block = self.__parse_stage( - dfp.baseimage, path, type, structure - ) - main_block.line = ( - structure[stage_indexes[0]].startline + 1 - if stage_indexes - else 1 - ) - main_block.code = "".join([struct.content for struct in structure]) - - main_block.path = path - return main_block - except Exception as e: - throw_exception(EXCEPTIONS["DOCKER_UNKNOW_ERROR"], str(e)) - main_block = UnitBlock(os.path.basename(path), type) - main_block.path = path - return main_block - - def parse_folder(self, path: str) -> Project: - project = Project(os.path.basename(os.path.normpath(path))) - project.blocks, project.modules = self._parse_folder(path) - return project - - def parse_module(self, path: str) -> Module: - module = Module(os.path.basename(os.path.normpath(path)), path) - module.blocks, module.modules = self._parse_folder(path) - return module - - def _parse_folder(self, path: str) -> Tuple[List[UnitBlock], List[Module]]: - files = [os.path.join(path, f) for f in os.listdir(path)] - dockerfiles = [ - f - for f in files - if os.path.isfile(f) and "Dockerfile" in os.path.basename(f) - ] - modules = [ - f - for f in files - if os.path.isdir(f) and DockerParser._contains_dockerfiles(f) - ] - - blocks = [self.parse_file(f, UnitBlockType.script) for f in dockerfiles] - modules = [self.parse_module(f) for f in modules] - return blocks, modules - - @staticmethod - def _contains_dockerfiles(path: str) -> bool: - if not os.path.exists(path): - return False - if not os.path.isdir(path): - return "Dockerfile" in os.path.basename(path) - for f in os.listdir(path): - contains_dockerfiles = DockerParser._contains_dockerfiles( - os.path.join(path, f) - ) - if contains_dockerfiles: - return True - return False - - @staticmethod - def __parse_stage( - name: str, path: str, unit_type: UnitBlockType, structure: List[DFPStructure] - ) -> UnitBlock: - u = UnitBlock(name, unit_type) - u.path = path - for s in structure: - try: - DockerParser.__parse_instruction(s, u) - except NotImplementedError: - throw_exception(EXCEPTIONS["DOCKER_NOT_IMPLEMENTED"], s.content) - return u - - @staticmethod - def __parse_instruction(element: DFPStructure, unit_block: UnitBlock) -> None: - instruction = element.instruction - if instruction in ["ENV", "USER", "ARG", "LABEL"]: - unit_block.variables += DockerParser.__create_variable_block(element) - elif instruction == "COMMENT": - c = Comment(element.value) - c.line = element.startline + 1 - c.code = element.content - unit_block.add_comment(c) - elif instruction in ["RUN", "CMD", "ENTRYPOINT"]: - try: - c_parser = CommandParser(element) - aus = c_parser.parse_command() - unit_block.atomic_units += aus - except Exception: - throw_exception(EXCEPTIONS["SHELL_COULD_NOT_PARSE"], element.content) - elif instruction == "ONBUILD": - dfp = DockerfileParser() - dfp.content = element.value - element = DFPStructure( - **dfp.structure[0], raw_content=dfp.structure[0]["content"] - ) - DockerParser.__parse_instruction(element, unit_block) - elif instruction == "COPY": - au = AtomicUnit("", "copy") - paths = [v for v in element.value.split(" ") if not v.startswith("--")] - au.add_attribute(Attribute("src", str(paths[0:-1]), False)) - au.add_attribute(Attribute("dest", paths[-1], False)) - for attr in au.attributes: - attr.code = element.content - attr.line = element.startline + 1 - au.code = element.content - au.line = element.startline + 1 - unit_block.add_atomic_unit(au) - # TODO: Investigate keywords and parse them - elif instruction in ["ADD", "VOLUME", "WORKDIR"]: - pass - elif instruction in ["STOPSIGNAL", "HEALTHCHECK", "SHELL"]: - pass - elif instruction == "EXPOSE": - pass - - @staticmethod - def __get_stages( - stage_indexes: List[int], structure: List[DFPStructure] - ) -> List[Tuple[str, List[DFPStructure]]]: - stages: List[Tuple[str, List[DFPStructure]]] = [] - for i, stage_i in enumerate(stage_indexes): - stage_image = structure[stage_i].value.split(" ")[0] - stage_start = stage_i if i != 0 else 0 - stage_end = ( - len(structure) if i == len(stage_indexes) - 1 else stage_indexes[i + 1] - ) - stages.append( - ( - stage_image, - DockerParser.__get_stage_structure( - structure, stage_start, stage_end - ), - ) - ) - return stages - - @staticmethod - def __get_stage_structure( - structure: List[DFPStructure], stage_start: int, stage_end: int - ): - sub_structure = structure[stage_start:stage_end].copy() - DockerParser.__add_user_tag(sub_structure) - return sub_structure - - @staticmethod - def __create_variable_block(element: DFPStructure) -> List[Variable]: - variables: List[Variable] = [] - if element.instruction == "USER": - variables.append(Variable("user-profile", element.value, False)) - elif element.instruction == "ARG": - value = element.value.split("=") - arg = value[0] - default = value[1] if len(value) == 2 else None - variables.append(Variable(arg, default if default else "ARG", True)) - elif element.instruction == "ENV": - if "=" in element.value: - # TODO: Improve code attribution for multiple values - return DockerParser.__parse_multiple_key_value_variables( - element.content, element.startline - ) - if len(element.value.split(" ")) != 2: - raise NotImplementedError() - env, value = element.value.split(" ") - variables.append(Variable(env, value, value.startswith("$"))) - else: # LABEL - return DockerParser.__parse_multiple_key_value_variables( - element.content, element.startline - ) - - for v in variables: - if v.value == '""' or v.value == "''": - v.value = "" - v.line = element.startline + 1 - v.code = element.content - return variables - - @staticmethod - def __parse_multiple_key_value_variables( - content: str, base_line: int - ) -> List[Variable]: - variables: List[Variable] = [] - for i, line in enumerate(content.split("\n")): - for match in re.finditer( - r"([\w_]*)=(?:(?:'|\")([\w\. <>@]*)(?:\"|')|([\w\.]*))", line - ): - value = match.group(2) or match.group(3) or "" - v = Variable(match.group(1), value, value.startswith("$")) - v.line = base_line + i + 1 - v.code = line - variables.append(v) - return variables - - @staticmethod - def __add_user_tag(structure: List[DFPStructure]) -> None: - if len([s for s in structure if s.instruction == "USER"]) > 0: - return - - index, line = -1, 0 - for i, s in enumerate(structure): - if s.instruction == "FROM": - index = i - line = s.startline - break - structure.insert( - index + 1, DFPStructure("USER root", line, "USER", line, "root", "") - ) - - -@dataclass -class ShellCommand: - sudo: bool - command: str - args: List[str] - code: str - options: Dict[str, Tuple[Union[str, bool, int, float], str]] = field( - default_factory=lambda: {} - ) - main_arg: Optional[str] = None - line: int = -1 - - def to_atomic_unit(self) -> AtomicUnit: - au = AtomicUnit(self.main_arg, self.command) - au.line = self.line - au.code = self.code - if self.sudo: - sudo = Attribute("sudo", "True", False) - sudo.code = "sudo" - sudo.line = self.line - au.add_attribute(sudo) - for key, (value, _) in self.options.items(): - has_variable = "$" in value if isinstance(value, str) else False - attr = Attribute(key, value, has_variable) # type: ignore - attr.code = self.code - attr.line = self.line - au.add_attribute(attr) - return au - - -class CommandParser: - def __init__(self, command: DFPStructure) -> None: - value = ( - command.content.replace("RUN ", "") - if command.instruction == "RUN" - else command.value - ) - if value.startswith("[") and value.endswith("]"): - c_list = ast.literal_eval(value) - value = " ".join(c_list) - self.dfp_structure = command - self.command = value - self.line = command.startline + 1 - - def parse_command(self) -> List[AtomicUnit]: - # TODO: Fix get commands lines for scripts with multiline values - commands = self.__get_sub_commands() - aus: List[AtomicUnit] = [] - for line, c in commands: - try: - aus.append(self.__parse_single_command(c, line)) - except IndexError: - throw_exception(EXCEPTIONS["SHELL_COULD_NOT_PARSE"].format(" ".join(c))) - return aus - - def __parse_single_command(self, command: List[str], line: int) -> AtomicUnit: - command, carriage_returns = CommandParser.__strip_shell_command(command) - line += carriage_returns - sudo = command[0] == "sudo" - name_index = 0 if not sudo else 1 - command_type = command[name_index] - if len(command) == name_index + 1: - return ShellCommand( - sudo=sudo, - command=command_type, - args=[], - main_arg=command_type, - line=line, - code=self.dfp_structure.raw_content, - ).to_atomic_unit() - c = ShellCommand( - sudo=sudo, - command=command_type, - args=command[name_index + 1 :], - line=line, - code=self.dfp_structure.raw_content, - ) - CommandParser.__parse_shell_command(c) - return c.to_atomic_unit() - - @staticmethod - def __strip_shell_command(command: List[str]) -> Tuple[List[str], int]: - non_empty_indexes = [ - i for i, c in enumerate(command) if c not in ["\n", "", " ", "\r"] - ] - if not non_empty_indexes: - return ([], 0) - start, end = non_empty_indexes[0], non_empty_indexes[-1] - return command[start : end + 1], sum(1 for c in command if c == "\n") - - @staticmethod - def __parse_shell_command(command: ShellCommand) -> None: - if command.command == "chmod": - reference = [arg for arg in command.args if "--reference" in arg] - command.args = [arg for arg in command.args if not arg.startswith("-")] - command.main_arg = command.args[-1] - if reference: - reference[0] - command.options["reference"] = (reference.split("=")[1], reference) # type: ignore - else: - command.options["mode"] = command.args[0], command.args[0] - else: - CommandParser.__parse_general_command(command) - - @staticmethod - def __parse_general_command(command: ShellCommand) -> None: - args = command.args.copy() - # TODO: Solve issue where last argument is part of a parameter - main_arg_index = -1 if not args[-1].startswith("-") else 0 - if len(args) >= 3 and args[-2].startswith("-") and not args[0].startswith("-"): - main_arg_index = 0 - main_arg = args[main_arg_index] - del args[main_arg_index] - command.main_arg = main_arg - - for i, o in enumerate(args): - if not o.startswith("-"): - continue - - code = o - o = o.lstrip("-") - if "=" in o: - option = o.split("=") - command.options[option[0]] = option[1], code - continue - - if len(args) == i + 1 or args[i + 1].startswith("-"): - command.options[o] = True, code - else: - command.options[o] = args[i + 1], f"{code} {args[i+1]}" - - def __get_sub_commands(self) -> List[Tuple[int, List[str]]]: - commands: List[Tuple[int, List[str]]] = [] - tmp: List[str] = [] - lines = ( - self.command.split("\n") - if not self.__contains_multi_line_values(self.command) - else [self.command] - ) - current_line = self.line - for i, line in enumerate(lines): - for part in bashlex.split(line): # type: ignore - if part in ["&&", "&", "|", ";"]: - commands.append((current_line, tmp)) - current_line = self.line + i - tmp = [] - continue - tmp.append(part) # type: ignore - commands.append((current_line, tmp)) - return commands - - @staticmethod - def __contains_multi_line_values(command: str) -> bool: - def is_multi_line_str(line: str) -> bool: - return line.count('"') % 2 != 0 or line.count("'") % 2 != 0 - - def has_open_parentheses(line: str) -> bool: - return ( - line.count("(") != line.count(")") - or line.count("[") != line.count("]") - or line.count("{") != line.count("}") - ) - - lines = command.split("\n") - return any( - (is_multi_line_str(line) or has_open_parentheses(line)) for line in lines - ) diff --git a/glitch/parsers/gha.py b/glitch/parsers/gha.py index b21ba741..3daa9b8d 100644 --- a/glitch/parsers/gha.py +++ b/glitch/parsers/gha.py @@ -7,97 +7,89 @@ from ruamel.yaml.main import YAML from ruamel.yaml.nodes import ( Node, - ScalarNode, MappingNode, - SequenceNode, - CollectionNode, ) -from pkg_resources import resource_filename +from importlib.resources import files from glitch.exceptions import EXCEPTIONS, throw_exception class GithubActionsParser(YamlParser): - @staticmethod - def __get_value(node: Node) -> Any: - if isinstance(node, ScalarNode): - return node.value - elif isinstance(node, MappingNode): - return { - GithubActionsParser.__get_value(key): GithubActionsParser.__get_value( - value - ) - for key, value in node.value - } - elif isinstance(node, SequenceNode): - return [GithubActionsParser.__get_value(value) for value in node.value] - elif isinstance(node, CollectionNode): - return node.value - else: - return None - - @staticmethod - def __parse_variable(key: Node, value: Node, lines: List[str]) -> Variable: - vars: List[KeyValue] = [] - - if isinstance(value, MappingNode): - var_value = None - for k, v in value.value: - vars.append(GithubActionsParser.__parse_variable(k, v, lines)) - else: - var_value = GithubActionsParser.__get_value(value) - - var = Variable(GithubActionsParser.__get_value(key), var_value, False) - if isinstance(var.value, str): - var.has_variable = "${{" in var.value - var.line, var.column = key.start_mark.line + 1, key.start_mark.column + 1 - var.code = GithubActionsParser._get_code(key, value, lines) - for child in vars: - var.keyvalues.append(child) + def __init__(self): + super().__init__({"variable_start_string": "${{"}) + + def __parse_dict(self, node: Node) -> Dict[str, Node]: + result: Dict[str, Node] = {} + if isinstance(node, MappingNode): + for key, value in node.value: + result[key.value] = value + return result + + def __parse_variable(self, key: Node, value: Node, lines: List[str]) -> Variable: + var_value = self.get_value(value, lines) + + name = self._get_code(key, key, lines) + code = self._get_code(key, value, lines) + + var = Variable( + name, + var_value, + ElementInfo( + key.start_mark.line + 1, + key.start_mark.column + 1, + value.end_mark.line + 1, + value.end_mark.column + 1, + code, + ), + ) return var - @staticmethod - def __parse_attribute(key: Node, value: Node, lines: List[str]) -> Attribute: - attrs: List[KeyValue] = [] - - if isinstance(value, MappingNode): - attr_value = None - for k, v in value.value: - attrs.append(GithubActionsParser.__parse_attribute(k, v, lines)) - else: - attr_value = GithubActionsParser.__get_value(value) - - attr = Attribute(GithubActionsParser.__get_value(key), attr_value, False) - if isinstance(attr.value, str): - attr.has_variable = "${{" in attr.value - attr.line, attr.column = key.start_mark.line + 1, key.start_mark.column + 1 - attr.code = GithubActionsParser._get_code(key, value, lines) - for child in attrs: - attr.keyvalues.append(child) + def __parse_attribute(self, key: Node, value: Node, lines: List[str]) -> Attribute: + attr_value = self.get_value(value, lines) + name = self._get_code(key, key, lines) + code = self._get_code(key, value, lines) + + attr = Attribute( + name, + attr_value, + ElementInfo( + key.start_mark.line + 1, + key.start_mark.column + 1, + value.end_mark.line + 1, + value.end_mark.column + 1, + code, + ), + ) return attr def __parse_job(self, key: Node, value: Node, lines: List[str]) -> UnitBlock: job = UnitBlock(key.value, UnitBlockType.block) job.line, job.column = key.start_mark.line + 1, key.start_mark.column + 1 - job.code = GithubActionsParser._get_code(key, value, lines) + job.code = self._get_code(key, value, lines) for attr_key, attr_value in value.value: if attr_key.value == "steps": for step in attr_value.value: - step_dict = self.__get_value(step) - name = "" if "name" not in step_dict else step_dict["name"] + step_dict: Dict[str, Node] = self.__parse_dict(step) + name = ( + Null() + if "name" not in step_dict + else self.get_value(step_dict["name"], lines) + ) if "run" in step_dict: au_type = "shell" else: # uses - au_type = step_dict["uses"] + au_type = self._get_code( + step_dict["uses"], step_dict["uses"], lines + ) au = AtomicUnit(name, au_type) au.line, au.column = ( step.start_mark.line + 1, step.start_mark.column + 1, ) - au.code = GithubActionsParser._get_code(step, step, lines) + au.code = self._get_code(step, step, lines) for key, value in step.value: if key.value in ["with", "env"]: @@ -119,10 +111,6 @@ def __parse_job(self, key: Node, value: Node, lines: List[str]) -> UnitBlock: return job def parse_file(self, path: str, type: UnitBlockType) -> Optional[UnitBlock]: - schema_path = resource_filename( - "glitch.parsers", "resources/github_workflow.json" - ) - with open(path) as f: try: parsed_file = YAML().compose(f) @@ -137,20 +125,28 @@ def parse_file(self, path: str, type: UnitBlockType) -> Optional[UnitBlock]: return None with open(path) as f: - with open(schema_path) as f_schema: - schema = json.load(f_schema) - yaml = YAML() - try: - jsonschema.validate(yaml.load(f.read()), schema) # type: ignore - except jsonschema.ValidationError: - throw_exception(EXCEPTIONS["GHA_COULD_NOT_PARSE"], path) - return None - - parsed_file_value = self.__get_value(parsed_file) + schema = json.loads( + files("glitch.parsers") + .joinpath("resources/github_workflow.json") + .read_text() + ) + yaml = YAML() + try: + jsonschema.validate(yaml.load(f.read()), schema) # type: ignore + except jsonschema.ValidationError: + throw_exception(EXCEPTIONS["GHA_COULD_NOT_PARSE"], path) + return None + + parsed_file_value = self.__parse_dict(parsed_file) if "name" not in parsed_file_value: unit_block = UnitBlock("", type) else: - unit_block = UnitBlock(parsed_file_value["name"], type) + unit_block = UnitBlock( + self._get_code( + parsed_file_value["name"], parsed_file_value["name"], lines + ), + type, + ) unit_block.path = path for key, value in parsed_file.value: @@ -167,7 +163,7 @@ def parse_file(self, path: str, type: UnitBlockType) -> Optional[UnitBlock]: unit_block.add_attribute(self.__parse_attribute(key, value, lines)) with open(path) as f: - comments = list(GithubActionsParser._get_comments(parsed_file, f)) + comments = list(self._get_comments(parsed_file, f)) for comment in sorted(comments, key=lambda x: x[0]): c = Comment(comment[1]) c.line = comment[0] diff --git a/glitch/parsers/puppet.py b/glitch/parsers/puppet.py index e8ff0934..96fd4f65 100644 --- a/glitch/parsers/puppet.py +++ b/glitch/parsers/puppet.py @@ -1,14 +1,14 @@ -# type: ignore -# TODO: The file needs a refactor so the types make sense import os +import re import traceback -from puppetparser.parser import parse as parse_puppet -import puppetparser.model as puppetmodel +import copy +from puppetparser.parser import parse as parse_puppet # type: ignore +import puppetparser.model as puppetmodel # type: ignore from glitch.exceptions import EXCEPTIONS, throw_exception import glitch.parsers.parser as p from glitch.repr.inter import * -from typing import List, Any, Tuple, Dict +from typing import List, Any, Dict, Callable class PuppetParser(p.Parser): @@ -16,27 +16,10 @@ class PuppetParser(p.Parser): def __process_unitblock_component( ce: CodeElement | List[CodeElement], unit_block: UnitBlock ) -> None: - def get_var(parent_name: str, vars: List[KeyValue]): - for var in vars: - if var.name == parent_name: - return var - return None - - def add_variable_to_unit_block( - variable: KeyValue, unit_block_vars: List[KeyValue] - ) -> None: - var_name = variable.name - var = get_var(var_name, unit_block_vars) - if var and var.value == None and variable.value == None: - for v in variable.keyvalues: - add_variable_to_unit_block(v, var.keyvalues) - else: - unit_block_vars.append(variable) - if isinstance(ce, Dependency): unit_block.add_dependency(ce) elif isinstance(ce, Variable): - add_variable_to_unit_block(ce, unit_block.variables) # type: ignore + unit_block.add_variable(ce) elif isinstance(ce, AtomicUnit): unit_block.add_atomic_unit(ce) elif isinstance(ce, UnitBlock): @@ -45,513 +28,717 @@ def add_variable_to_unit_block( unit_block.add_attribute(ce) elif isinstance(ce, ConditionalStatement): unit_block.add_statement(ce) + elif isinstance(ce, UnitBlock): + unit_block.add_unit_block(ce) elif isinstance(ce, list): for c in ce: PuppetParser.__process_unitblock_component(c, unit_block) @staticmethod - def __process_codeelement( - codeelement: puppetmodel.CodeElement, path: str, code: List[str] + def __get_code(ce: puppetmodel.CodeElement, code: List[str]) -> str: + if ce.line == ce.end_line: + res = code[ce.line - 1][max(0, ce.col - 1) : ce.end_col - 1] + else: + res = code[ce.line - 1][max(0, ce.col - 1) :] + + for line in range(ce.line, ce.end_line - 1): + res += code[line] + + if ce.line != ce.end_line: + res += code[ce.end_line - 1][: ce.end_col - 1] + + return res + + @staticmethod + def __get_info(ce: puppetmodel.CodeElement, code: List[str]) -> ElementInfo: + return ElementInfo( + ce.line, ce.col, ce.end_line, ce.end_col, PuppetParser.__get_code(ce, code) + ) + + @staticmethod + def __process_string_value( + codeelement: puppetmodel.Value[str], path: str, code: List[str] ): - def get_code(ce: puppetmodel.CodeElement): - if ce.line == ce.end_line: - res = code[ce.line - 1][max(0, ce.col - 1) : ce.end_col - 1] - else: - res = code[ce.line - 1] - - for line in range(ce.line, ce.end_line - 1): - res += code[line] - - if ce.line != ce.end_line: - res += code[ce.end_line - 1][: ce.end_col - 1] - - return res - - def process_hash_value( - name: str, temp_value: Any - ) -> Tuple[str, Dict[str, Any]]: - if "[" in name and "]" in name: - start = name.find("[") + 1 - end = name.find("]") - key_name = name[start:end] - name_without_key = name[: start - 1] + name[end + 1 :] - n, d = process_hash_value(name_without_key, temp_value) - if d == {}: - d[key_name] = temp_value - return n, d + def fix_info(e: CodeElement, line: int, col: int): + e.line = line + e.end_line = line + e.column = e.column + col + e.end_column = e.end_column + col + for _, value in e.__dict__.items(): + if isinstance(value, CodeElement): + fix_info(value, line, col) + + interpolation = re.split(r"\$\{(.*?)\}", codeelement.value) + if len(interpolation) == 1: + return String(codeelement.value, PuppetParser.__get_info(codeelement, code)) + + elements: List[Expr] = [] + info = PuppetParser.__get_info(codeelement, code) + current_col = info.column + 1 # quote + current_line = info.line + + for i in range(len(interpolation)): + if i % 2 == 1: + current_col += 2 + element, _ = parse_puppet(interpolation[i]) + assert len(element) == 1 + expr = PuppetParser.__process_codeelement( + element[0], + path, + interpolation[i].split("\n"), + ) + # it starts at 1 + fix_info(expr, current_line, current_col - 1) + assert isinstance(expr, Expr) + elements.append(expr) + current_col += len(interpolation[i]) + 1 + elif interpolation[i] != "": + expr = String(interpolation[i], info) + + if interpolation[i].count("\n") > 0: + expr.line, expr.column = current_line, current_col + current_line += interpolation[i].count("\n") + current_col = len(interpolation[i].split("\n")[-1]) + expr.end_line, expr.end_column = current_line, current_col else: - new_d: Dict[str, Any] = {} - new_d[key_name] = d - return n, new_d + expr.line, expr.end_line = current_line, current_line + expr.column = current_col + expr.end_column = current_col + len(interpolation[i]) + current_col += len(interpolation[i]) + + elements.append(expr) + + if len(elements) == 1: + return elements[0] + + result = Sum( + ElementInfo( + info.line, + info.column, + elements[1].end_line, + elements[1].end_column, + PuppetParser.__get_code(codeelement, code), + ), + elements[0], + elements[1], + ) + for i in range(2, len(elements)): + result = Sum( + ElementInfo( + result.line, + result.column, + elements[i].end_line, + elements[i].end_column, + result.code, + ), + result, + elements[i], + ) + # quote + result.end_column += 1 + + return result + + @staticmethod + def __process_value( + codeelement: puppetmodel.Value[Any], path: str, code: List[str] + ) -> Expr: + if isinstance(codeelement, puppetmodel.Hash): + res_dict: Dict[Expr, Expr] = {} + + for key, value in codeelement.value.items(): + key = PuppetParser.__process_codeelement(key, path, code) + value = PuppetParser.__process_codeelement(value, path, code) + assert isinstance(key, Expr) + assert isinstance(value, Expr) + res_dict[key] = value + + return Hash(res_dict, PuppetParser.__get_info(codeelement, code)) + elif isinstance(codeelement, puppetmodel.Array): + res_list: List[Expr] = [] + for value in codeelement.value: + value = PuppetParser.__process_codeelement(value, path, code) + assert isinstance(value, Expr) + res_list.append(value) + + return Array(res_list, PuppetParser.__get_info(codeelement, code)) + elif isinstance(codeelement, puppetmodel.Id): + if codeelement.value.startswith("$"): + return VariableReference( + codeelement.value[1:], PuppetParser.__get_info(codeelement, code) + ) + return VariableReference( + codeelement.value, PuppetParser.__get_info(codeelement, code) + ) + elif isinstance(codeelement.value, str): + return PuppetParser.__process_string_value(codeelement, path, code) + elif isinstance(codeelement.value, bool): + return Boolean( + codeelement.value, PuppetParser.__get_info(codeelement, code) + ) + elif isinstance(codeelement.value, int): + return Integer( + codeelement.value, PuppetParser.__get_info(codeelement, code) + ) + elif isinstance(codeelement.value, float): + return Float(codeelement.value, PuppetParser.__get_info(codeelement, code)) + elif codeelement.value is None: + return Undef(info=PuppetParser.__get_info(codeelement, code)) + else: + return Null(info=PuppetParser.__get_info(codeelement, code)) + + @staticmethod + def __process_string( + codeelement: puppetmodel.CodeElement | None, code: List[str] + ) -> str: + if codeelement is None: + return "" + elif isinstance(codeelement, puppetmodel.Value) and isinstance(codeelement.value, str): # type: ignore + return codeelement.value + elif isinstance(codeelement, puppetmodel.ResourceCollector): + return PuppetParser.__get_code(codeelement, code) + + raise ValueError(f"Unsupported code element: {codeelement} ({type(codeelement)})") # type: ignore + + @staticmethod + def __process_expr( + codeelement: puppetmodel.CodeElement, path: str, code: List[str] + ) -> Expr: + expr = PuppetParser.__process_codeelement(codeelement, path, code) + assert isinstance(expr, Expr) + return expr + + @staticmethod + def __process_conditional( + codeelement: puppetmodel.If | puppetmodel.Unless, path: str, code: List[str] + ) -> ConditionalStatement: + if codeelement.condition is not None: + condition = PuppetParser.__process_codeelement( + codeelement.condition, path, code + ) + assert isinstance(condition, Expr) + else: + condition = Null() + + condition_statement = ConditionalStatement( + condition, ConditionalStatement.ConditionType.IF + ) + condition_statement.line, condition_statement.column = ( + codeelement.line, + codeelement.col, + ) + condition_statement.end_line, condition_statement.end_column = ( + codeelement.end_line, + codeelement.end_col, + ) + + for statement in codeelement.block: + ce = PuppetParser.__process_codeelement(statement, path, code) + condition_statement.add_statement(ce) + + if codeelement.elseblock is not None: + else_statement = PuppetParser.__process_codeelement( + codeelement.elseblock, path, code + ) + assert isinstance(else_statement, ConditionalStatement) + condition_statement.else_statement = else_statement + + return condition_statement + + @staticmethod + def __process_dependency( + codeelement: puppetmodel.Include | puppetmodel.Require | puppetmodel.Contain, + path: str, + code: List[str], + ) -> Dependency: + if isinstance(codeelement, puppetmodel.Include): + deps = codeelement.inc + elif isinstance(codeelement, puppetmodel.Require): + deps = codeelement.req + else: + deps = codeelement.cont + + dependencies: List[str] = [] + for dep in deps: + d = PuppetParser.__process_string(dep, code) + dependencies.append(d) + + d = Dependency(dependencies) + d.line, d.column = codeelement.line, codeelement.col + d.code = PuppetParser.__get_code(codeelement, code) + return d + + @staticmethod + def __process_case_statement( + codeelement: puppetmodel.Case, path: str, code: List[str] + ) -> ConditionalStatement: + control = PuppetParser.__process_codeelement(codeelement.control, path, code) + assert isinstance(control, Expr) + + conditional_statements: List[ConditionalStatement] = [] + for match in codeelement.matches: + condition: Expr = Null() + + for expression in match.expressions: + right = PuppetParser.__process_codeelement(expression, path, code) + assert isinstance(right, Expr) + + if not isinstance(right, String) or right.value != "default": + if condition == Null(): + condition = Equal( + ElementInfo.from_code_element(right), control, right + ) + else: + condition = Or( + ElementInfo.from_code_element(control), + condition, + Equal(ElementInfo.from_code_element(right), control, right), + ) + + if condition == Null(): + conditional_statement = ConditionalStatement( + Null(), ConditionalStatement.ConditionType.SWITCH, True + ) else: - return name, {} + conditional_statement = ConditionalStatement( + condition, + ConditionalStatement.ConditionType.SWITCH, + ) - if isinstance(codeelement, puppetmodel.Value): - if isinstance(codeelement, puppetmodel.Hash): - res = {} + conditional_statement.line, conditional_statement.column = ( + match.line, + match.col, + ) + conditional_statement.end_line, conditional_statement.end_column = ( + match.end_line, + match.end_col, + ) + conditional_statement.code = PuppetParser.__get_code(match, code) - for key, value in codeelement.value.items(): - res[PuppetParser.__process_codeelement(key, path, code)] = ( - PuppetParser.__process_codeelement(value, path, code) - ) + for statement in match.block: + ce = PuppetParser.__process_codeelement(statement, path, code) + conditional_statement.add_statement(ce) - return res - elif isinstance(codeelement, puppetmodel.Array): - return str( - PuppetParser.__process_codeelement(codeelement.value, path, code) + conditional_statements.append(conditional_statement) + + for i in range(1, len(conditional_statements)): + conditional_statements[i - 1].else_statement = conditional_statements[i] + conditional_statements[0].is_top = True + + return conditional_statements[0] + + @staticmethod + def __process_selector( + codeelement: puppetmodel.Selector, path: str, code: List[str] + ): + control = PuppetParser.__process_codeelement(codeelement.control, path, code) + assert isinstance(control, Expr) + + conditional_statements: List[ConditionalStatement] = [] + for key_element, value_element in codeelement.hash.value.items(): + right = PuppetParser.__process_codeelement(key_element, path, code) + assert isinstance(right, Expr) + value = PuppetParser.__process_codeelement(value_element, path, code) + + if isinstance(right, String) and right.value == "default": + conditional_statement = ConditionalStatement( + Null(), ConditionalStatement.ConditionType.SWITCH, True + ) + else: + condition = Equal( + ElementInfo.from_code_element(control), control, right ) - elif codeelement.value is None: - return "" - return str(codeelement.value) + conditional_statement = ConditionalStatement( + condition, + ConditionalStatement.ConditionType.SWITCH, + ) + + conditional_statement.line, conditional_statement.column = ( + key_element.line, + key_element.col, + ) + conditional_statement.end_line, conditional_statement.end_column = ( + value_element.end_line, + value_element.end_col, + ) + conditional_statement.code = PuppetParser.__get_code( + key_element, code + ) + PuppetParser.__get_code(value_element, code) + + conditional_statement.add_statement(value) + conditional_statements.append(conditional_statement) + + for i in range(1, len(conditional_statements)): + conditional_statements[i - 1].else_statement = conditional_statements[i] + conditional_statements[0].is_top = True + + return conditional_statements[0] + + @staticmethod + def __process_operation( + codeelement: puppetmodel.Operation, path: str, code: List[str] + ): + def unary_operation(type: Callable[[ElementInfo, Expr], Expr]) -> Expr: + return type( + PuppetParser.__get_info(codeelement, code), + PuppetParser.__process_expr(codeelement.arguments[0], path, code), + ) + + def binary_operation(type: Callable[[ElementInfo, Expr, Expr], Expr]) -> Expr: + return type( + PuppetParser.__get_info(codeelement, code), + PuppetParser.__process_expr(codeelement.arguments[0], path, code), + PuppetParser.__process_expr(codeelement.arguments[1], path, code), + ) + + if codeelement.operator == "==": + return binary_operation(Equal) + elif codeelement.operator == "!=": + return binary_operation(NotEqual) + elif codeelement.operator == "and": + return binary_operation(And) + elif codeelement.operator == "or": + return binary_operation(Or) + elif codeelement.operator == "!": + return unary_operation(Not) + elif codeelement.operator == "[,]": + # FIXME: Not yet supported + return Null() + elif codeelement.operator == "<": + return binary_operation(LessThan) + elif codeelement.operator == ">": + return binary_operation(GreaterThan) + elif codeelement.operator == "<=": + return binary_operation(LessThanOrEqual) + elif codeelement.operator == ">=": + return binary_operation(GreaterThanOrEqual) + elif codeelement.operator == "~=": + # FIXME: Not yet supported + return Null() + elif codeelement.operator == "!~": + # FIXME: Not yet supported + return Null() + elif codeelement.operator == "in": + return binary_operation(In) + elif codeelement.operator == "-" and len(codeelement.arguments) == 1: + return unary_operation(Minus) + elif codeelement.operator == "-" and len(codeelement.arguments) == 2: + return binary_operation(Subtract) + elif codeelement.operator == "+": + return binary_operation(Sum) + elif codeelement.operator == "/": + return binary_operation(Divide) + elif codeelement.operator == "*" and len(codeelement.arguments) == 1: + # FIXME: Not yet supported + return Null() + elif codeelement.operator == "*" and len(codeelement.arguments) == 2: + return binary_operation(Multiply) + elif codeelement.operator == "%": + return binary_operation(Modulo) + elif codeelement.operator == ">>": + return binary_operation(RightShift) + elif codeelement.operator == "<<": + return binary_operation(LeftShift) + elif codeelement.operator in ["=~", "!~"]: + # TODO + return Null() + elif codeelement.operator == "[]" and len(codeelement.arguments) == 2: + return binary_operation(Access) + + raise ValueError(f"Unsupported operation: {codeelement.operator}") + + @staticmethod + def __process_unitblock( + codeelement: ( + puppetmodel.PuppetClass + | puppetmodel.Node + | puppetmodel.Function + | puppetmodel.ResourceDeclaration + ), + path: str, + code: List[str], + type: UnitBlockType, + ) -> UnitBlock: + unit_block: UnitBlock = UnitBlock(codeelement.name, type) + + if isinstance(codeelement, puppetmodel.Function): + block = codeelement.body + else: + block = codeelement.block + + for ce in list( + map( + lambda ce: PuppetParser.__process_codeelement(ce, path, code), + block, + ) + ): + PuppetParser.__process_unitblock_component(ce, unit_block) + + unit_block.line, unit_block.column = codeelement.line, codeelement.col + unit_block.code = PuppetParser.__get_code(codeelement, code) + return unit_block + + @staticmethod + def __process_codeelement( + codeelement: puppetmodel.CodeElement, path: str, code: List[str] + ) -> CodeElement: + if isinstance(codeelement, puppetmodel.Value): + return PuppetParser.__process_value(codeelement, path, code) # type: ignore elif isinstance(codeelement, puppetmodel.Attribute): - name = PuppetParser.__process_codeelement(codeelement.key, path, code) - temp_value = PuppetParser.__process_codeelement( - codeelement.value, path, code + name = PuppetParser.__process_string(codeelement.key, code) + if name.startswith("$"): + name = name[1:] + value = PuppetParser.__process_codeelement(codeelement.value, path, code) + assert isinstance(value, Expr) + # This allows to have strings without the quotes in the attributes + if isinstance( + codeelement.value, puppetmodel.Id + ) and not codeelement.value.value.startswith("$"): + value = String( + codeelement.value.value, + PuppetParser.__get_info(codeelement.value, code), + ) + + attribute = Attribute( + name, value, PuppetParser.__get_info(codeelement, code) ) - value = "" if temp_value == "undef" else temp_value - has_variable = not isinstance(value, str) or value.startswith("$") - attribute = Attribute(name, value, has_variable) - attribute.line, attribute.column = codeelement.line, codeelement.col - attribute.code = get_code(codeelement) return attribute elif isinstance(codeelement, puppetmodel.Resource): - resource: AtomicUnit = AtomicUnit( - PuppetParser.__process_codeelement(codeelement.title, path, code), - PuppetParser.__process_codeelement(codeelement.type, path, code), - ) - for attr in codeelement.attributes: - resource.add_attribute( - PuppetParser.__process_codeelement(attr, path, code) + if isinstance(codeelement.title, puppetmodel.Array): + titles = codeelement.title.value + re_aux = UnitBlock("resource_expression", UnitBlockType.block) + re_aux.line, re_aux.column = codeelement.line, codeelement.col + re_aux.code = PuppetParser.__get_code(codeelement, code) + re_aux.end_line, re_aux.end_column = ( + codeelement.end_line, + codeelement.end_col, ) - resource.line, resource.column = codeelement.line, codeelement.col - resource.code = get_code(codeelement) - return resource + else: + titles = [codeelement.title] + re_aux = None + + for title in titles: + if title is None: + title = Null() + else: + title = PuppetParser.__process_codeelement(title, path, code) + assert isinstance(title, Expr) + resource: AtomicUnit = AtomicUnit( + title, + PuppetParser.__process_string(codeelement.type, code), + ) + for attr in codeelement.attributes: + attr = PuppetParser.__process_codeelement(attr, path, code) + assert isinstance(attr, Attribute) + resource.add_attribute(attr) + resource.line, resource.column = codeelement.line, codeelement.col + resource.code = PuppetParser.__get_code(codeelement, code) + + if re_aux is not None: + re_aux.add_atomic_unit(resource) + else: + return resource + + return re_aux # type: ignore elif isinstance(codeelement, puppetmodel.ClassAsResource): - resource: AtomicUnit = AtomicUnit( - PuppetParser.__process_codeelement(codeelement.title, path, code), - "class", - ) + assert codeelement.title is not None + title = PuppetParser.__process_codeelement(codeelement.title, path, code) + assert isinstance(title, Expr) + resource: AtomicUnit = AtomicUnit(title, "class") for attr in codeelement.attributes: - resource.add_attribute( - PuppetParser.__process_codeelement(attr, path, code) - ) + attr = PuppetParser.__process_codeelement(attr, path, code) + assert isinstance(attr, Attribute) + resource.add_attribute(attr) resource.line, resource.column = codeelement.line, codeelement.col - resource.code = get_code(codeelement) + resource.code = PuppetParser.__get_code(codeelement, code) return resource elif isinstance(codeelement, puppetmodel.ResourceDeclaration): - unit_block: UnitBlock = UnitBlock( - PuppetParser.__process_codeelement(codeelement.name, path, code), - UnitBlockType.block, + unit_block: UnitBlock = PuppetParser.__process_unitblock( + codeelement, path, code, UnitBlockType.definition ) unit_block.path = path - if codeelement.block is not None: - for ce in list( - map( - lambda ce: PuppetParser.__process_codeelement(ce, path, code), - codeelement.block, - ) - ): - PuppetParser.__process_unitblock_component(ce, unit_block) - for p in codeelement.parameters: - unit_block.add_attribute( - PuppetParser.__process_codeelement(p, path, code) - ) - - unit_block.line, unit_block.column = codeelement.line, codeelement.col - unit_block.code = get_code(codeelement) + attr = PuppetParser.__process_codeelement(p, path, code) + assert isinstance(attr, Attribute) + unit_block.add_attribute(attr) return unit_block elif isinstance(codeelement, puppetmodel.Parameter): - # FIXME Parameters are not yet supported - name = PuppetParser.__process_codeelement(codeelement.name, path, code) if codeelement.default is not None: - temp_value = PuppetParser.__process_codeelement( + value = PuppetParser.__process_codeelement( codeelement.default, path, code ) - value = "" if temp_value == "undef" else temp_value + assert isinstance(value, Expr) else: - value = None - has_variable = ( - not isinstance(value, str) - or temp_value.startswith("$") - or codeelement.default is None + value = Undef() + + name = ( + codeelement.name[1:] + if codeelement.name.startswith("$") + else codeelement.name ) - attribute = Attribute(name, value, has_variable) - attribute.line, attribute.column = codeelement.line, codeelement.col - attribute.code = get_code(codeelement) + attribute = Attribute( + name, value, PuppetParser.__get_info(codeelement, code) + ) + return attribute elif isinstance(codeelement, puppetmodel.Assignment): - name = PuppetParser.__process_codeelement(codeelement.name, path, code) - temp_value = PuppetParser.__process_codeelement( - codeelement.value, path, code + name = PuppetParser.__get_code(codeelement.name, code) + if name.startswith("$"): + name = name[1:] + value = PuppetParser.__process_codeelement(codeelement.value, path, code) + assert isinstance(value, Expr) + + variable: Variable = Variable( + name, value, PuppetParser.__get_info(codeelement, code) ) - if "[" in name and "]" in name: - name, temp_value = process_hash_value(name, temp_value) - if not isinstance(temp_value, dict): - if codeelement.value is not None: - value = "" if temp_value == "undef" else temp_value - else: - value = None - has_variable = not isinstance(value, str) or value.startswith("$") - variable: Variable = Variable(name, value, has_variable) - variable.line, variable.column = codeelement.line, codeelement.col - variable.code = get_code(codeelement) - return variable - else: - variable: Variable = Variable(name, None, False) - variable.line, variable.column = codeelement.line, codeelement.col - variable.code = get_code(codeelement) - for key, value in temp_value.items(): - variable.keyvalues.append( - PuppetParser.__process_codeelement( - puppetmodel.Assignment( - codeelement.line, - codeelement.col, - codeelement.end_line, - codeelement.end_col, - key, - value, - ), - path, - code, - ) - ) - - return variable + return variable elif isinstance(codeelement, puppetmodel.PuppetClass): # FIXME there are components of the class that are not considered - unit_block: UnitBlock = UnitBlock( - PuppetParser.__process_codeelement(codeelement.name, path, code), - UnitBlockType.block, + unit_block = PuppetParser.__process_unitblock( + codeelement, path, code, UnitBlockType.definition ) unit_block.path = path - if codeelement.block is not None: - for ce in list( - map( - lambda ce: PuppetParser.__process_codeelement(ce, path, code), - codeelement.block, - ) - ): - PuppetParser.__process_unitblock_component(ce, unit_block) - for p in codeelement.parameters: - unit_block.add_attribute( - PuppetParser.__process_codeelement(p, path, code) - ) + attr = PuppetParser.__process_codeelement(p, path, code) + assert isinstance(attr, Attribute) + unit_block.add_attribute(attr) - unit_block.line, unit_block.column = codeelement.line, codeelement.col - unit_block.code = get_code(codeelement) return unit_block elif isinstance(codeelement, puppetmodel.Node): - # FIXME Nodes are not yet supported - if codeelement.block is not None: - return list( - map( - lambda ce: PuppetParser.__process_codeelement(ce, path, code), - codeelement.block, - ) - ) - else: - return [] + unit_block = PuppetParser.__process_unitblock( + codeelement, path, code, UnitBlockType.block + ) + unit_block.name = "node" + return unit_block elif isinstance(codeelement, puppetmodel.Operation): - if len(codeelement.arguments) == 1: - return codeelement.operator + PuppetParser.__process_codeelement( - codeelement.arguments[0], path, code - ) - elif codeelement.operator == "[]": - return ( - PuppetParser.__process_codeelement( - codeelement.arguments[0], path, code - ) - + "[" - + ",".join( - PuppetParser.__process_codeelement( - codeelement.arguments[1], path, code - ) - ) - + "]" - ) - elif len(codeelement.arguments) == 2: - return ( - str( - PuppetParser.__process_codeelement( - codeelement.arguments[0], path, code - ) - ) - + codeelement.operator - + str( - PuppetParser.__process_codeelement( - codeelement.arguments[1], path, code - ) - ) - ) - elif codeelement.operator == "[,]": - return ( - PuppetParser.__process_codeelement( - codeelement.arguments[0], path, code - ) - + "[" - + PuppetParser.__process_codeelement( - codeelement.arguments[1], path, code - ) - + "," - + PuppetParser.__process_codeelement( - codeelement.arguments[2], path, code - ) - + "]" - ) + return PuppetParser.__process_operation(codeelement, path, code) elif isinstance(codeelement, puppetmodel.Lambda): # FIXME Lambdas are not yet supported - if codeelement.block is not None: - args = [] - for arg in codeelement.parameters: - attr = PuppetParser.__process_codeelement(arg, path, code) - variable = Variable(attr.name, "", True) - variable.line = arg.line - variable.column = arg.col - args.append(Variable(variable)) - return ( - list( - map( - lambda ce: PuppetParser.__process_codeelement( - ce, path, code - ), - codeelement.block, - ) - ) - + args - ) - else: - return [] + return Null() elif isinstance(codeelement, puppetmodel.FunctionCall): - # FIXME Function calls are not yet supported - res = PuppetParser.__process_codeelement(codeelement.name, path, code) + "(" + name = PuppetParser.__process_string(codeelement.name, code) + + args: List[Expr] = [] for arg in codeelement.arguments: - res += repr(PuppetParser.__process_codeelement(arg, path, code)) + "," - res = res[:-1] - res += ")" - lamb = PuppetParser.__process_codeelement(codeelement.lamb, path, code) - if lamb != "": - return [res] + lamb - else: - return res + arg = PuppetParser.__process_codeelement(arg, path, code) + assert isinstance(arg, Expr) + args.append(arg) + + if codeelement.lamb is not None: + lamb = PuppetParser.__process_codeelement(codeelement.lamb, path, code) + assert isinstance(lamb, Expr) + args.append(lamb) + + return FunctionCall(name, args, PuppetParser.__get_info(codeelement, code)) elif isinstance(codeelement, puppetmodel.If): - condition = PuppetParser.__process_codeelement( - codeelement.condition, path, code - ) - condition = ConditionalStatement( - condition, ConditionalStatement.ConditionType.IF - ) - body = list( - map( - lambda ce: PuppetParser.__process_codeelement(ce, path, code), - codeelement.block, - ) - ) - for statement in body: - # FIXME: this should probably be more general (e.g. recursive lists) - if isinstance(statement, list): - for s in statement: - condition.add_statement(s) - # Avoids unsupported concepts - elif statement is not None: - condition.add_statement(statement) - - if codeelement.elseblock is not None: - condition.else_statement = PuppetParser.__process_codeelement( - codeelement.elseblock, path, code - ) - return condition + conditional = PuppetParser.__process_conditional(codeelement, path, code) + conditional.is_top = True + return conditional elif isinstance(codeelement, puppetmodel.Unless): - # FIXME Unless is not yet supported - res = list( - map( - lambda ce: PuppetParser.__process_codeelement(ce, path, code), - codeelement.block, - ) + conditional = PuppetParser.__process_conditional(codeelement, path, code) + conditional.condition = Not( + ElementInfo.from_code_element(conditional.condition), + conditional.condition, ) - if codeelement.elseblock is not None: - res += PuppetParser.__process_codeelement( - codeelement.elseblock, path, code - ) - return res - elif isinstance(codeelement, puppetmodel.Include): - dependencies = [] - for inc in codeelement.inc: - d = Dependency(PuppetParser.__process_codeelement(inc, path, code)) - d.line, d.column = codeelement.line, codeelement.col - d.code = get_code(codeelement) - dependencies.append(d) - return dependencies - elif isinstance(codeelement, puppetmodel.Require): - dependencies = [] - for req in codeelement.req: - d = Dependency(PuppetParser.__process_codeelement(req, path, code)) - d.line, d.column = codeelement.line, codeelement.col - d.code = get_code(codeelement) - dependencies.append(d) - return dependencies - elif isinstance(codeelement, puppetmodel.Contain): - dependencies = [] - for cont in codeelement.cont: - d = Dependency(PuppetParser.__process_codeelement(cont, path, code)) - d.line, d.column = codeelement.line, codeelement.col - d.code = get_code(codeelement) - dependencies.append(d) - return dependencies + conditional.is_top = True + return conditional + elif isinstance( + codeelement, (puppetmodel.Include, puppetmodel.Require, puppetmodel.Contain) + ): + return PuppetParser.__process_dependency(codeelement, path, code) elif isinstance( codeelement, (puppetmodel.Debug, puppetmodel.Fail, puppetmodel.Realize, puppetmodel.Tag), ): # FIXME Ignored unsupported concepts - pass + return Null() elif isinstance(codeelement, puppetmodel.Match): - # FIXME Matches are not yet supported - return [ - list( - map( - lambda ce: PuppetParser.__process_codeelement(ce, path, code), - codeelement.block, - ) - ) - ] + raise ValueError("Matches should only appear in case statements") elif isinstance(codeelement, puppetmodel.Case): - control = PuppetParser.__process_codeelement( - codeelement.control, path, code - ) - conditions = [] - - for match in codeelement.matches: - expressions = PuppetParser.__process_codeelement( - match.expressions, path, code - ) - for expression in expressions: - if expression != "default": - condition = ConditionalStatement( - control + "==" + expression, - ConditionalStatement.ConditionType.SWITCH, - False, - ) - condition.line, condition.column = match.line, match.col - condition.code = get_code(match) - conditions.append(condition) - else: - condition = ConditionalStatement( - "", ConditionalStatement.ConditionType.SWITCH, True - ) - condition.line, condition.column = match.line, match.col - condition.code = get_code(match) - conditions.append(condition) - - for i in range(1, len(conditions)): - conditions[i - 1].else_statement = conditions[i] - - return [conditions[0]] + list( - map( - lambda ce: PuppetParser.__process_codeelement(ce, path, code), - codeelement.matches, - ) - ) + return PuppetParser.__process_case_statement(codeelement, path, code) elif isinstance(codeelement, puppetmodel.Selector): - control = PuppetParser.__process_codeelement( - codeelement.control, path, code - ) - conditions = [] - - for key_element, value_element in codeelement.hash.value.items(): - key = PuppetParser.__process_codeelement(key_element, path, code) - value = PuppetParser.__process_codeelement(value_element, path, code) - - if key != "default": - condition = ConditionalStatement( - control + "==" + key, - ConditionalStatement.ConditionType.SWITCH, - False, - ) - condition.line, condition.column = key_element.line, key_element.col - # HACK: the get_code function should be changed to receive a range - key_element.end_line, key_element.end_col = ( - value_element.end_line, - value_element.end_col, - ) - condition.code = get_code(key_element) - conditions.append(condition) - else: - condition = ConditionalStatement( - "", ConditionalStatement.ConditionType.SWITCH, True - ) - condition.line, condition.column = key_element.line, key_element.col - key_element.end_line, key_element.end_col = ( - value_element.end_line, - value_element.end_col, - ) - condition.code = get_code(key_element) - conditions.append(condition) - - for i in range(1, len(conditions)): - conditions[i - 1].else_statement = conditions[i] - - return conditions[0] + return PuppetParser.__process_selector(codeelement, path, code) elif isinstance(codeelement, puppetmodel.Reference): - res = codeelement.type + "[" - for r in codeelement.references: - temp = PuppetParser.__process_codeelement(r, path, code) - res += "" if temp is None else temp - res += "]" - return res + # FIXME: Reference not yet supported + return Null() elif isinstance(codeelement, puppetmodel.Function): - # FIXME Functions definitions are not yet supported - return list( - map( - lambda ce: PuppetParser.__process_codeelement(ce, path, code), - codeelement.body, - ) + unit_block: UnitBlock = PuppetParser.__process_unitblock( + codeelement, path, code, UnitBlockType.function ) + + for p in codeelement.parameters: + attr = PuppetParser.__process_codeelement(p, path, code) + assert isinstance(attr, Attribute) + unit_block.add_attribute(attr) + + return unit_block elif isinstance(codeelement, puppetmodel.ResourceCollector): - res = codeelement.resource_type + "<|" - res += ( - PuppetParser.__process_codeelement(codeelement.search, path, code) - + "|>" - ) - return res + # FIXME: Resource collectors not yet supported + return Null() elif isinstance(codeelement, puppetmodel.ResourceExpression): - resources = [] - resources.append( - PuppetParser.__process_codeelement(codeelement.default, path, code) + unit_block = UnitBlock("resource_expression", UnitBlockType.block) + unit_block.line, unit_block.column = codeelement.line, codeelement.col + unit_block.code = PuppetParser.__get_code(codeelement, code) + unit_block.end_line, unit_block.end_column = ( + codeelement.end_line, + codeelement.end_col, ) - for resource in codeelement.resources: - resources.append( - PuppetParser.__process_codeelement(resource, path, code) + + default_attributes: Dict[str, Attribute] = {} + if codeelement.default is not None: + default = PuppetParser.__process_codeelement( + codeelement.default, path, code ) - return resources + assert isinstance(default, AtomicUnit) + for attr in default.attributes: + default_attributes[attr.name] = attr + + for rsc in codeelement.resources: + resources: List[puppetmodel.Resource | puppetmodel.ClassAsResource] = [] + if isinstance(rsc.title, puppetmodel.Array): + for t in rsc.title.value: + r = copy.deepcopy(rsc) + assert isinstance(t, puppetmodel.Value) + r.title = t + resources.append(r) + else: + resources.append(rsc) + + for rsc in resources: + au = PuppetParser.__process_codeelement(rsc, path, code) + assert isinstance(au, AtomicUnit) + attrs = list(map(lambda a: a.name, au.attributes)) + # Add default attributes + for name, attr in default_attributes.items(): + if name not in attrs: + au.add_attribute(attr) + unit_block.add_atomic_unit(au) + + return unit_block elif isinstance(codeelement, puppetmodel.Chaining): - # FIXME Chaining not yet supported - res = [] - op1 = PuppetParser.__process_codeelement(codeelement.op1, path, code) - op2 = PuppetParser.__process_codeelement(codeelement.op2, path, code) - if isinstance(op1, list): - res += op1 - else: - res.append(op1) - if isinstance(op2, list): - res += op2 - else: - res.append(op2) - return res - elif isinstance(codeelement, list): - return list( - map( - lambda ce: PuppetParser.__process_codeelement(ce, path, code), - codeelement, - ) + # FIXME: This is an HACK to temporarily support chaining + unit_block = UnitBlock("Chaining", UnitBlockType.block) + unit_block.line, unit_block.column = codeelement.line, codeelement.col + unit_block.end_line, unit_block.end_column = ( + codeelement.end_line, + codeelement.end_col, ) - elif codeelement is None: - return "" - else: - return codeelement + unit_block.code = PuppetParser.__get_code(codeelement, code) + + left = PuppetParser.__process_codeelement(codeelement.op1, path, code) + right = PuppetParser.__process_codeelement(codeelement.op2, path, code) + + PuppetParser.__process_unitblock_component(left, unit_block) + PuppetParser.__process_unitblock_component(right, unit_block) + + return unit_block + + raise ValueError( + f"Unsupported code element: {codeelement} ({type(codeelement)})" + ) def parse_module(self, path: str) -> Module: res: Module = Module(os.path.basename(os.path.normpath(path)), path) @@ -561,11 +748,14 @@ def parse_module(self, path: str) -> Module: for name in files: name_split = name.split(".") if len(name_split) == 2 and name_split[-1] == "pp": - res.add_block(self.parse_file(os.path.join(root, name), "")) - + block = self.parse_file( + os.path.join(root, name), UnitBlockType.script + ) + assert block is not None + res.add_block(block) return res - def parse_file(self, path: str, type: UnitBlockType) -> UnitBlock: + def parse_file(self, path: str, type: UnitBlockType) -> UnitBlock | None: unit_block: UnitBlock = UnitBlock(os.path.basename(path), UnitBlockType.script) unit_block.path = path @@ -582,13 +772,15 @@ def parse_file(self, path: str, type: UnitBlockType) -> UnitBlock: comment.code = "".join(code[c.line - 1 : c.end_line]) unit_block.add_comment(comment) - PuppetParser.__process_unitblock_component( - PuppetParser.__process_codeelement(parsed_script, path, code), - unit_block, - ) + for ce in parsed_script: + PuppetParser.__process_unitblock_component( + PuppetParser.__process_codeelement(ce, path, code), + unit_block, + ) except Exception: traceback.print_exc() throw_exception(EXCEPTIONS["PUPPET_COULD_NOT_PARSE"], path) + return None return unit_block def parse_folder(self, path: str) -> Project: @@ -606,7 +798,9 @@ def parse_folder(self, path: str) -> Project: for f in os.scandir(path): name_split = f.name.split(".") if f.is_file() and len(name_split) == 2 and name_split[-1] == "pp": - res.add_block(self.parse_file(f.path, "")) + block = self.parse_file(f.path, UnitBlockType.script) + assert block is not None + res.add_block(block) subfolders = [ f.path for f in os.scandir(f"{path}") if f.is_dir() and not f.is_symlink() diff --git a/glitch/parsers/ripper_parser.py b/glitch/parsers/ripper_parser.py index ce318197..5d2fb7ac 100644 --- a/glitch/parsers/ripper_parser.py +++ b/glitch/parsers/ripper_parser.py @@ -1,9 +1,10 @@ # pyright: reportUnusedFunction=false, reportUnusedVariable=false from ply.lex import lex, LexToken from ply.yacc import yacc, YaccProduction +from typing import Tuple, List -def parser_yacc(script_ast: str): +def parser_yacc(script_ast: str) -> Tuple[List[Tuple[str, int]], List[Tuple[str, int]]]: tokens = ( "LPAREN", "RPAREN", @@ -131,4 +132,4 @@ def p_error(p: YaccProduction) -> None: # Build the parser parser = yacc() - return parser.parse(script_ast) + return parser.parse(script_ast, lexer=lexer.clone()) diff --git a/glitch/parsers/terraform.py b/glitch/parsers/terraform.py index f76fe318..9dfa51e4 100644 --- a/glitch/parsers/terraform.py +++ b/glitch/parsers/terraform.py @@ -1,220 +1,396 @@ -# type: ignore -# TODO: The file needs a refactor so the types make sense +# type: ignore (#TODO) import os -import re -import hcl2 +from hcl2.parser import hcl2 import glitch.parsers.parser as p from glitch.exceptions import EXCEPTIONS, throw_exception from glitch.repr.inter import * -from typing import Sequence, List, Dict, Any +from typing import List, Any +from lark.tree import Meta, Tree +from lark.lexer import Token +from lark.visitors import Transformer, v_args, Discard + + +class GLITCHTransformer(Transformer): + """Takes a syntax tree generated by the parser and + transforms it to a dict. + """ + + def __init__(self, code: List[str]): + self.code = code + self.comments = [] + super().__init__() + + def new_line_or_comment(self, args: List) -> List: + for arg in args: + if isinstance(arg, Token): + if arg.value.startswith(("//", "#", "/*")): + comment = Comment(arg.value) + comment.line, comment.column = arg.line, arg.column + comment.end_line, comment.end_column = arg.end_line, arg.end_column + self.comments.append(comment) + return Discard + + def new_line_and_or_comma(self, args: List) -> List: + return Discard + + def __get_element_code( + self, start_line: int, start_col: int, end_line: int, end_col: int + ) -> str: + if start_line == end_line: + res = self.code[start_line - 1][max(0, start_col - 1) : end_col - 1] + else: + res = self.code[start_line - 1][max(0, start_col - 1) :] + + for line in range(start_line, end_line - 1): + res += self.code[line] + + if start_line != end_line: + res += self.code[end_line - 1][: end_col - 1] -class TerraformParser(p.Parser): - @staticmethod - def __get_element_code(start_line: int, end_line: int, code: List[str]) -> str: - lines = code[start_line - 1 : end_line] - res = "" - for line in lines: - res += line return res - def parse_keyvalues( - self, - unit_block: UnitBlock, - keyvalues: Dict[Any, Any], - code: List[str], - type: str, - ) -> List[KeyValue]: - def create_keyvalue(start_line: int, end_line: int, name: str, value: str): - has_variable = ( - ("${" in f"{value}") and ("}" in f"{value}") if value != None else False # type: ignore - ) - pattern = r"^[+-]?\d+(\.\d+)?$" - if has_variable and re.match(pattern, re.sub(r"^\${(.*)}$", r"\1", value)): - value = re.sub(r"^\${(.*)}$", r"\1", value) - has_variable = False - if value == "null": - value = "" - - if isinstance(value, int): - value = str(value) - - if type == "attribute": - keyvalue = Attribute(str(name), value, has_variable) - else: - keyvalue = Variable(str(name), value, has_variable) - - keyvalue.line = start_line - keyvalue.code = TerraformParser.__get_element_code( - start_line, end_line, code + def __get_element_info(self, meta: Meta | Token) -> ElementInfo: + return ElementInfo( + meta.line, + meta.column, + meta.end_line, + meta.end_column, + self.__get_element_code( + meta.line, + meta.column, + meta.end_line, + meta.end_column, + ), + ) + + def __get_element_info_from_tokens(self, start: Token, end: Token) -> ElementInfo: + return ElementInfo( + start.line, + start.column, + end.end_line, + end.end_column, + self.__get_element_code( + start.line, + start.column, + end.end_line, + end.end_column, + ), + ) + + def __parse_heredoc(self, tree: Tree) -> str: + res = "" + for arg in tree.children: + res += arg.value + return "\n".join(res.split("\n")[1:-1]) + + @v_args(meta=True) + def binary_op(self, meta: Meta, args: List) -> Any: + op_to_ir = { + "+": Sum, + "-": Subtract, + "*": Multiply, + "/": Divide, + "%": Modulo, + "&&": And, + "||": Or, + "==": Equal, + "!=": NotEqual, + ">": GreaterThan, + "<": LessThan, + ">=": GreaterThanOrEqual, + "<=": LessThanOrEqual, + } + + if args[1].children[0].children[0] in op_to_ir: + return op_to_ir[args[1].children[0].children[0]]( + self.__get_element_info_from_tokens(args[0], args[1].children[1]), + args[0], + args[1].children[1], ) - return keyvalue + @v_args(meta=True) + def unary_op(self, meta: Meta, args: List) -> Any: + if args[0] == "-": + return Minus(self.__get_element_info(meta), args[1]) + elif args[0] == "!": + return Not(self.__get_element_info(meta), args[1]) - def process_list(name: str, value: str, start_line: int, end_line: int) -> None: - for i, v in enumerate(value): - if isinstance(v, dict): - k = create_keyvalue(start_line, end_line, name + f"[{i}]", None) # type: ignore - k.keyvalues = self.parse_keyvalues(unit_block, v, code, type) - k_values.append(k) - elif isinstance(v, list): - process_list(name + f"[{i}]", v, start_line, end_line) - else: - k = create_keyvalue(start_line, end_line, name + f"[{i}]", v) - k_values.append(k) - - k_values: List[KeyValue] = [] - for name, keyvalue in keyvalues.items(): - if name == "__start_line__" or name == "__end_line__": - continue - - if isinstance( - keyvalue, dict - ): # Note: local values (variables) can only enter here - value = keyvalue["value"] - if isinstance(value, dict): # (ex: labels = {}) - k = create_keyvalue( - keyvalue["__start_line__"], keyvalue["__end_line__"], name, None # type: ignore - ) - k.keyvalues = self.parse_keyvalues(unit_block, value, code, type) - k_values.append(k) - elif isinstance(value, list): # (ex: x = [1,2,3]) - process_list( - name, - value, - keyvalue["__start_line__"], - keyvalue["__end_line__"], - ) - else: # (ex: x = 'test') - if value == None: # (ex: x = null) - value = "null" - k = create_keyvalue( - keyvalue["__start_line__"], - keyvalue["__end_line__"], - name, - value, - ) - k_values.append(k) - elif isinstance(keyvalue, list) and type == "attribute": # type: ignore - # block (ex: access {} or dynamic setting {}; blocks of attributes; not allowed inside local values (variables)) - try: - for block_attributes in keyvalue: - k = create_keyvalue( - block_attributes["__start_line__"], - block_attributes["__end_line__"], - name, - None, - ) - k.keyvalues = self.parse_keyvalues( - unit_block, block_attributes, code, type - ) - k_values.append(k) - except KeyError: - for block in keyvalue: - for block_name, block_attributes in block.items(): - k = create_keyvalue( - block_attributes["__start_line__"], - block_attributes["__end_line__"], - f"{name}.{block_name}", - None, - ) - k.keyvalues = self.parse_keyvalues( - unit_block, block_attributes, code, type - ) - k_values.append(k) - - return k_values - - def parse_atomic_unit( - self, type: str, unit_block: UnitBlock, dict, code: List[str] - ) -> None: - def create_atomic_unit( - start_line: int, end_line: int, type: str, name: str, code: List[str] - ) -> AtomicUnit: - au = AtomicUnit(name, type) - au.line = start_line - au.code = TerraformParser.__get_element_code(start_line, end_line, code) - return au + @v_args(meta=True) + def get_attr(self, meta: Meta, args: List) -> Any: + return args[0] - def parse_resource() -> None: - for resource_type, resource in dict.items(): - for name, attributes in resource.items(): - au = create_atomic_unit( - attributes["__start_line__"], - attributes["__end_line__"], - f"{type}.{resource_type}", - name, - code, - ) - au.attributes = self.parse_keyvalues( - unit_block, attributes, code, "attribute" + @v_args(meta=True) + def index(self, meta: Meta, args: List) -> Any: + return args[0] + + @v_args(meta=True) + def index_expr_term(self, meta: Meta, args: List) -> Any: + return Access(self.__get_element_info(meta), args[0], args[1]) + + @v_args(meta=True) + def get_attr_expr_term(self, meta: Meta, args: List) -> Any: + return Access(self.__get_element_info(meta), args[0], args[1]) + + @v_args(meta=True) + def int_lit(self, meta: Meta, args: List) -> int: + return Integer(int("".join(args)), self.__get_element_info(meta)) + + @v_args(meta=True) + def float_lit(self, meta: Meta, args: List) -> float: + return Float(float("".join(args)), self.__get_element_info(meta)) + + @v_args(meta=True) + def interpolation_maybe_nested(self, meta: Meta, args: List) -> Any: + return args[0] + + @v_args(meta=True) + def string_with_interpolation(self, meta: Meta, args: List) -> str: + if len(args) == 1: + if isinstance(args[0], Token): + return String( + args[0].value, + self.__get_element_info(meta), + ) + return args[0] + else: + for i in range(len(args)): + if isinstance(args[i], Token): + args[i] = String( + args[i].value, + self.__get_element_info(args[i]), ) - unit_block.add_atomic_unit(au) - - def parse_simple_unit() -> None: - for name, attributes in dict.items(): - au = create_atomic_unit( - attributes["__start_line__"], - attributes["__end_line__"], - type, - name, - code, + + res = Sum( + ElementInfo( + args[0].line, + args[0].column, + args[1].end_line, + args[1].end_column, + self.__get_element_code( + args[0].line, + args[0].column, + args[1].end_line, + args[1].end_column, + ), + ), + args[0], + args[1], + ) + for i in range(2, len(args)): + res = Sum( + ElementInfo( + res.line, + res.column, + args[i].end_line, + args[i].end_column, + self.__get_element_code( + res.line, + res.column, + args[i].end_line, + args[i].end_column, + ), + ), + res, + args[i], ) - au.attributes = self.parse_keyvalues( - unit_block, attributes, code, "attribute" + res.line, res.column = meta.line, meta.column + res.end_line, res.end_column = meta.end_line, meta.end_column + + return res + + @v_args(meta=True) + def expr_term(self, meta: Meta, args: List) -> Expr: + if len(args) == 0: + return Null(self.__get_element_info(meta)) + elif len(args) == 1: + if isinstance(args[0], Tree) and args[0].data == "heredoc_template": + return String( + self.__parse_heredoc(args[0]), + self.__get_element_info(meta), ) - unit_block.add_atomic_unit(au) - - if type in ["resource", "data"]: - parse_resource() - elif type in ["variable", "module", "output"]: - parse_simple_unit() - - def parse_comments( - self, unit_block: UnitBlock, comments: Sequence[str], code: List[str] - ) -> None: - def create_comment(value: str, start_line: int, end_line: int, code: List[str]): - c = Comment(value) - c.line = start_line - c.code = TerraformParser.__get_element_code(start_line, end_line, code) - return c - - for comment in comments: - unit_block.add_comment( - create_comment( - comment["value"], - comment["__start_line__"], - comment["__end_line__"], - code, + if isinstance(args[0], Expr): + return args[0] + if args[0].type == "STRING_LIT": + return String( + args[0].value[1:-1], + self.__get_element_info(args[0]), ) + return args[0] + return args + + def object_elem(self, args: List) -> Expr: + if len(args) == 2: + return (args[0], args[1]) + else: + return (args[0], args[2]) + + @v_args(meta=True) + def object(self, meta: Meta, args: List) -> Any: + object_elems = {} + for k, v in args: + object_elems[k] = v + res = Hash(object_elems, self.__get_element_info(meta)) + return res + + @v_args(meta=True) + def tuple(self, meta: Meta, args: List) -> Any: + return Array(args, self.__get_element_info(meta)) + + @v_args(meta=True) + def block(self, meta: Meta, args: List) -> Any: + if args[0].value == "resource": + au = AtomicUnit( + String( + args[2].value[1:-1], # Remove quotes + self.__get_element_info(args[2]), + ), + args[1].value[1:-1], + ) + au.attributes = [] + au.set_element_info(self.__get_element_info(meta)) + for arg in args[-1]: + if isinstance(arg, Attribute): + au.attributes.append(arg) + else: + au.add_statement(arg) + return au + elif args[0].value == "data": + au = AtomicUnit( + String( + args[2].value[1:-1], # Remove quotes + self.__get_element_info(args[2]), + ), + f"data.{args[1].value[1:-1]}", + ) + au.attributes = [] + au.set_element_info(self.__get_element_info(meta)) + for arg in args[-1]: + if isinstance(arg, Attribute): + au.attributes.append(arg) + else: + au.add_statement(arg) + return au + else: + ub = UnitBlock(args[0].value, UnitBlockType.block) + for arg in args[-1]: + if isinstance(arg, AtomicUnit): + ub.add_atomic_unit(arg) + elif isinstance(arg, UnitBlock): + ub.add_unit_block(arg) + elif isinstance(arg, Attribute): + if args[0].value in ["locals"]: + ub.add_variable( + Variable( + arg.name, + arg.value, + ElementInfo.from_code_element(arg), + ) + ) + else: + ub.add_attribute(arg) + + ub.set_element_info(self.__get_element_info(meta)) + return ub + + def body(self, args: List) -> Any: + return args + + @v_args(meta=True) + def conditional(self, meta: Meta, args: List) -> Any: + condition = ConditionalStatement( + args[0], + ConditionalStatement.ConditionType.IF, + ) + condition.line, condition.column = meta.line, meta.column + condition.end_line, condition.end_column = meta.end_line, meta.end_column + condition.add_statement(args[1]) + condition.else_statement = ConditionalStatement( + Null(), + ConditionalStatement.ConditionType.IF, + ) + condition.else_statement.add_statement(args[2]) + return condition + + @v_args(meta=True) + def attribute(self, meta: Meta, args: List) -> Attribute: + return Attribute(args[0].value, args[2], self.__get_element_info(meta)) + + @v_args(meta=True) + def identifier(self, meta: Meta, value: Any) -> Expr: + if value[0] == "null": + return Null(self.__get_element_info(meta)) + elif value[0] in ["true", "false"]: + return Boolean(value[0] == "true", self.__get_element_info(meta)) + name = value[0] + if isinstance(name, Token): + name = name.value + return VariableReference(name, self.__get_element_info(meta)) + + @v_args(meta=True) + def attr_splat_expr_term(self, meta: Meta, args: List) -> Any: + # TODO: Not supported yet + return Null(self.__get_element_info(meta)) + + @v_args(meta=True) + def full_splat_expr_term(self, meta: Meta, args: List) -> Any: + # TODO: Not supported yet + return Null(self.__get_element_info(meta)) + + @v_args(meta=True) + def for_tuple_expr(self, meta: Meta, args: List) -> Any: + # TODO: Not supported yet + return Null(self.__get_element_info(meta)) + + @v_args(meta=True) + def for_object_expr(self, meta: Meta, args: List) -> Any: + # TODO: Not supported yet + return Null(self.__get_element_info(meta)) + + @v_args(meta=True) + def function_call(self, meta: Meta, args: List) -> Any: + if len(args) == 1: + return FunctionCall( + args[0].value, + [], + self.__get_element_info(meta), ) + return FunctionCall( + args[0].value, + args[1], + self.__get_element_info(meta), + ) + + def arguments(self, args: List) -> Any: + return args + + def start(self, args: List): + return args[0] + +class TerraformParser(p.Parser): def parse_file(self, path: str, type: UnitBlockType) -> UnitBlock: unit_block = UnitBlock(path, type) unit_block.path = path try: with open(path) as f: - parsed_hcl = hcl2.load(f, True) + tree = hcl2.parse(f.read() + "\n") f.seek(0, 0) code = f.readlines() - for key, value in parsed_hcl.items(): - if key in ["resource", "data", "variable", "module", "output"]: - for v in value: - self.parse_atomic_unit(key, unit_block, v, code) - elif key == "__comments__": - self.parse_comments(unit_block, value, code) - elif key == "locals": - for local in value: - unit_block.variables += self.parse_keyvalues( - unit_block, local, code, "variable" - ) - elif key in ["provider", "terraform"]: - continue - else: - throw_exception(EXCEPTIONS["TERRAFORM_COULD_NOT_PARSE"], path) + transformer = GLITCHTransformer(code) + elements = transformer.transform(tree) + for el in elements: + if isinstance(el, AtomicUnit): + unit_block.add_atomic_unit(el) + elif isinstance(el, UnitBlock): + unit_block.add_unit_block(el) + for c in transformer.comments: + unit_block.add_comment(c) except: throw_exception(EXCEPTIONS["TERRAFORM_COULD_NOT_PARSE"], path) + return None + return unit_block def parse_module(self, path: str) -> Module: diff --git a/glitch/parsers/yaml.py b/glitch/parsers/yaml.py index 994234b3..02ac57f2 100644 --- a/glitch/parsers/yaml.py +++ b/glitch/parsers/yaml.py @@ -1,17 +1,26 @@ +import jinja2 +import jinja2.nodes import glitch.parsers.parser as p -from typing import List, Tuple, TextIO, Union +from typing import List, Tuple, TextIO, Union, Any from ruamel.yaml.nodes import Node, MappingNode, SequenceNode, ScalarNode from ruamel.yaml.tokens import Token, CommentToken +from jinja2 import Environment from abc import ABC +from copy import deepcopy + +from glitch.repr.inter import * RecursiveTokenList = List[Union[Token, "RecursiveTokenList", None]] class YamlParser(p.Parser, ABC): - @staticmethod + def __init__(self, options: dict[str, Any] = {}): + self.env = Environment(**options) + def _get_code( + self, start_token: Token | Node, end_token: List[Token | Node] | Token | Node | str, code: List[str], @@ -41,8 +50,7 @@ def _get_code( return res - @staticmethod - def _get_comments(d: Node, file: TextIO) -> set[Tuple[int, str]]: + def _get_comments(self, d: Node, file: TextIO) -> set[Tuple[int, str]]: """Extracts comments from a YAML file and returns a set of tuples with the line number and the comment itself. Args: @@ -110,3 +118,332 @@ def yaml_comments(d: Node) -> List[Tuple[int, str]]: comments.append((i + 1, line.strip())) return set(comments) + + def __get_content(self, info: ElementInfo, code: List[str]) -> str: + content = code[info.line - 1 : info.end_line] + if info.line == info.end_line: + content[0] = content[0][info.column - 1 : info.end_column - 1] + else: + content[0] = content[0][info.column - 1 :] + content[-1] = content[-1][: info.end_column - 1] + content = "".join(content) + + if content.startswith("|"): + content = content[1:] + info.column += 1 + if content.startswith(">-"): + content = content[2:] + info.column += 2 + + l_rmvd = content[: len(content) - len(content.lstrip())] + content = content.lstrip() + for c in l_rmvd: + if c == "\n": + info.line += 1 + info.column = 1 + else: + info.column += 1 + + r_rmvd = content[len(content.rstrip()) :] + content = content.rstrip() + for c in r_rmvd[::-1]: + if c == "\n" and info.end_line > info.line: + info.end_line -= 1 + info.end_column = len(code[info.end_line - 1]) + elif info.end_line > info.line: + info.end_column -= 1 + + return content + + def __parse_jinja_node( + self, node: jinja2.nodes.Node, base_info: ElementInfo + ) -> Expr: + info = deepcopy(base_info) + code = base_info.code.split("\n") + + # The lineno and col of the filter start + # in the | so we ignore them + if not isinstance(node, jinja2.nodes.Filter): + info.line = base_info.line + node.lineno - 1 + info.end_line = base_info.line + (node.end_lineno - node.lineno) + code = code[node.lineno - 1 : node.end_lineno] + + info.column = base_info.column + node.col + if info.line == info.end_line: + info.end_column = base_info.column + node.end_col + else: + info.end_column = node.end_col + 1 + + code[-1] = code[-1][: node.end_col] + code[0] = code[0][node.col :] + info.code = "\n".join(code) + + if isinstance(node, jinja2.nodes.TemplateData): + return String(node.data, info) + elif isinstance(node, jinja2.nodes.Name): + return VariableReference(node.name, info) + elif isinstance(node, jinja2.nodes.Const): + if isinstance(node.value, str): + return String(node.value, info) + elif isinstance(node.value, int): + return Integer(node.value, info) + elif isinstance(node.value, float): + return Float(node.value, info) + elif isinstance(node.value, bool): + return Boolean(node.value, info) + else: + raise ValueError("Const not supported") + elif isinstance(node, jinja2.nodes.Call): + if isinstance(node.node, jinja2.nodes.Name): + return FunctionCall( + node.node.name, + [self.__parse_jinja_node(arg, base_info) for arg in node.args], + info, + ) # type: ignore + else: + # TODO: When the node is for instance a Getattr + return Null() + elif isinstance(node, jinja2.nodes.Filter): + assert node.node is not None + args: List[Expr] = [] + args.append(self.__parse_jinja_node(node.node, base_info)) + for arg in node.args: + args.append(self.__parse_jinja_node(arg, base_info)) + return FunctionCall("filter|" + node.name, args, info) + elif isinstance(node, jinja2.nodes.List): + return Array([self.__parse_jinja_node(n, base_info) for n in node.items], base_info) # type: ignore + elif isinstance(node, jinja2.nodes.Add): + return Sum( + info, + self.__parse_jinja_node(node.left, base_info), + self.__parse_jinja_node(node.right, base_info), + ) + elif isinstance(node, jinja2.nodes.Getattr): + attr_info = deepcopy(info) + attr_info.column = info.end_column - len(node.attr) + attr_info.code = attr_info.code[-len(node.attr) :] + return Access( + info, + self.__parse_jinja_node(node.node, base_info), + String(node.attr, attr_info), + ) + elif isinstance(node, jinja2.nodes.Getitem): + return Access( + info, + self.__parse_jinja_node(node.node, base_info), + self.__parse_jinja_node(node.arg, base_info), + ) + elif isinstance(node, jinja2.nodes.Div): + return Divide( + info, + self.__parse_jinja_node(node.left, base_info), + self.__parse_jinja_node(node.right, base_info), + ) + elif isinstance(node, jinja2.nodes.Or): + return Or( + info, + self.__parse_jinja_node(node.left, base_info), + self.__parse_jinja_node(node.right, base_info), + ) + elif isinstance(node, jinja2.nodes.Mul): + return Multiply( + info, + self.__parse_jinja_node(node.left, base_info), + self.__parse_jinja_node(node.right, base_info), + ) + + elif isinstance(node, jinja2.nodes.Tuple): + return Array( + [self.__parse_jinja_node(n, base_info) for n in node.items], info + ) + elif isinstance(node, jinja2.nodes.Dict): + value: Dict[Expr, Expr] = {} + for item in node.items: + value[self.__parse_jinja_node(item.key, base_info)] = ( + self.__parse_jinja_node(item.value, base_info) + ) + return Hash(value, info) + elif isinstance(node, jinja2.nodes.Not): + return Not(info, self.__parse_jinja_node(node.node, base_info)) + elif isinstance(node, jinja2.nodes.Sub): + return Subtract( + info, + self.__parse_jinja_node(node.left, base_info), + self.__parse_jinja_node(node.right, base_info), + ) + elif isinstance(node, jinja2.nodes.Pow): + return Power( + info, + self.__parse_jinja_node(node.left, base_info), + self.__parse_jinja_node(node.right, base_info), + ) + elif isinstance(node, jinja2.nodes.Mod): + return Modulo( + info, + self.__parse_jinja_node(node.left, base_info), + self.__parse_jinja_node(node.right, base_info), + ) + elif isinstance(node, jinja2.nodes.CondExpr): + c = ConditionalStatement( + self.__parse_jinja_node(node.test, base_info), + ConditionalStatement.ConditionType.IF, + is_top=True, + ) + c.add_statement(self.__parse_jinja_node(node.expr1, base_info)) + if node.expr2 is not None: + c.else_statement = ConditionalStatement( + Null(), ConditionalStatement.ConditionType.IF, is_default=True + ) + c.else_statement.add_statement( + self.__parse_jinja_node(node.expr2, base_info) + ) + return c + elif isinstance(node, jinja2.nodes.Assign): + return Assign( + info, + self.__parse_jinja_node(node.target, base_info), + self.__parse_jinja_node(node.node, base_info), + ) + elif isinstance(node, jinja2.nodes.And): + return And( + info, + self.__parse_jinja_node(node.left, base_info), + self.__parse_jinja_node(node.right, base_info), + ) + elif isinstance(node, jinja2.nodes.Concat): + c = Null() + for n in node.nodes: + c = Sum(info, c, self.__parse_jinja_node(n, base_info)) + return c + elif isinstance(node, jinja2.nodes.Not): + return Not(info, self.__parse_jinja_node(node.node, base_info)) + elif isinstance( + node, + ( + jinja2.nodes.Compare, + jinja2.nodes.Test, + jinja2.nodes.Slice, + jinja2.nodes.Output, + jinja2.nodes.FloorDiv, + jinja2.nodes.For, + jinja2.nodes.If, + ), + ): + # TODO Support these nodes + return Null() + else: + raise ValueError(f"Node not supported {node}") + + def __parse_string(self, v: str, info: ElementInfo) -> Expr: + """ + Parses a string to the intermediate representation and unrolls + the interpolation. + """ + if v in ["null", "~"]: + return Null(info) + quotes = v.startswith(("'", '"')) and v.endswith(("'", '"')) + + body = self.env.parse(v).body + if len(body) == 0: + return String(v, info) + + jinja_nodes = list(body[0].iter_child_nodes()) + + for node in jinja_nodes[::-1]: + if isinstance(node, jinja2.nodes.TemplateData) and node.data.strip() in [ + "'", + '"', + ]: + jinja_nodes.remove(node) + + if len(jinja_nodes) > 1: + parts: List[Expr] = [] + for node in jinja_nodes: + parts.append(self.__parse_jinja_node(node, info)) + + if ( + isinstance(parts[0], String) + and quotes + and parts[0].value.startswith(("'", '"')) + ): + parts[0].value = parts[0].value[1:] + parts[0].column += 1 + + if ( + isinstance(parts[-1], String) + and quotes + and parts[-1].value.endswith(("'", '"')) + ): + parts[-1].value = parts[-1].value[:-1] + parts[-1].end_column -= 1 + + expr = Sum(info, parts[0], parts[1]) + for part in parts[2:]: + expr = Sum(info, expr, part) + + return expr + elif len(jinja_nodes) == 1: + expr = self.__parse_jinja_node(jinja_nodes[0], info) + if ( + isinstance(expr, String) + and quotes + and expr.value.startswith(("'", '"')) + and expr.value.endswith(("'", '"')) + ): + expr.value = expr.value[1:-1] + return expr + + raise ValueError("No Jinja nodes found") + + def get_value(self, value: Node, code: List[str]) -> Expr: + info = ElementInfo( + value.start_mark.line + 1, + value.start_mark.column + 1, + value.end_mark.line + 1, + value.end_mark.column + 1, + self._get_code(value, value, code), + ) + v: Any = value.value + + if value.tag.endswith("bool"): + if isinstance(v, str): + v_lower = v.lower() + if v_lower in ["true", "yes", "on", "y"]: + return Boolean(True, info) + elif v_lower in ["false", "no", "off", "n"]: + return Boolean(False, info) + return Boolean(bool(v), info) + elif value.tag.endswith("int"): + return Integer(int(v), info) + elif value.tag.endswith("float"): + return Float(float(v), info) + elif value.tag.endswith("str"): + content = self.__get_content(info, code) + return self.__parse_string("".join(content), info) + elif v is None or value.tag.endswith("null"): + return Null(info) + elif isinstance(value, MappingNode) and isinstance(v, list): + content = self.__get_content(info, code) + # Required for Jinja2 expressions not wrapped in quotes + if content.startswith("{{") and content.endswith("}}"): + return self.__parse_string("".join(content), info) + else: + return Hash( + { + self.get_value(key, code): self.get_value(val, code) # type: ignore + for key, val in v # type: ignore + }, + info, + ) + elif isinstance(v, list): + return Array([self.get_value(val, code) for val in v], info) # type: ignore + elif isinstance(v, dict): + return Hash( + { + self.get_value(key, code): self.get_value(val, code) # type: ignore + for key, val in v.items() # type: ignore + }, + info, + ) + else: + raise ValueError(f"Unknown value type with tag {value.tag}: {type(v)}") diff --git a/glitch/tests/design/__init__.py b/glitch/rego/__init__.py similarity index 100% rename from glitch/tests/design/__init__.py rename to glitch/rego/__init__.py diff --git a/glitch/rego/engine.py b/glitch/rego/engine.py new file mode 100644 index 00000000..c54b6d52 --- /dev/null +++ b/glitch/rego/engine.py @@ -0,0 +1,419 @@ +import json +import os + +from typing import List, Dict, Any, Optional, cast +from glitch.rego.rego_python.src.rego_python import run_rego +from glitch.repr.inter import * +from glitch.analysis.rules import Error + + +def run_analyses( + input: str, + config: Dict[str, Dict[str, List[str]]], + rego_modules: Dict[str, str], +) -> List[Error]: + input_data = json.loads(input) + + data: Dict[str, Any] = config + + if not rego_modules: + # No modules to run, return empty errors + return [] + + result = run_rego(input_data, data, rego_modules) + + if result is None: + # Nothing to process + return [] + + if isinstance(result, dict) and "error" in result: + print("Error:", result["error"]) # type: ignore + return [] + + errors: List[Error] = [] + + flat_values: List[Dict[str, Any]] = [] + + # Parse the Go Rego engine output to a set of errors + # It can be a list or a list of lists, so we put everything in a single list + result = cast(List[Dict[str, Any]], result) + for entry in result: + for expr in entry.get("expressions", []): + values_list = expr.get("value", []) + # Normalize nested structure to flat list of dicts + for item in values_list: + if isinstance(item, list): + flat_values.extend(item) # type: ignore + elif isinstance(item, dict): + flat_values.append(item) # type: ignore + + # Create the rego errors + for val in flat_values: + if "element" in val: + element = element_from_dict(val["element"]) + errors.append( + Error( + code=val.get("type", ""), + el=element, + path=val.get("path", ""), + repr=repr(element), + opt_msg=val.get("description"), + ) + ) + + return errors + + +def load_rego_from_path(file_path: str, result: dict[str, str]) -> None: + key: str = os.path.split(file_path)[-1] + with open(file_path, "r") as f: + result[key] = f.read() + + +def element_from_dict(data: Dict[str, Any]) -> CodeElement: + """ + Recursively builds a CodeElement from a dict like the Rego query output. + """ + + # In case we just return a key-value pair from an Hash + if "key" in data and "value" in data and "ir_type" not in data: + key_elem = element_from_dict(data["key"]) + value_elem = element_from_dict(data["value"]) + + assert isinstance( + value_elem, Expr + ), f"Value in Key Value Hash mapping {value_elem.__class__.__name__} is not an Expr" + + # Derive position info from key and value, combining their code + info = ElementInfo( + line=getattr(key_elem, "line", -1), + column=getattr(key_elem, "column", -1), + end_line=getattr(value_elem, "end_line", -1), + end_column=getattr(value_elem, "end_column", -1), + code=f"{key_elem.code}: {value_elem.code}", # Combined code + ) + + # Extract the name from the key + if isinstance(key_elem, String): + name = key_elem.value + else: + name = getattr(key_elem, "value", None) + if isinstance(name, str): + pass + else: + name = str(getattr(key_elem, "code", key_elem)) + + return KeyValue(name, value_elem, info) + + # Step 1: Extract common element info + info = ElementInfo( + line=data.get("line", -1), + column=data.get("column", -1), + end_line=data.get("end_line", -1), + end_column=data.get("end_column", -1), + code=data.get("code", ""), + ) + + ir_type: Optional[str] = data.get("ir_type") + + # Step 2: Handle by type + if ir_type == "String": + return String(data["value"], info) + + elif ir_type == "Integer": + return Integer(data["value"], info) + + elif ir_type == "Float": + return Float(data["value"], info) + + elif ir_type == "Boolean": + return Boolean(data["value"], info) + + elif ir_type == "Complex": + return Complex(data["value"], info) + + elif ir_type == "Null": + return Null(info) + + elif ir_type == "Undef": + return Undef(info) + + elif ir_type == "Array": + values_data = data.get("value", []) + values: List[Expr] = [] + for v in values_data: + elem = element_from_dict(v) + assert isinstance( + elem, Expr + ), f"Element in Array {elem.__class__.__name__} is not an Expr" + values.append(elem) + return Array(values, info) + + elif ir_type == "Hash": + # Hash is stored as dict[Expr, Expr] + hash_value: Dict[Expr, Expr] = {} + for pair in data["value"]: + key = element_from_dict(pair["key"]) + val = element_from_dict(pair["value"]) + assert isinstance( + key, Expr + ), f"Key in Hash Mapping {key.__class__.__name__} is not an Expr" + assert isinstance( + val, Expr + ), f"Value in Hash Mapping {val.__class__.__name__} is not an Expr" + hash_value[key] = val + return Hash(hash_value, info) + + elif ir_type == "VariableReference": + return VariableReference(data["value"], info) + + elif ir_type == "Attribute": + value_elem = element_from_dict(data["value"]) + assert isinstance( + value_elem, Expr + ), f"value in Attribute {value_elem.__class__.__name__} is not an Expr" + return Attribute(data["name"], value_elem, info) + + elif ir_type == "Variable": + value_elem = element_from_dict(data["value"]) + assert isinstance( + value_elem, Expr + ), f"value in Variable {value_elem.__class__.__name__} is not an Expr" + return Variable(data["name"], value_elem, info) + + elif ir_type == "KeyValue": + value_elem = element_from_dict(data["value"]) + assert isinstance( + value_elem, Expr + ), f"value in KeyValue {value_elem.__class__.__name__} is not an Expr" + return KeyValue(data["name"], value_elem, info) + + # Expressions and binary/unary operations + elif ir_type in { + "Or", + "And", + "Sum", + "Equal", + "NotEqual", + "LessThan", + "LessThanOrEqual", + "GreaterThan", + "GreaterThanOrEqual", + "In", + "Subtract", + "Multiply", + "Divide", + "Modulo", + "Power", + "RightShift", + "LeftShift", + "Access", + "BitwiseAnd", + "BitwiseOr", + "BitwiseXor", + "Assign", + }: + left_elem = element_from_dict(data["left"]) + right_elem = element_from_dict(data["right"]) + + if ir_type == "Equal" and left_elem.__class__.__name__ == "CodeElement": + print("left: ", data["left"]) + + if ir_type == "Equal" and right_elem.__class__.__name__ == "CodeElement": + print("right: ", data["right"]) + + assert isinstance( + left_elem, Expr + ), f"left_elem in BinaryOperation {ir_type}, {left_elem.__class__.__name__} is not an Expr" + assert isinstance( + right_elem, Expr + ), f"right_elem in BinaryOperation {ir_type}, {right_elem.__class__.__name__} is not an Expr" + + binary_op_classes = { + "Or": Or, + "And": And, + "Sum": Sum, + "Equal": Equal, + "NotEqual": NotEqual, + "LessThan": LessThan, + "LessThanOrEqual": LessThanOrEqual, + "GreaterThan": GreaterThan, + "GreaterThanOrEqual": GreaterThanOrEqual, + "In": In, + "Subtract": Subtract, + "Multiply": Multiply, + "Divide": Divide, + "Modulo": Modulo, + "Power": Power, + "RightShift": RightShift, + "LeftShift": LeftShift, + "Access": Access, + "BitwiseAnd": BitwiseAnd, + "BitwiseOr": BitwiseOr, + "BitwiseXor": BitwiseXor, + "Assign": Assign, + } + cls = binary_op_classes[ir_type] + return cls(info, left_elem, right_elem) + + elif ir_type in {"Not", "Minus"}: + expr_elem = element_from_dict(data["expr"]) + + assert isinstance( + expr_elem, Expr + ), f"expr_elem in UnaryRelation {ir_type}, {expr_elem.__class__.__name__} is not an Expr" + + unary_op_classes = {"Not": Not, "Minus": Minus} + cls = unary_op_classes[ir_type] + return cls(info, expr_elem) + + elif ir_type == "FunctionCall": + name = data.get("name", "") + args_data = data.get("args", []) + args: List[Expr] = [] + for a in args_data: + arg_elem = element_from_dict(a) + assert isinstance( + arg_elem, Expr + ), f"arg_elem in FunctionCall {arg_elem.__class__.__name__} is not an Expr" + return FunctionCall(name, args, info) + + elif ir_type == "MethodCall": + receiver_elem = element_from_dict(data["receiver"]) + assert isinstance( + receiver_elem, Expr + ), f"receiver_elem in MethodCall {receiver_elem.__class__.__name__} is not an Expr" + + args_data = data.get("args", []) + args: List[Expr] = [] + for a in args_data: + arg_elem = element_from_dict(a) + assert isinstance( + arg_elem, Expr + ), f"arg_elem in MethodCall {arg_elem.__class__.__name__} is not an Expr" + return MethodCall(receiver_elem, data["method"], args, info) + + elif ir_type == "BlockExpr": + block = BlockExpr(info) + for s in data.get("statements", []): + stmt_elem = element_from_dict(s) + block.add_statement(stmt_elem) + return block + + elif ir_type == "AddArgs": + value = data.get("value", []) + args: List[Expr] = [] + for v in value: + arg_elem = element_from_dict(v) + assert isinstance( + arg_elem, Expr + ), f"arg_elem in AddArgs {arg_elem.__class__.__name__} is not an Expr" + args.append(arg_elem) + return AddArgs(args, info) + + elif ir_type == "Comment": + return Comment(data.get("content", ""), info) + + elif ir_type == "ConditionalStatement": + condition = element_from_dict(data["condition"]) + assert isinstance( + condition, Expr + ), f"condition in ConditionalStatement {condition.__class__.__name__} is not an Expr" + cond_type = getattr(ConditionalStatement.ConditionType, data["type"]) + cond = ConditionalStatement( + condition, + cond_type, + data.get("is_default", False), + data.get("is_top", False), + info, + ) + + else_stmt_data = data.get("else_statement") + if else_stmt_data: + else_elem = element_from_dict(else_stmt_data) + assert isinstance( + else_elem, ConditionalStatement + ), f"else_elem in ConditionalStatement {else_elem.__class__.__name__} is not an ConditionalStatement" + cond.else_statement = else_elem + + # Inherits from Block, which can have a list of statements + for s in data.get("statements", []): + cond.add_statement(element_from_dict(s)) + return cond + + elif ir_type == "Block": + blk = Block() + blk.set_element_info(info) + for s in data.get("statements", []): + blk.add_statement(element_from_dict(s)) + return blk + + elif ir_type == "AtomicUnit": + name = element_from_dict(data["name"]) + assert isinstance( + name, Expr + ), f"name in AtomicUnit {name.__class__.__name__} is not an Expr" + unit = AtomicUnit(name, data.get("type", "unknown")) + unit.set_element_info(info) + for attr in data.get("attributes", []): + attr_elem = element_from_dict(attr) + assert isinstance( + attr_elem, Attribute + ), f"attr_elem in AtomicUnit {attr_elem.__class__.__name__} is not an Attribute" + unit.add_attribute(attr_elem) + return unit + + elif ir_type == "UnitBlock": + unit = UnitBlock(data.get("name", ""), data.get("type", "unknown")) + for attr_data in data.get("attributes", []): + attr_elem = element_from_dict(attr_data) + assert isinstance( + attr_elem, Attribute + ), f"attr_elem in UnitBlock {attr_elem.__class__.__name__} is not an Attribute" + unit.add_attribute(attr_elem) + + for comment_data in data.get("comments", []): + comment_elem = element_from_dict(comment_data) + assert isinstance( + comment_elem, Comment + ), f"comment_elem in UnitBlock {comment_elem.__class__.__name__} is not an Comment" + unit.add_comment(comment_elem) + + for var_data in data.get("variables", []): + var_elem = element_from_dict(var_data) + assert isinstance( + var_elem, Variable + ), f"var_elem in UnitBlock {var_elem.__class__.__name__} is not an Variable" + unit.add_variable(var_elem) + + for au_data in data.get("atomic_units", []): + au_elem = element_from_dict(au_data) + assert isinstance( + au_elem, AtomicUnit + ), f"au_data in UnitBlock {au_elem.__class__.__name__} is not an AtomicUnit" + unit.add_atomic_unit(au_elem) + + for ub_data in data.get("unit_blocks", []): + ub_elem = element_from_dict(ub_data) + assert isinstance( + ub_elem, UnitBlock + ), f"ub_elem in UnitBlock {ub_elem.__class__.__name__} is not an UnitBlock" + unit.add_unit_block(ub_elem) + + for dep_data in data.get("dependencies", []): + dep_elem = element_from_dict(dep_data) + assert isinstance( + dep_elem, Dependency + ), f"dep_data in UnitBlock {dep_elem.__class__.__name__} is not an Dependency" + unit.add_dependency(dep_elem) + + unit.path = data.get("path", "") + return unit + + elif ir_type == "Dependency": + # Dont know if Info is needed + return Dependency(data.get("names", [])) + + else: + # Unknown type: treat as generic CodeElement + return CodeElement(info) diff --git a/glitch/rego/queries/design/design_avoid_comments.rego b/glitch/rego/queries/design/design_avoid_comments.rego new file mode 100644 index 00000000..9fd1fa42 --- /dev/null +++ b/glitch/rego/queries/design/design_avoid_comments.rego @@ -0,0 +1,35 @@ +package glitch + +import data.glitch_lib + +get_first_line(elements) = line { + count(elements) > 0 + line = { elements[0].line } +} +get_first_line(elements) = set() { + count(elements) == 0 +} + +Glitch_Analysis[result] { + parent := glitch_lib._gather_parent_unit_blocks[_] + parent.path != "" + count(parent.comments) > 0 + + lines := { parent.line | parent.line > 0 } + | get_first_line(parent.atomic_units) + | get_first_line(parent.statements) + | get_first_line(parent.unit_blocks) + + count(lines) > 0 + line := min(lines) + + comment := parent.comments[_] + comment.line >= line + + result := {{ + "type": "design_avoid_comments", + "element": comment, + "path": parent.path, + "description": "Avoid comments - Comments may lead to bad code or be used as a way to justify bad code." + }} +} diff --git a/glitch/rego/queries/design/design_imperative_abstraction.rego b/glitch/rego/queries/design/design_imperative_abstraction.rego new file mode 100644 index 00000000..fed0c425 --- /dev/null +++ b/glitch/rego/queries/design/design_imperative_abstraction.rego @@ -0,0 +1,24 @@ +package glitch + +import data.glitch_lib + +Glitch_Analysis[result] { + parent := glitch_lib._gather_parent_unit_blocks[_] + parent.path != "" + + resources := count(glitch_lib.all_atomic_units(parent)) + executions := count({au | + au := glitch_lib.all_atomic_units(parent)[_] + au.type == data.design.exec_atomic_units[_] + }) + + executions > 2 + (executions / resources) > 0.2 + + result := {{ + "type": "design_imperative_abstraction", + "element": parent, + "path": parent.path, + "description": "Imperative abstraction - The presence of imperative statements defies the purpose of IaC declarative languages." + }} +} diff --git a/glitch/rego/queries/design/design_long_resource.rego b/glitch/rego/queries/design/design_long_resource.rego new file mode 100644 index 00000000..ea413829 --- /dev/null +++ b/glitch/rego/queries/design/design_long_resource.rego @@ -0,0 +1,27 @@ +package glitch + +import data.glitch_lib + +Glitch_Analysis[result] { + parent := glitch_lib._gather_parent_unit_blocks[_] + parent.path != "" + atomic_units := glitch_lib.all_atomic_units(parent) + node := atomic_units[_] + node.type == data.design.exec_atomic_units[_] + + lines := [ + line | + attr := node.attributes[_] + line := split(attr.code, "\n")[_] + not regex.match("^\\s*$", line) + ] + + count(lines) > 7 + + result := {{ + "type": "design_long_resource", + "element": node, + "path": parent.path, + "description": "Long Resource - Long resources may decrease the readability and maintainability of the code." + }} +} diff --git a/glitch/rego/queries/design/design_misplaced_attribute.rego b/glitch/rego/queries/design/design_misplaced_attribute.rego new file mode 100644 index 00000000..b9ffb0c6 --- /dev/null +++ b/glitch/rego/queries/design/design_misplaced_attribute.rego @@ -0,0 +1,86 @@ +package glitch + +import data.glitch_lib + +# Second option for Puppet was not tested in real code due to lack of test data + +chef_priority(attr) = index { + attr == "source" + index = 1 +} else = index { + attr == "owner" + index = 2 +} else = index { + attr == "group" + index = 2 +} else = index { + attr == "mode" + index = 3 +} else = index { + attr == "action" + index = 4 +} + +# Chef +Glitch_Analysis[result] { + parent := glitch_lib._gather_parent_unit_blocks[_] + parent.path != "" + endswith(parent.name, ".rb") + + atomic_units := glitch_lib.all_atomic_units(parent) + node := atomic_units[_] + + attr_names := ["source", "owner", "group", "mode", "action"] + order := [ + chef_priority(attr) | + attr := node.attributes[_].name + attr = attr_names[_] + ] + + order != sort(order) + + result := {{ + "type": "design_misplaced_attribute", + "element": node, + "path": parent.path, + "description": "Misplaced attribute - The developers should try to follow the languages' style guides. These style guides define the expected attribute order." + }} +} + +# Puppet +Glitch_Analysis[result] { + parent := glitch_lib._gather_parent_unit_blocks[_] + parent.path != "" + endswith(parent.name, ".pp") + atomic_units := glitch_lib.all_atomic_units(parent) + node := atomic_units[_] + + some n + n > 0 + node.attributes[n].name == "ensure" + + result := {{ + "type": "design_misplaced_attribute", + "element": node, + "path": parent.path, + "description": "Misplaced attribute - The developers should try to follow the languages' style guides. These style guides define the expected attribute order." + }} +} + +Glitch_Analysis[result] { + parent := glitch_lib._gather_parent_unit_blocks[_] + parent.path != "" + endswith(parent.name, ".pp") + + some i + i >= 0 + i < count(parent.attributes) - 1 + parent.attributes[i].value != {} + + result := {{ + "type": "design_misplaced_attribute", + "element": parent, + "path": parent.path, + "description": "Misplaced attribute - The developers should try to follow the languages' style guides. These style guides define the expected attribute order." + }} +} diff --git a/glitch/rego/queries/design/design_multifaceted_abstraction.rego b/glitch/rego/queries/design/design_multifaceted_abstraction.rego new file mode 100644 index 00000000..9120cc23 --- /dev/null +++ b/glitch/rego/queries/design/design_multifaceted_abstraction.rego @@ -0,0 +1,32 @@ +package glitch + +import data.glitch_lib + +checker(node) { + regex.match("(&&|;|\\|)", node.name) +} +checker(node) { + attr := node.attributes[_] + regex.match("(&&|;|\\|)", attr.value.value) +} +checker(node) { + attr := node.attributes[_] + regex.match("(&&|;|\\|)", attr.value.code) +} + +Glitch_Analysis[result] { + parent := glitch_lib._gather_parent_unit_blocks[_] + parent.path != "" + atomic_units := glitch_lib.all_atomic_units(parent) + node := atomic_units[_] + node.type == data.design.exec_atomic_units[_] + + checker(node) + + result := {{ + "type": "design_multifaceted_abstraction", + "element": node, + "path": parent.path, + "description": "Multifaceted Abstraction - Each block should only specify the properties of a single piece of software." + }} +} diff --git a/glitch/rego/queries/design/implementation_too_many_variables.rego b/glitch/rego/queries/design/implementation_too_many_variables.rego new file mode 100644 index 00000000..bae50cc6 --- /dev/null +++ b/glitch/rego/queries/design/implementation_too_many_variables.rego @@ -0,0 +1,23 @@ +package glitch + +import data.glitch_lib + +ALLOWED_TYPES = ["unkown", "script", "tasks"] + +Glitch_Analysis[result] { + parent := glitch_lib._gather_parent_unit_blocks[_] + parent.path != "" + type := ALLOWED_TYPES[_] + parent.type = type + + vars := glitch_lib.count_nodes_with_irtype(parent, "Variable") + + (vars / parent.lines) > 0.3 + + result := {{ + "type": "implementation_too_many_variables", + "element": parent, + "path": parent.path, + "description": "Too many variables - The existence of too many variables in a single IaC script may reveal that the script is being used for too many purposes." + }} +} diff --git a/glitch/rego/queries/library/glitch_lib.rego b/glitch/rego/queries/library/glitch_lib.rego new file mode 100644 index 00000000..26876f95 --- /dev/null +++ b/glitch/rego/queries/library/glitch_lib.rego @@ -0,0 +1,116 @@ +package glitch_lib + +_gather_parent_unit_blocks[ub] { + walk(input, [_, ub]) + ub.ir_type == "UnitBlock" +} + + +all_atomic_units(node) = units { + units = {n | + walk(node, [path, n]) + n.ir_type == "AtomicUnit" + } +} + +all_attributes(node) = attrs { + attrs = {n | + walk(node, [path, n]) + n.ir_type == "Attribute" + # We only want KeyValues inside it, not itself + n.value.ir_type != "BlockExpr" + } +} + +all_variables(node) = vars { + vars = {n | + walk(node, [path, n]) + n.ir_type == "Variable" + # We only want KeyValues inside it, not itself + n.value.ir_type != "BlockExpr" + } +} + +# This allows us to stop at the first level of conditional statement +all_conditional_statements(node) = conditions { + conditions = {n | + walk(node, [path, n]) + n.ir_type == "ConditionalStatement" + } +} + +count_nodes_with_irtype(root, t) = n { + n = count({ + v | + [_, v] := walk(root) + v.ir_type == t + }) +} + +traverse(node, pattern) { + # Use walk to safely traverse all nodes + walk(node, [path, n]) + check_leaf(n, pattern) # Only check leaf nodes +} + +check_leaf(node, pattern) { + node.ir_type == "String" + check_string(node, pattern) +} else { + node.ir_type == "MethodCall" + check_string(node.receiver, pattern) +} else { + node.ir_type == "Boolean" + check_boolean(node, pattern) +} else { + node.ir_type == "VariableReference" + check_string(node, pattern) +} else { + node.ir_type == "FunctionCall" + check_function_call(node, pattern) +} + +check_string(node, pattern) { + # If it is a string, + is_string(pattern) + regex.match(pattern, node.value) +} else { + # If it is an array or set + not is_string(pattern) + contains(node.value, pattern[_]) +} + +check_boolean(node, value) { + node.value == value +} + +# We are simply checking the name of the function called +check_function_call(node, pattern) { + # If it is a string, + is_string(pattern) + regex.match(pattern, node.name) +} else { + # If it is an array or set + not is_string(pattern) + contains(node.name, pattern[_]) +} + +# Implemented as a substitute for VarChecker, checks if there is at least one VariableReference and passes if there isn't +traverse_var(node) { + not has_variable_reference(node) +} + +has_variable_reference(node) { + walk(node, [_, n]) + n.ir_type == "VariableReference" +} + +# Check if a string contains a substring +contains(str, substr) { + regex.match(sprintf("(?i).*%s.*", [substr]), str) +} + +is_ir_type_in(value, allowed) { + t := allowed[_] + value.ir_type == t +} \ No newline at end of file diff --git a/glitch/rego/queries/security/sec_def_admin.rego b/glitch/rego/queries/security/sec_def_admin.rego new file mode 100644 index 00000000..b0727c31 --- /dev/null +++ b/glitch/rego/queries/security/sec_def_admin.rego @@ -0,0 +1,62 @@ +package glitch + +import data.glitch_lib + +check_def_admin_pair(name, value) { + # Check if the name of the node matches any of the roles or users defined in security + combined := array.concat(data.security.roles, data.security.users) + element := combined[_] + pattern := sprintf("[_A-Za-z0-9$/\\.\\[\\]-]*%s\\b", [element]) + regex.match(pattern, name) + + # Check if there is not a VariableReference object + glitch_lib.traverse_var(value) + + # Check if it is a admin user + glitch_lib.traverse(value, data.security.admin) +} + +Glitch_Analysis[result] { + parent := glitch_lib._gather_parent_unit_blocks[_] + parent.path != "" + attr := glitch_lib.all_attributes(parent) + variables := glitch_lib.all_variables(parent) + all_nodes := attr | variables + node := all_nodes[_] + + # We need to use walk since we can have Hashs inside one another + glitch_lib.is_ir_type_in(node.value, ["String"]) + check_def_admin_pair(node.name, node.value) + matched_node := node + + result := {{ + "type": "sec_def_admin", + "element": matched_node, + "path": parent.path, + "description": "Admin by default - Developers should always try to give the least privileges possible. Admin privileges may indicate a security problem. (CWE-250)" + }} +} + +Glitch_Analysis[result] { + parent := glitch_lib._gather_parent_unit_blocks[_] + parent.path != "" + attr := glitch_lib.all_attributes(parent) + variables := glitch_lib.all_variables(parent) + all_nodes := attr | variables + node := all_nodes[_] + + # We need to use walk since we can have Hashs inside one another + walk(node, [_, n]) + n.ir_type == "Hash" + current_pair := n.value[_] + glitch_lib.is_ir_type_in(current_pair.value, ["String"]) + check_def_admin_pair(current_pair.key.value, current_pair.value) + matched_node := current_pair + + result := {{ + "type": "sec_def_admin", + "element": matched_node, + "path": parent.path, + "description": "Admin by default - Developers should always try to give the least privileges possible. Admin privileges may indicate a security problem. (CWE-250)" + }} +} diff --git a/glitch/rego/queries/security/sec_empty_pass.rego b/glitch/rego/queries/security/sec_empty_pass.rego new file mode 100644 index 00000000..eca1492d --- /dev/null +++ b/glitch/rego/queries/security/sec_empty_pass.rego @@ -0,0 +1,90 @@ +package glitch + +import data.glitch_lib + +whitelist_contains(name) { + whitelist := array.concat(data.security.secrets_white_list, data.security.profile) + whitelist[_] == name +} + +checking_value(value) { + value.ir_type == "String" + + value.value == "" +} else { + value.ir_type == "Null" + + value.value == null + + null_values := data.security.null_values + + possible_value := null_values[_] + + glitch_lib.contains(value.code, possible_value) +} else { + value.ir_type == "Undef" + + value.value == null + + null_values := data.security.null_values + + glitch_lib.contains(value.code, "undef") +} + +check_pair_empty_password(name, value) { + hardcoded := data.security.passwords + + item := hardcoded[_] + hard_coded_pattern := sprintf("[_A-Za-z0-9$/\\.\\[\\]-]*%s\\b", [item]) + + regex.match(hard_coded_pattern, name) + + not whitelist_contains(lower(name)) + + glitch_lib.traverse_var(value) + + checking_value(value) +} + +Glitch_Analysis[result] { + parent := glitch_lib._gather_parent_unit_blocks[_] + parent.path != "" + attr := glitch_lib.all_attributes(parent) + variables := glitch_lib.all_variables(parent) + all_nodes := attr | variables + node := all_nodes[_] + + walk(node, [_, n]) + n.value.ir_type != "Hash" + check_pair_empty_password(node.name, node.value) + matched_node := node + + result := {{ + "type": "sec_empty_pass", + "element": matched_node, + "path": parent.path, + "description": "Empty password - An empty password is indicative of a weak password which may lead to a security breach. (CWE-258)" + }} +} + +Glitch_Analysis[result] { + parent := glitch_lib._gather_parent_unit_blocks[_] + parent.path != "" + attr := glitch_lib.all_attributes(parent) + variables := glitch_lib.all_variables(parent) + all_nodes := attr | variables + node := all_nodes[_] + + walk(node, [_, n]) + n.ir_type == "Hash" + current_pair := n.value[_] + check_pair_empty_password(current_pair.key.value, current_pair.value) + matched_node := current_pair + + result := {{ + "type": "sec_empty_pass", + "element": matched_node, + "path": parent.path, + "description": "Empty password - An empty password is indicative of a weak password which may lead to a security breach. (CWE-258)" + }} +} diff --git a/glitch/rego/queries/security/sec_full_permission_filesystem.rego b/glitch/rego/queries/security/sec_full_permission_filesystem.rego new file mode 100644 index 00000000..6cd31b61 --- /dev/null +++ b/glitch/rego/queries/security/sec_full_permission_filesystem.rego @@ -0,0 +1,35 @@ +package glitch + +import data.glitch_lib + +mode_set := {"mode", "m"} + +check_permission(value) { + value.ir_type == "String" + regex.match("(?:^0?777$)|(?:(?:^|(?:ugo)|o|a)\\+[rwx]{3})", value.value) +} else { + value.ir_type == "Integer" + value.value == 777 +} + +Glitch_Analysis[result] { + parent := glitch_lib._gather_parent_unit_blocks[_] + parent.path != "" + atomic_units := glitch_lib.all_atomic_units(parent) + node := atomic_units[_] + + node.type == data.security.file_commands[_] + + attrs := glitch_lib.all_attributes(node) + attr := attrs[_] + + attr.name == mode_set[_] + check_permission(attr.value) + + result := {{ + "type": "sec_full_permission_filesystem", + "element": attr, + "path": parent.path, + "description": "Full permission to the filesystem - Files should not have full permissions to every user. (CWE-732)" + }} +} diff --git a/glitch/rego/queries/security/sec_hard_pass.rego b/glitch/rego/queries/security/sec_hard_pass.rego new file mode 100644 index 00000000..07845955 --- /dev/null +++ b/glitch/rego/queries/security/sec_hard_pass.rego @@ -0,0 +1,80 @@ +package glitch + +import data.glitch_lib + +whitelist_contains(name) { + whitelist := array.concat(data.security.secrets_white_list, data.security.profile) + whitelist[_] == name +} + +check_pair_hard_password(name, value) { + hardcoded := data.security.passwords + + item := hardcoded[_] + hard_coded_pattern := sprintf("[_A-Za-z0-9$/\\.\\[\\]-]*%s\\b", [item]) + + regex.match(hard_coded_pattern, lower(name)) + + not whitelist_contains(lower(name)) + + glitch_lib.traverse_var(value) + + value.value != "" +} else { + # Check for sensitive data with secret value assignments + sensitive_item := data.security.sensitive_data[_] + glitch_lib.contains(lower(name), lower(sensitive_item)) + + + secret_value := data.security.secret_value_assign[_] + glitch_lib.contains(lower(value.value), lower(secret_value)) + + # Exclude password cases (those will be handled by sec_hard_pass) + glitch_lib.contains(lower(secret_value), "password") +} + +Glitch_Analysis[result] { + parent := glitch_lib._gather_parent_unit_blocks[_] + parent.path != "" + attr := glitch_lib.all_attributes(parent) + variables := glitch_lib.all_variables(parent) + all_nodes := attr | variables + node := all_nodes[_] + + # We need to use walk since we can have Hashs inside one another + walk(node, [_, n]) + glitch_lib.is_ir_type_in(node.value, ["String", "Array"]) + check_pair_hard_password(node.name, node.value) + matched_node := node + + result := {{ + "type": "sec_hard_pass", + "element": matched_node, + "path": parent.path, + "description": "Hard-coded password - Developers should not reveal sensitive information in the source code. (CWE-259)" + }} +} + +Glitch_Analysis[result] { + parent := glitch_lib._gather_parent_unit_blocks[_] + parent.path != "" + attr := glitch_lib.all_attributes(parent) + variables := glitch_lib.all_variables(parent) + all_nodes := attr | variables + node := all_nodes[_] + + # We need to use walk since we can have Hashs inside one another + walk(node, [_, n]) + n.ir_type == "Hash" + current_pair := n.value[_] + glitch_lib.is_ir_type_in(current_pair.value, ["String", "Array"]) + check_pair_hard_password(current_pair.key.value, current_pair.value) + matched_node := current_pair + + result := {{ + "type": "sec_hard_pass", + "element": matched_node, + "path": parent.path, + "description": "Hard-coded password - Developers should not reveal sensitive information in the source code. (CWE-259)" + }} +} diff --git a/glitch/rego/queries/security/sec_hard_secr.rego b/glitch/rego/queries/security/sec_hard_secr.rego new file mode 100644 index 00000000..184e04c7 --- /dev/null +++ b/glitch/rego/queries/security/sec_hard_secr.rego @@ -0,0 +1,112 @@ +package glitch + +import data.glitch_lib + +whitelist_contains(name) { + whitelist := array.concat(data.security.secrets_white_list, data.security.profile) + whitelist[_] == name +} + +# Flatten variable names by converting brackets and quotes to dots +flatten_name(name) = flat_name { + step1 := replace(name, "['", ".") + step2 := replace(step1, "']", "") + step3 := replace(step2, "[\"", ".") + step4 := replace(step3, "\"]", "") + step5 := replace(step4, "[", ".") + step6 := replace(step5, "]", "") + + # Clean up multiple dots + step7 := replace(step6, "..", ".") + flat_name := trim(step7, ".") +} + +check_pair_hard_secr(name, value) { + hardcoded := data.security.secrets + + item := hardcoded[_] + hard_coded_pattern := sprintf("[_A-Za-z0-9$/\\.\\[\\]-]*%s\\b", [item]) + + regex.match(hard_coded_pattern, lower(name)) + + not whitelist_contains(lower(name)) + + glitch_lib.traverse_var(value) +} else { + item := lower(data.security.ssh_dirs[_]) + + glitch_lib.contains(lower(name), item) + + pattern := ".*\\/id_rsa.*" + + glitch_lib.traverse(value, pattern) +} else { + # Check for sensitive data with secret value assignments + sensitive_item := data.security.sensitive_data[_] + glitch_lib.contains(lower(name), lower(sensitive_item)) + + secret_value := data.security.secret_value_assign[_] + glitch_lib.contains(lower(value.value), lower(secret_value)) + + # Exclude password cases (those will be handled by sec_hard_pass) + not glitch_lib.contains(lower(secret_value), "password") + +} else { + item := data.security.misc_secrets[_] + flat_name := flatten_name(name) + + pattern := sprintf( + "([_A-Za-z0-9$-]*[-_]%s([-_].*)?$)|(^%s([-_].*)?$)", + [item, item] + ) + + regex.match(pattern, flat_name) + value.value != "" + glitch_lib.traverse_var(value) +} + +Glitch_Analysis[result] { + parent := glitch_lib._gather_parent_unit_blocks[_] + parent.path != "" + attr := glitch_lib.all_attributes(parent) + variables := glitch_lib.all_variables(parent) + all_nodes := attr | variables + node := all_nodes[_] + + # We need to use walk since we can have Hashs inside one another + walk(node, [_, n]) + glitch_lib.is_ir_type_in(node.value, ["String", "Array"]) + check_pair_hard_secr(node.name, node.value) + matched_node := node + + result := {{ + "type": "sec_hard_secr", + "element": matched_node, + "path": parent.path, + "description": "Hard-coded secret - Developers should not reveal sensitive information in the source code. (CWE-798)" + }} +} + +Glitch_Analysis[result] { + parent := glitch_lib._gather_parent_unit_blocks[_] + parent.path != "" + attr := glitch_lib.all_attributes(parent) + variables := glitch_lib.all_variables(parent) + all_nodes := attr | variables + node := all_nodes[_] + + # We need to use walk since we can have Hashs inside one another + walk(node, [_, n]) + n.ir_type == "Hash" + current_pair := n.value[_] + glitch_lib.is_ir_type_in(current_pair.value, ["String", "Array"]) + check_pair_hard_secr(current_pair.key.value, current_pair.value) + matched_node := current_pair + + result := {{ + "type": "sec_hard_secr", + "element": matched_node, + "path": parent.path, + "description": "Hard-coded secret - Developers should not reveal sensitive information in the source code. (CWE-798)" + }} +} \ No newline at end of file diff --git a/glitch/rego/queries/security/sec_hard_user.rego b/glitch/rego/queries/security/sec_hard_user.rego new file mode 100644 index 00000000..6a6eca5c --- /dev/null +++ b/glitch/rego/queries/security/sec_hard_user.rego @@ -0,0 +1,68 @@ +package glitch + +import data.glitch_lib + +whitelist_contains(name) { + whitelist := array.concat(data.security.secrets_white_list, data.security.profile) + whitelist[_] == name +} + + +check_pair_hard_users(name, value) { + hardcoded := data.security.users + + item := hardcoded[_] + hard_coded_pattern := sprintf("[_A-Za-z0-9$/\\.\\[\\]-]*%s\\b", [item]) + + regex.match(hard_coded_pattern, lower(name)) + + not whitelist_contains(lower(name)) + + glitch_lib.traverse_var(value) +} + +Glitch_Analysis[result] { + parent := glitch_lib._gather_parent_unit_blocks[_] + parent.path != "" + attr := glitch_lib.all_attributes(parent) + variables := glitch_lib.all_variables(parent) + all_nodes := attr | variables + node := all_nodes[_] + + # We need to use walk since we can have Hashs inside one another + walk(node, [_, n]) + glitch_lib.is_ir_type_in(node.value, ["String", "Null", "Array"]) + check_pair_hard_users(node.name, node.value) + matched_node := node + + result := {{ + "type": "sec_hard_user", + "element": matched_node, + "path": parent.path, + "description": "Hard-coded user - Developers should not reveal sensitive information in the source code. (CWE-798)" + }} +} + +Glitch_Analysis[result] { + parent := glitch_lib._gather_parent_unit_blocks[_] + parent.path != "" + attr := glitch_lib.all_attributes(parent) + variables := glitch_lib.all_variables(parent) + all_nodes := attr | variables + node := all_nodes[_] + + # We need to use walk since we can have Hashs inside one another + walk(node, [_, n]) + n.ir_type == "Hash" + current_pair := n.value[_] + glitch_lib.is_ir_type_in(current_pair.value, ["String", "Null", "Array"]) + check_pair_hard_users(current_pair.key.value, current_pair.value) + matched_node := current_pair + + result := {{ + "type": "sec_hard_user", + "element": matched_node, + "path": parent.path, + "description": "Hard-coded user - Developers should not reveal sensitive information in the source code. (CWE-798)" + }} +} \ No newline at end of file diff --git a/glitch/rego/queries/security/sec_https.rego b/glitch/rego/queries/security/sec_https.rego new file mode 100644 index 00000000..1d8ec5c9 --- /dev/null +++ b/glitch/rego/queries/security/sec_https.rego @@ -0,0 +1,121 @@ +package glitch + +import data.glitch_lib + +url_regex := "^(http:\\/\\/www\\.|https:\\/\\/www\\.|http:\\/\\/|https:\\/\\/)?(?:[a-zA-Z0-9]+([_\\-\\.][a-zA-Z0-9]+)*\\.[a-zA-Z]{2,}|(?:[0-9]{1,3}\\.){3}[0-9]{1,3})(:[0-9]{1,5})?(\\/.*)?$" + +normalize_url(url) = out { + # Remove backslash line breaks - use replace instead of regex.replace + no_breaks := replace(url, "\\\n", "") + + # Remove internal quotes + no_quotes := replace(no_breaks, "'", "") + no_quotes2 := replace(no_quotes, "\"", "") + + # Collapse whitespace + out := replace(no_quotes2, " ", "") +} +# In this case, I am assuming that link strings are either in a Sum node or String node +check_http_without_tls(node) { + node.ir_type == "Sum" + url := lower(normalize_url(node.code)) + url_is_http(url) + not url_is_whitelisted(url) +} else { + node.ir_type == "String" + url_is_http(node.value) + not url_is_whitelisted(node.value) +} + +# Check if string contains template variables like {{ var }}, $var, ${var}, etc. +has_template_variable(url) { + regex.match(".*\\{\\{.*\\}\\}.*", url) # Matches {{ ... }} +} else { + regex.match(".*\\$\\{.*\\}.*", url) # Matches ${ ... } +} else { + regex.match(".*\\$[a-zA-Z_][a-zA-Z0-9_]*.*", url) # Matches $var +} + +url_is_http(url) { + # If it has template variables, skip regex validation + has_template_variable(url) + url_has_http_or_www(lower(url)) + not glitch_lib.contains(url, "https://") +} else { + # If no template variables, use full regex validation + regex.match(url_regex, lower(url)) + url_has_http_or_www(lower(url)) + not glitch_lib.contains(url, "https://") +} + +url_has_http_or_www(url) { + glitch_lib.contains(url, "http://") +} else { + glitch_lib.contains(url, "www") +} + +url_is_whitelisted(url) { + host := extract_hostname(url) + host == data.security.url_http_white_list[_] +} + +extract_hostname(value) = output { + parts := split(value, "://") + rest := select_rest(parts, value) + host_port := split(rest, "/")[0] + output := split(host_port, ":")[0] +} else = "" { + true +} + +select_rest(parts, original) = output { + count(parts) > 1 + output := parts[1] +} else = output { + output := trim_left(original, "/") +} + +Glitch_Analysis[result] { + parent := glitch_lib._gather_parent_unit_blocks[_] + parent.path != "" + attr := glitch_lib.all_attributes(parent) + variables := glitch_lib.all_variables(parent) + all_nodes := attr | variables + node := all_nodes[_] + + # We need to use walk since we can have Hashs inside one another + walk(node, [_, n]) + n.value.ir_type != "Hash" + check_http_without_tls(n.value) + matched_node := n + + result := {{ + "type": "sec_https", + "element": matched_node, + "path": parent.path, + "description": "Use of HTTP without TLS - The developers should always favor the usage of HTTPS. (CWE-319)" + }} +} + +Glitch_Analysis[result] { + parent := glitch_lib._gather_parent_unit_blocks[_] + parent.path != "" + attr := glitch_lib.all_attributes(parent) + variables := glitch_lib.all_variables(parent) + all_nodes := attr | variables + node := all_nodes[_] + + # We need to use walk since we can have Hashs inside one another + walk(node, [_, n]) + n.ir_type == "Hash" + current_pair := n.value[_] + check_http_without_tls(current_pair.value) + matched_node := current_pair + + result := {{ + "type": "sec_https", + "element": matched_node, + "path": parent.path, + "description": "Use of HTTP without TLS - The developers should always favor the usage of HTTPS. (CWE-319)" + }} +} \ No newline at end of file diff --git a/glitch/rego/queries/security/sec_invalid_bind.rego b/glitch/rego/queries/security/sec_invalid_bind.rego new file mode 100644 index 00000000..ae43433d --- /dev/null +++ b/glitch/rego/queries/security/sec_invalid_bind.rego @@ -0,0 +1,89 @@ +package glitch + +import data.glitch_lib + +ipv6_not_allowed_strings := {"*", "::"} + +check_inv_bind(name, value) { + # Check the value of the attribute is 0.0.0.0 + glitch_lib.traverse(value, "^(https?://)?0\\.0\\.0\\.0.*") +} else { + # Check if it is a ipv6 wildcard with the name of ip + name == "ip" + glitch_lib.traverse(value, ipv6_not_allowed_strings) +} else { + # Check if it is a ipv6 wildcard with the name in config + name == data.security.ip_binding_commands[_] + glitch_lib.traverse(value, ipv6_not_allowed_strings) +} else { + # Check if it is a boolean with the name in config + value.ir_type == "Boolean" + name == data.security.ip_binding_commands[_] + value.value == "true" +} + +Glitch_Analysis[result] { + parent := glitch_lib._gather_parent_unit_blocks[_] + parent.path != "" + attr := glitch_lib.all_attributes(parent) + variables := glitch_lib.all_variables(parent) + all_nodes := attr | variables + node := all_nodes[_] + + glitch_lib.is_ir_type_in(node.value, ["String"]) + check_inv_bind(node.name, node.value) + matched_node := node + + result := {{ + "type": "sec_invalid_bind", + "element": matched_node, + "path": parent.path, + "description": "Invalid IP address binding - Binding to the address 0.0.0.0 allows connections from every possible network which might be a security issues. (CWE-284)" + }} +} + +Glitch_Analysis[result] { + parent := glitch_lib._gather_parent_unit_blocks[_] + parent.path != "" + attr := glitch_lib.all_attributes(parent) + variables := glitch_lib.all_variables(parent) + all_nodes := attr | variables + node := all_nodes[_] + + walk(node, [_, n]) + n.ir_type == "Hash" + current_pair := n.value[_] + glitch_lib.is_ir_type_in(current_pair.value, ["String"]) + check_inv_bind(current_pair.key.value, current_pair.value) + matched_node := current_pair + + result := {{ + "type": "sec_invalid_bind", + "element": matched_node, + "path": parent.path, + "description": "Invalid IP address binding - Binding to the address 0.0.0.0 allows connections from every possible network which might be a security issues. (CWE-284)" + }} +} + +# Check for invalid bindings in Array values +Glitch_Analysis[result] { + parent := glitch_lib._gather_parent_unit_blocks[_] + parent.path != "" + attr := glitch_lib.all_attributes(parent) + variables := glitch_lib.all_variables(parent) + all_nodes := attr | variables + node := all_nodes[_] + + node.value.ir_type == "Array" + array_item := node.value.value[_] + glitch_lib.is_ir_type_in(array_item, ["String"]) + check_inv_bind(node.name, array_item) + matched_node := node + + result := {{ + "type": "sec_invalid_bind", + "element": matched_node, + "path": parent.path, + "description": "Invalid IP address binding - Binding to the address 0.0.0.0 allows connections from every possible network which might be a security issues. (CWE-284)" + }} +} \ No newline at end of file diff --git a/glitch/rego/queries/security/sec_no_default_switch.rego b/glitch/rego/queries/security/sec_no_default_switch.rego new file mode 100644 index 00000000..84dcdf14 --- /dev/null +++ b/glitch/rego/queries/security/sec_no_default_switch.rego @@ -0,0 +1,41 @@ +package glitch + +import data.glitch_lib + +path_contains_statements(path) { + some i + path[i] == "statements" +} + +has_default_case(node) { + some path, value + walk(node, [path, value]) + count(path) > 0 + last := path[count(path) - 1] + last == "is_default" + value == true + not path_contains_statements(path) +} + +check_missing_default(node) { + not has_default_case(node) +} + +Glitch_Analysis[result] { + parent := glitch_lib._gather_parent_unit_blocks[_] + parent.path != "" + conditions := glitch_lib.all_conditional_statements(parent) + node := conditions[_] + + node.is_top == true + node.type == "SWITCH" + + check_missing_default(node) + + result := {{ + "type": "sec_no_default_switch", + "element": node, + "path": parent.path, + "description": "Missing default case statement - Not handling every possible input combination might allow an attacker to trigger an error for an unhandled value. (CWE-478)" + }} +} \ No newline at end of file diff --git a/glitch/rego/queries/security/sec_no_int_check.rego b/glitch/rego/queries/security/sec_no_int_check.rego new file mode 100644 index 00000000..a52443d3 --- /dev/null +++ b/glitch/rego/queries/security/sec_no_int_check.rego @@ -0,0 +1,77 @@ +package glitch + +import data.glitch_lib + +attr_has_any_checksum(attr, checksum_values) { + check := checksum_values[_] + pattern := sprintf("(?i).*%s.*", [check]) + # We use traverse so it can find all strings and test them inside + glitch_lib.traverse(attr, pattern) +} + +check_integrity_check_atomic_unit(node) { + attributes = glitch_lib.all_attributes(node) + + attr = attributes[_] + + download = data.security.download_extensions[_] + + # We use code, since it can be a SUM atribute + regex.match(sprintf("(http|https|www)[^ ,]*\\.%s", [download]), attr.code) + + checksum_values = data.security.checksum + + attributes_without_checksum := {attr | + attr := attributes[_] + not attr_has_any_checksum(attr, checksum_values) + } + + # Trigger integrity check only if ALL attributes lack a checksum keyword + count(attributes_without_checksum) == count(attributes) +} + +check_integrity_check_keyvalues(node) { + value = data.security.checksum[_] + glitch_lib.contains(node.name, value) + false_pattern = "^(?i)(no|false)$" + glitch_lib.traverse(node, false_pattern) +} else { + # This case is repeated since for puppet it becames a boolean and ansible a string + value = data.security.checksum[_] + glitch_lib.contains(node.name, value) + glitch_lib.traverse(node, false) +} + +Glitch_Analysis[result] { + parent := glitch_lib._gather_parent_unit_blocks[_] + parent.path != "" + atomic_units := glitch_lib.all_atomic_units(parent) + node := atomic_units[_] + + check_integrity_check_atomic_unit(node) + + result := { + "type": "sec_no_int_check", + "element": node, + "path": parent.path, + "description": "No integrity check - The content of files downloaded from the internet should be checked. (CWE-353)" + } +} + +Glitch_Analysis[result] { + parent := glitch_lib._gather_parent_unit_blocks[_] + parent.path != "" + attr := glitch_lib.all_attributes(parent) + variables := glitch_lib.all_variables(parent) + all_nodes := attr | variables + node := all_nodes[_] + + check_integrity_check_keyvalues(node) + + result := { + "type": "sec_no_int_check", + "element": node, + "path": parent.path, + "description": "No integrity check - The content of files downloaded from the internet should be checked. (CWE-353)" + } +} diff --git a/glitch/rego/queries/security/sec_obsolete_command.rego b/glitch/rego/queries/security/sec_obsolete_command.rego new file mode 100644 index 00000000..3b26a9cd --- /dev/null +++ b/glitch/rego/queries/security/sec_obsolete_command.rego @@ -0,0 +1,40 @@ +package glitch + +import data.glitch_lib + +Glitch_Analysis[result] { + parent := glitch_lib._gather_parent_unit_blocks[_] + parent.path != "" + atomic_units := glitch_lib.all_atomic_units(parent) + node := atomic_units[_] + + node.type == data.security.obsolete_commands[_] + + result := { + "type": "sec_obsolete_command", + "element": node, + "path": parent.path, + "description": "Use of obsolete command or function - Avoid using obsolete or deprecated commands and functions. (CWE-477)" + } +} + +Glitch_Analysis[result] { + parent := glitch_lib._gather_parent_unit_blocks[_] + parent.path != "" + atomic_units := glitch_lib.all_atomic_units(parent) + node := atomic_units[_] + + endswith(node.type, data.security.shell_resources[_]) + attrs := glitch_lib.all_attributes(node) + attr := attrs[_] + attr.value.ir_type == "String" + cmd := split(attr.value.value, " ")[0] + cmd == data.security.obsolete_commands[_] + + result := { + "type": "sec_obsolete_command", + "element": attr, + "path": parent.path, + "description": "Use of obsolete command or function - Avoid using obsolete or deprecated commands and functions. (CWE-477)" + } +} \ No newline at end of file diff --git a/glitch/rego/queries/security/sec_susp_comm.rego b/glitch/rego/queries/security/sec_susp_comm.rego new file mode 100644 index 00000000..85009b30 --- /dev/null +++ b/glitch/rego/queries/security/sec_susp_comm.rego @@ -0,0 +1,31 @@ +package glitch + +import data.glitch_lib + +check_susp_comment(comment) { + lines := split(comment.content, "\n") + + line := lines[_] + + lower_line := lower(line) + + element := data.security.suspicious_words[_] + pattern := sprintf(".*%s.*", [element]) + + regex.match(pattern, lower_line) +} + +Glitch_Analysis[result] { + parent := glitch_lib._gather_parent_unit_blocks[_] + parent.path != "" + comment := parent.comments[_] + + check_susp_comment(comment) + + result := {{ + "type": "sec_susp_comm", + "element": comment, + "path": parent.path, + "description": "Suspicious comment - Comments with keywords such as TODO HACK or FIXME may reveal problems possibly exploitable. (CWE-546)" + }} +} diff --git a/glitch/rego/queries/security/sec_weak_crypt.rego b/glitch/rego/queries/security/sec_weak_crypt.rego new file mode 100644 index 00000000..c5ba7a01 --- /dev/null +++ b/glitch/rego/queries/security/sec_weak_crypt.rego @@ -0,0 +1,71 @@ +package glitch + +import data.glitch_lib + +name_in_whitelist(name) { + glitch_lib.contains(name, data.security.weak_crypt_whitelist[_]) +} + +check_weak_crypt(value, name) { + glitch_lib.traverse(value, data.security.weak_crypt) + not glitch_lib.traverse(value, data.security.weak_crypt_whitelist) + not name_in_whitelist(name) +} + +Glitch_Analysis[result] { + parent := glitch_lib._gather_parent_unit_blocks[_] + parent.path != "" + attr := glitch_lib.all_attributes(parent) + variables := glitch_lib.all_variables(parent) + all_nodes := attr | variables + node := all_nodes[_] + + node.value.ir_type != "Hash" + check_weak_crypt(node.value, node.name) + matched_node := node + + result := { + "type": "sec_weak_crypt", + "element": matched_node, + "path": parent.path, + "description": "Weak Crypto Algorithm - Weak crypto algorithms should be avoided since they are susceptible to security issues. (CWE-326 | CWE-327)" + } +} + +Glitch_Analysis[result] { + parent := glitch_lib._gather_parent_unit_blocks[_] + parent.path != "" + attr := glitch_lib.all_attributes(parent) + variables := glitch_lib.all_variables(parent) + all_nodes := attr | variables + node := all_nodes[_] + + walk(node, [_, n]) + n.ir_type == "Hash" + current_pair := n.value[_] + check_weak_crypt(current_pair.value, current_pair.key.value) + matched_node := current_pair + + result := { + "type": "sec_weak_crypt", + "element": matched_node, + "path": parent.path, + "description": "Weak Crypto Algorithm - Weak crypto algorithms should be avoided since they are susceptible to security issues. (CWE-326 | CWE-327)" + } +} + +Glitch_Analysis[result] { + parent := glitch_lib._gather_parent_unit_blocks[_] + parent.path != "" + atomic_units := glitch_lib.all_atomic_units(parent) + node := atomic_units[_] + + check_weak_crypt(node.type, node.name) + + result := { + "type": "sec_weak_crypt", + "element": node, + "path": parent.path, + "description": "Weak Crypto Algorithm - Weak crypto algorithms should be avoided since they are susceptible to security issues. (CWE-326 | CWE-327)" + } +} diff --git a/glitch/rego/rego_python/LICENSE b/glitch/rego/rego_python/LICENSE new file mode 100644 index 00000000..e72bfdda --- /dev/null +++ b/glitch/rego/rego_python/LICENSE @@ -0,0 +1,674 @@ + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. \ No newline at end of file diff --git a/glitch/rego/rego_python/README.md b/glitch/rego/rego_python/README.md new file mode 100644 index 00000000..9318bfda --- /dev/null +++ b/glitch/rego/rego_python/README.md @@ -0,0 +1,84 @@ +# rego_python + +Python wrapper around a Go-based [OPA (Open Policy Agent)](https://www.openpolicyagent.org/) Rego library. +This package provides a simple interface to evaluate Rego policies from Python without requiring users to build the Go library themselves. + +## Installation + +```bash +pip install rego_python +``` +## Supported Architectures +We currently ship precompiled binaries for the following platforms: +- Linux (x86_64) + + + +## Usage + +The main entrypoint is the function ```run_rego```, which executes Rego policies against given input and data. + +``` +from rego_python import run_rego + +result = run_rego( + input_data={"user": "alice"}, + data={"roles": {"alice": ["admin"]}}, + rego_modules={ + "policy.rego": """ + package example + + default allow = false + + allow { + input.user == "alice" + data.roles[input.user][_] == "admin" + } + """ + } +) + +print(result) +``` + +## API Reference + +``` +run_rego(input_data: dict, data: dict, rego_modules: dict) -> dict +``` + +Executes a Rego policy using the underlying Go library and returns the evaluation result as a Python dictionary. + +### Parameters: +- input_data (dict) + Represents the input document passed to the Rego evaluation. + Example: + ``` + {"user": "alice"} + ``` + +- data (dict) + Represents contextual data available to the policy during evaluation. + Example: + ``` + {"roles": {"alice": ["admin"]}} + ``` + +- rego_modules (dict) + A mapping of file names to Rego source code strings. Each key is a filename and the value is the Rego policy source code. + Example: + ``` + {"policy.rego": "package example\nallow { input.user == \"alice\" }"} + + ``` + +### Returns: +- dict + The evaluation result, parsed from the JSON response returned by the Go library. + +## Source Code: +You can view the source code [on GitHub](https://github.com/infragov-project/GLITCH/tree/rego_integration/glitch/rego/rego_python). + + + + diff --git a/glitch/rego/rego_python/pyproject.toml b/glitch/rego/rego_python/pyproject.toml new file mode 100644 index 00000000..1356b5db --- /dev/null +++ b/glitch/rego/rego_python/pyproject.toml @@ -0,0 +1,21 @@ +[build-system] +requires = ["setuptools>=61.0", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "rego_python" +version = "0.2.0" +description = "Python wrapper for Rego engine" +authors = [ + {name = "Daniel Carvalho", email = "daniel.m.carvalho2002@gmail.com"}, + {name = "Rodrigo Coelho", email = "racoelhosilva@gmail.com"} +] +license = {text = "GNU GENERAL PUBLIC LICENSE v3.0"} +readme = "README.md" +requires-python = ">=3.12" +dependencies = [] + +[tool.setuptools.package-data] +"rego_python" = [ + "bin/**/*" # include all compiled binaries +] diff --git a/glitch/rego/rego_python/src/rego_python/__init__.py b/glitch/rego/rego_python/src/rego_python/__init__.py new file mode 100644 index 00000000..ef438783 --- /dev/null +++ b/glitch/rego/rego_python/src/rego_python/__init__.py @@ -0,0 +1,3 @@ +from .wrapper import run_rego, is_rego_available, get_rego_error + +__all__ = ["run_rego", "is_rego_available", "get_rego_error"] diff --git a/glitch/tests/design/ansible/__init__.py b/glitch/rego/rego_python/src/rego_python/bin/.gitkeep similarity index 100% rename from glitch/tests/design/ansible/__init__.py rename to glitch/rego/rego_python/src/rego_python/bin/.gitkeep diff --git a/glitch/rego/rego_python/src/rego_python/go/go.mod b/glitch/rego/rego_python/src/rego_python/go/go.mod new file mode 100644 index 00000000..3143cd84 --- /dev/null +++ b/glitch/rego/rego_python/src/rego_python/go/go.mod @@ -0,0 +1,52 @@ +module librego + +go 1.24.6 + +require github.com/open-policy-agent/opa v1.12.3 + +require ( + github.com/agnivade/levenshtein v1.2.1 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 // indirect + github.com/go-ini/ini v1.67.0 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/gobwas/glob v0.2.3 // indirect + github.com/goccy/go-json v0.10.5 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/lestrrat-go/blackmagic v1.0.4 // indirect + github.com/lestrrat-go/dsig v1.0.0 // indirect + github.com/lestrrat-go/dsig-secp256k1 v1.0.0 // indirect + github.com/lestrrat-go/httpcc v1.0.1 // indirect + github.com/lestrrat-go/httprc/v3 v3.0.1 // indirect + github.com/lestrrat-go/jwx/v3 v3.0.12 // indirect + github.com/lestrrat-go/option v1.0.1 // indirect + github.com/lestrrat-go/option/v2 v2.0.0 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/prometheus/client_golang v1.23.2 // indirect + github.com/prometheus/client_model v0.6.2 // indirect + github.com/prometheus/common v0.66.1 // indirect + github.com/prometheus/procfs v0.17.0 // indirect + github.com/rcrowley/go-metrics v0.0.0-20250401214520-65e299d6c5c9 // indirect + github.com/segmentio/asm v1.2.1 // indirect + github.com/sirupsen/logrus v1.9.4-0.20230606125235-dd1b4c2e81af // indirect + github.com/tchap/go-patricia/v2 v2.3.3 // indirect + github.com/valyala/fastjson v1.6.4 // indirect + github.com/vektah/gqlparser/v2 v2.5.31 // indirect + github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect + github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect + github.com/yashtewari/glob-intersection v0.2.0 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/otel v1.38.0 // indirect + go.opentelemetry.io/otel/metric v1.38.0 // indirect + go.opentelemetry.io/otel/sdk v1.38.0 // indirect + go.opentelemetry.io/otel/trace v1.38.0 // indirect + go.yaml.in/yaml/v2 v2.4.2 // indirect + golang.org/x/crypto v0.45.0 // indirect + golang.org/x/sync v0.18.0 // indirect + golang.org/x/sys v0.38.0 // indirect + google.golang.org/protobuf v1.36.10 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + sigs.k8s.io/yaml v1.6.0 // indirect +) diff --git a/glitch/rego/rego_python/src/rego_python/go/go.sum b/glitch/rego/rego_python/src/rego_python/go/go.sum new file mode 100644 index 00000000..67ffadb5 --- /dev/null +++ b/glitch/rego/rego_python/src/rego_python/go/go.sum @@ -0,0 +1,179 @@ +github.com/agnivade/levenshtein v1.2.1 h1:EHBY3UOn1gwdy/VbFwgo4cxecRznFk7fKWN1KOX7eoM= +github.com/agnivade/levenshtein v1.2.1/go.mod h1:QVVI16kDrtSuwcpd0p1+xMC6Z/VfhtCyDIjcwga4/DU= +github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883 h1:bvNMNQO63//z+xNgfBlViaCIJKLlCJ6/fmUseuG0wVQ= +github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= +github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0 h1:jfIu9sQUG6Ig+0+Ap1h4unLjW6YQJpKZVmUzxsD4E/Q= +github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0/go.mod h1:t2tdKJDJF9BV14lnkjHmOQgcvEKgtqs5a1N3LNdJhGE= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bytecodealliance/wasmtime-go/v39 v39.0.1 h1:RibaT47yiyCRxMOj/l2cvL8cWiWBSqDXHyqsa9sGcCE= +github.com/bytecodealliance/wasmtime-go/v39 v39.0.1/go.mod h1:miR4NYIEBXeDNamZIzpskhJ0z/p8al+lwMWylQ/ZJb4= +github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= +github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 h1:NMZiJj8QnKe1LgsbDayM4UoHwbvwDRwnI3hwNaAHRnc= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0/go.mod h1:ZXNYxsqcloTdSy/rNShjYzMhyjf0LaoftYK0p+A3h40= +github.com/dgraph-io/badger/v4 v4.8.0 h1:JYph1ChBijCw8SLeybvPINizbDKWZ5n/GYbz2yhN/bs= +github.com/dgraph-io/badger/v4 v4.8.0/go.mod h1:U6on6e8k/RTbUWxqKR0MvugJuVmkxSNc79ap4917h4w= +github.com/dgraph-io/ristretto/v2 v2.2.0 h1:bkY3XzJcXoMuELV8F+vS8kzNgicwQFAaGINAEJdWGOM= +github.com/dgraph-io/ristretto/v2 v2.2.0/go.mod h1:RZrm63UmcBAaYWC1DotLYBmTvgkrs0+XhBd7Npn7/zI= +github.com/dgryski/trifles v0.0.0-20230903005119-f50d829f2e54 h1:SG7nF6SRlWhcT7cNTs5R6Hk4V2lcmLz2NsG2VnInyNo= +github.com/dgryski/trifles v0.0.0-20230903005119-f50d829f2e54/go.mod h1:if7Fbed8SFyPtHLHbg49SI7NAdJiC5WIA09pe59rfAA= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= +github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= +github.com/foxcpp/go-mockdns v1.1.0 h1:jI0rD8M0wuYAxL7r/ynTrCQQq0BVqfB99Vgk7DlmewI= +github.com/foxcpp/go-mockdns v1.1.0/go.mod h1:IhLeSFGed3mJIAXPH2aiRQB+kqz7oqu8ld2qVbOu7Wk= +github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= +github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= +github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= +github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= +github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/google/flatbuffers v25.2.10+incompatible h1:F3vclr7C3HpB1k9mxCGRMXq6FdUalZ6H/pNX4FP1v0Q= +github.com/google/flatbuffers v25.2.10+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2 h1:8Tjv8EJ+pM1xP8mK6egEbD1OgnVTyacbefKhmbLhIhU= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2/go.mod h1:pkJQ2tZHJ0aFOVEEot6oZmaVEZcRme73eIFmhiVuRWs= +github.com/klauspost/compress v1.18.1 h1:bcSGx7UbpBqMChDtsF28Lw6v/G94LPrrbMbdC3JH2co= +github.com/klauspost/compress v1.18.1/go.mod h1:ZQFFVG+MdnR0P+l6wpXgIL4NTtwiKIdBnrBd8Nrxr+0= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/lestrrat-go/blackmagic v1.0.4 h1:IwQibdnf8l2KoO+qC3uT4OaTWsW7tuRQXy9TRN9QanA= +github.com/lestrrat-go/blackmagic v1.0.4/go.mod h1:6AWFyKNNj0zEXQYfTMPfZrAXUWUfTIZ5ECEUEJaijtw= +github.com/lestrrat-go/dsig v1.0.0 h1:OE09s2r9Z81kxzJYRn07TFM9XA4akrUdoMwr0L8xj38= +github.com/lestrrat-go/dsig v1.0.0/go.mod h1:dEgoOYYEJvW6XGbLasr8TFcAxoWrKlbQvmJgCR0qkDo= +github.com/lestrrat-go/dsig-secp256k1 v1.0.0 h1:JpDe4Aybfl0soBvoVwjqDbp+9S1Y2OM7gcrVVMFPOzY= +github.com/lestrrat-go/dsig-secp256k1 v1.0.0/go.mod h1:CxUgAhssb8FToqbL8NjSPoGQlnO4w3LG1P0qPWQm/NU= +github.com/lestrrat-go/httpcc v1.0.1 h1:ydWCStUeJLkpYyjLDHihupbn2tYmZ7m22BGkcvZZrIE= +github.com/lestrrat-go/httpcc v1.0.1/go.mod h1:qiltp3Mt56+55GPVCbTdM9MlqhvzyuL6W/NMDA8vA5E= +github.com/lestrrat-go/httprc/v3 v3.0.1 h1:3n7Es68YYGZb2Jf+k//llA4FTZMl3yCwIjFIk4ubevI= +github.com/lestrrat-go/httprc/v3 v3.0.1/go.mod h1:2uAvmbXE4Xq8kAUjVrZOq1tZVYYYs5iP62Cmtru00xk= +github.com/lestrrat-go/jwx/v3 v3.0.12 h1:p25r68Y4KrbBdYjIsQweYxq794CtGCzcrc5dGzJIRjg= +github.com/lestrrat-go/jwx/v3 v3.0.12/go.mod h1:HiUSaNmMLXgZ08OmGBaPVvoZQgJVOQphSrGr5zMamS8= +github.com/lestrrat-go/option v1.0.1 h1:oAzP2fvZGQKWkvHa1/SAcFolBEca1oN+mQ7eooNBEYU= +github.com/lestrrat-go/option v1.0.1/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I= +github.com/lestrrat-go/option/v2 v2.0.0 h1:XxrcaJESE1fokHy3FpaQ/cXW8ZsIdWcdFzzLOcID3Ss= +github.com/lestrrat-go/option/v2 v2.0.0/go.mod h1:oSySsmzMoR0iRzCDCaUfsCzxQHUEuhOViQObyy7S6Vg= +github.com/miekg/dns v1.1.57 h1:Jzi7ApEIzwEPLHWRcafCN9LZSBbqQpxjt/wpgvg7wcM= +github.com/miekg/dns v1.1.57/go.mod h1:uqRjCRUuEAA6qsOiJvDd+CFo/vW+y5WR6SNmHE55hZk= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/open-policy-agent/opa v1.12.3 h1:qe3m/w52baKC/HJtippw+hYBUKCzuBCPjB+D5P9knfc= +github.com/open-policy-agent/opa v1.12.3/go.mod h1:RnDgm04GA1RjEXJvrsG9uNT/+FyBNmozcPvA2qz60M4= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= +github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= +github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= +github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= +github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs= +github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA= +github.com/prometheus/procfs v0.17.0 h1:FuLQ+05u4ZI+SS/w9+BWEM2TXiHKsUQ9TADiRH7DuK0= +github.com/prometheus/procfs v0.17.0/go.mod h1:oPQLaDAMRbA+u8H5Pbfq+dl3VDAvHxMUOVhe0wYB2zw= +github.com/rcrowley/go-metrics v0.0.0-20250401214520-65e299d6c5c9 h1:bsUq1dX0N8AOIL7EB/X911+m4EHsnWEHeJ0c+3TTBrg= +github.com/rcrowley/go-metrics v0.0.0-20250401214520-65e299d6c5c9/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/segmentio/asm v1.2.1 h1:DTNbBqs57ioxAD4PrArqftgypG4/qNpXoJx8TVXxPR0= +github.com/segmentio/asm v1.2.1/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs= +github.com/sergi/go-diff v1.4.0 h1:n/SP9D5ad1fORl+llWyN+D6qoUETXNZARKjyY2/KVCw= +github.com/sergi/go-diff v1.4.0/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= +github.com/sirupsen/logrus v1.9.4-0.20230606125235-dd1b4c2e81af h1:Sp5TG9f7K39yfB+If0vjp97vuT74F72r8hfRpP8jLU0= +github.com/sirupsen/logrus v1.9.4-0.20230606125235-dd1b4c2e81af/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/tchap/go-patricia/v2 v2.3.3 h1:xfNEsODumaEcCcY3gI0hYPZ/PcpVv5ju6RMAhgwZDDc= +github.com/tchap/go-patricia/v2 v2.3.3/go.mod h1:VZRHKAb53DLaG+nA9EaYYiaEx6YztwDlLElMsnSHD4k= +github.com/valyala/fastjson v1.6.4 h1:uAUNq9Z6ymTgGhcm0UynUAB6tlbakBrz6CQFax3BXVQ= +github.com/valyala/fastjson v1.6.4/go.mod h1:CLCAqky6SMuOcxStkYQvblddUtoRxhYMGLrsQns1aXY= +github.com/vektah/gqlparser/v2 v2.5.31 h1:YhWGA1mfTjID7qJhd1+Vxhpk5HTgydrGU9IgkWBTJ7k= +github.com/vektah/gqlparser/v2 v2.5.31/go.mod h1:c1I28gSOVNzlfc4WuDlqU7voQnsqI6OG2amkBAFmgts= +github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo= +github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= +github.com/yashtewari/glob-intersection v0.2.0 h1:8iuHdN88yYuCzCdjt0gDe+6bAhUwBeEWqThExu54RFg= +github.com/yashtewari/glob-intersection v0.2.0/go.mod h1:LK7pIC3piUjovexikBbJ26Yml7g8xa5bsjfx2v1fwok= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 h1:RbKq8BG0FI8OiXhBfcRtqqHcZcka+gU3cskNuf05R18= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0/go.mod h1:h06DGIukJOevXaj/xrNjhi/2098RZzcLTbc0jDAUbsg= +go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8= +go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0 h1:GqRJVj7UmLjCVyVJ3ZFLdPRmhDUp2zFmQe3RHIOsw24= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0/go.mod h1:ri3aaHSmCTVYu2AWv44YMauwAQc0aqI9gHKIcSbI1pU= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.38.0 h1:lwI4Dc5leUqENgGuQImwLo4WnuXFPetmPpkLi2IrX54= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.38.0/go.mod h1:Kz/oCE7z5wuyhPxsXDuaPteSWqjSBD5YaSdbxZYGbGk= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.38.0 h1:aTL7F04bJHUlztTsNGJ2l+6he8c+y/b//eR0jjjemT4= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.38.0/go.mod h1:kldtb7jDTeol0l3ewcmd8SDvx3EmIE7lyvqbasU3QC4= +go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA= +go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI= +go.opentelemetry.io/otel/sdk v1.38.0 h1:l48sr5YbNf2hpCUj/FoGhW9yDkl+Ma+LrVl8qaM5b+E= +go.opentelemetry.io/otel/sdk v1.38.0/go.mod h1:ghmNdGlVemJI3+ZB5iDEuk4bWA3GkTpW+DOoZMYBVVg= +go.opentelemetry.io/otel/sdk/metric v1.38.0 h1:aSH66iL0aZqo//xXzQLYozmWrXxyFkBJ6qT5wthqPoM= +go.opentelemetry.io/otel/sdk/metric v1.38.0/go.mod h1:dg9PBnW9XdQ1Hd6ZnRz689CbtrUp0wMMs9iPcgT9EZA= +go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE= +go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs= +go.opentelemetry.io/proto/otlp v1.7.1 h1:gTOMpGDb0WTBOP8JaO72iL3auEZhVmAQg4ipjOVAtj4= +go.opentelemetry.io/proto/otlp v1.7.1/go.mod h1:b2rVh6rfI/s2pHWNlB7ILJcRALpcNDzKhACevjI+ZnE= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= +go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q= +golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4= +golang.org/x/mod v0.29.0 h1:HV8lRxZC4l2cr3Zq1LvtOsi/ThTgWnUk/y64QSs8GwA= +golang.org/x/mod v0.29.0/go.mod h1:NyhrlYXJ2H4eJiRy/WDBO6HMqZQ6q9nk4JzS3NuCK+w= +golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= +golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= +golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I= +golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= +golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= +golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= +golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ= +golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs= +google.golang.org/genproto/googleapis/api v0.0.0-20251022142026-3a174f9686a8 h1:mepRgnBZa07I4TRuomDE4sTIYieg/osKmzIf4USdWS4= +google.golang.org/genproto/googleapis/api v0.0.0-20251022142026-3a174f9686a8/go.mod h1:fDMmzKV90WSg1NbozdqrE64fkuTv6mlq2zxo9ad+3yo= +google.golang.org/genproto/googleapis/rpc v0.0.0-20251022142026-3a174f9686a8 h1:M1rk8KBnUsBDg1oPGHNCxG4vc1f49epmTO7xscSajMk= +google.golang.org/genproto/googleapis/rpc v0.0.0-20251022142026-3a174f9686a8/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk= +google.golang.org/grpc v1.77.0 h1:wVVY6/8cGA6vvffn+wWK5ToddbgdU3d8MNENr4evgXM= +google.golang.org/grpc v1.77.0/go.mod h1:z0BY1iVj0q8E1uSQCjL9cppRj+gnZjzDnzV0dHhrNig= +google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE= +google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= +sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= diff --git a/glitch/rego/rego_python/src/rego_python/go/regolib.go b/glitch/rego/rego_python/src/rego_python/go/regolib.go new file mode 100644 index 00000000..2a4254cd --- /dev/null +++ b/glitch/rego/rego_python/src/rego_python/go/regolib.go @@ -0,0 +1,72 @@ +package main + +/* +#include +*/ +import "C" + +import ( + "context" + "encoding/json" + "fmt" + "unsafe" + + "github.com/open-policy-agent/opa/rego" + "github.com/open-policy-agent/opa/storage/inmem" +) + +//export RunRego +func RunRego(inputJSON *C.char, dataJSON *C.char, modulesJSON *C.char) *C.char { + input := C.GoString(inputJSON) + data := C.GoString(dataJSON) + modules := C.GoString(modulesJSON) + + var inputVal interface{} + var dataVal map[string]interface{} + var moduleMap map[string]string + + if err := json.Unmarshal([]byte(input), &inputVal); err != nil { + return C.CString(fmt.Sprintf(`{"error": "failed to parse input: %s"}`, err)) + } + + if err := json.Unmarshal([]byte(data), &dataVal); err != nil { + return C.CString(fmt.Sprintf(`{"error": "failed to parse data: %s"}`, err)) + } + + if err := json.Unmarshal([]byte(modules), &moduleMap); err != nil { + return C.CString(fmt.Sprintf(`{"error": "failed to parse modules: %s"}`, err)) + } + store := inmem.NewFromObject(dataVal) + + regoArgs := []func(*rego.Rego){} + regoArgs = append(regoArgs, rego.Query("data.glitch.Glitch_Analysis")) + regoArgs = append(regoArgs, rego.Input(inputVal)) + regoArgs = append(regoArgs, rego.Store(store)) + + for name, code := range moduleMap { + regoArgs = append(regoArgs, rego.Module(name, code)) + } + + r := rego.New(regoArgs...) + + ctx := context.Background() + results, err := r.Eval(ctx) + + if err != nil { + return C.CString(fmt.Sprintf(`{"error": "rego evaluation failed: %s"}`, err)) + } + + out, err := json.Marshal(results) + if err != nil { + return C.CString(fmt.Sprintf(`{"error": "output serialization failed: %s"}`, err)) + } + + return C.CString(string(out)) +} + +//export FreeCString +func FreeCString(str *C.char) { + C.free(unsafe.Pointer(str)) +} + +func main() {} diff --git a/glitch/rego/rego_python/src/rego_python/wrapper.py b/glitch/rego/rego_python/src/rego_python/wrapper.py new file mode 100644 index 00000000..7bb0bcf2 --- /dev/null +++ b/glitch/rego/rego_python/src/rego_python/wrapper.py @@ -0,0 +1,91 @@ +import ctypes +import json +import platform +from pathlib import Path +from typing import Optional, Any + +_rego_available: bool = False +_rego_error: Optional[str] = None +_lib: Any = None + + +def _get_lib_path(): + system = platform.system().lower() # 'windows', 'linux', 'darwin' + machine = platform.machine().lower() # 'x86_64', 'amd64', 'arm64', 'aarch64', etc. + + base = Path(__file__).parent / "bin" + + EXTENSIONS = { + "linux": "so", + "darwin": "dylib", + "windows": "dll", + } + + # normalize architecture names + ARCH_MAP = { + "amd64": "amd64", + "x86_64": "amd64", + "aarch64": "arm64", + "arm64": "arm64", + } + + try: + arch = ARCH_MAP[machine] + ext = EXTENSIONS[system] + except KeyError: + raise OSError(f"Unsupported platform: {system} {machine}") + + filename = f"librego-{system}-{arch}.{ext}" + return base / filename + + +def _load_lib() -> Any: + global _rego_available, _rego_error, _lib + try: + lib_path = _get_lib_path() + if not lib_path.exists(): + _rego_error = f"Rego library not found at {lib_path}" + return None + _lib = ctypes.CDLL(str(lib_path)) + _lib.RunRego.argtypes = [ctypes.c_char_p, ctypes.c_char_p, ctypes.c_char_p] + _lib.RunRego.restype = ctypes.c_void_p + _lib.FreeCString.argtypes = [ctypes.c_void_p] + _lib.FreeCString.restype = None + _rego_available = True + return _lib + except OSError as e: + _rego_error = str(e) + return None + + +_load_lib() + + +def is_rego_available() -> bool: + return _rego_available + + +def get_rego_error() -> Optional[str]: + return _rego_error + + +def run_rego( + input_data: dict[str, str], data: dict[str, str], rego_modules: dict[str, str] +): + if _lib is None: + raise RuntimeError(f"Rego library is not available: {_rego_error}") + + input_str = json.dumps(input_data) + data_str = json.dumps(data) + rego_modules_str = json.dumps(rego_modules) + + result_ptr = _lib.RunRego( + input_str.encode("utf-8"), + data_str.encode("utf-8"), + rego_modules_str.encode("utf-8"), + ) + + result_json = ctypes.string_at(result_ptr).decode("utf-8") + _lib.FreeCString(result_ptr) + + return json.loads(result_json) diff --git a/glitch/repair/interactive/compiler/compiler.py b/glitch/repair/interactive/compiler/compiler.py index e94bbf20..e5162024 100644 --- a/glitch/repair/interactive/compiler/compiler.py +++ b/glitch/repair/interactive/compiler/compiler.py @@ -1,246 +1,485 @@ -from typing import Optional, Dict, Tuple +import random +import logging +from typing import Optional, Dict, Tuple, Type, Set from glitch.tech import Tech from glitch.repr.inter import * from glitch.repair.interactive.delta_p import * -from glitch.repair.interactive.compiler.names_database import NamesDatabase from glitch.repair.interactive.compiler.labeler import LabeledUnitBlock -from glitch.repair.interactive.values import DefaultValue class DeltaPCompiler: - _sketched = -1 - _condition = 0 + AU_TYPE_ATTRIBUTES = { + "user": ["state"], + "package": ["state"], + "service": ["state", "enabled"], + "aws_iam_role": ["state", "assume_role_policy"], + "aws_instance": ["state", "instance_type", "availability_zone"], + "aws_s3_bucket": ["state", "acl"], + } - class __Attributes: - def __init__(self, au_type: str, tech: Tech) -> None: - self.au_type = NamesDatabase.get_au_type(au_type, tech) + def __init__(self, labeled_script: LabeledUnitBlock) -> None: + self._sketched = -1 + self._literal = 0 + self._condition = 0 + self.scope: List[str] = [] + self.vars: Set[str] = set() + self._labeled_script = labeled_script + self.seen_resources: List[Tuple[List[str], str]] = [] + + class _AtomicUnitCompiler: + def __init__( + self, name: str, compiler: "DeltaPCompiler", attributes: List[str] + ) -> None: + self.__name = name + self._compiler = compiler + self.__attributes = attributes + + def compile( + self, + atomic_unit: AtomicUnit, + attributes: "DeltaPCompiler._Attributes", + ) -> PStatement: + name = attributes["name"] + if name == PEUndef(): + name = self._compiler._compile_expr(atomic_unit.name) + self._compiler._labeled_script.add_location( + atomic_unit, atomic_unit.name + ) + path = PEBinOP(PAdd(), PEConst(PStr(self.__name + ":")), name) + + if self._compiler._check_seen_resource(path): + return PSkip() + + name_attr = attributes.get_attribute("name") + if name_attr is not None: + self._compiler._labeled_script.add_location(atomic_unit, name_attr) + self._compiler._labeled_script.add_location(name_attr, name_attr.value) + + statement = PSkip() + for attribute in self.__attributes[::-1]: + statement = PSeq( + self._compiler._handle_attr( + atomic_unit, attributes, path, attribute + ), + statement, + ) + + return statement + + class _Attributes: + def __init__( + self, compiler: "DeltaPCompiler", au_type: str, tech: Tech + ) -> None: + self.__compiler = compiler self.__tech = tech self.__attributes: Dict[str, Tuple[PExpr, Attribute]] = {} def add_attribute(self, attribute: Attribute) -> None: - attr_name = NamesDatabase.get_attr_name( - attribute.name, self.au_type, self.__tech - ) - - self.__attributes[attr_name] = ( # type: ignore - DeltaPCompiler._compile_expr( - NamesDatabase.get_attr_value( - attribute.value, # type: ignore - attr_name, - self.au_type, - self.__tech, - ), - self.__tech, - ), - attribute, - ) + expr = self.__compiler._compile_expr(attribute.value) + self.__attributes[attribute.name] = (expr, attribute) def get_attribute(self, attr_name: str) -> Optional[Attribute]: return self.__attributes.get(attr_name, (None, None))[1] def get_attribute_value(self, attr_name: str) -> PExpr: default = PEUndef() - if attr_name == "state": - default = DefaultValue.DEFAULT_STATE - elif attr_name == "mode": - default = DefaultValue.DEFAULT_MODE - elif attr_name == "owner": - default = DefaultValue.DEFAULT_OWNER - elif attr_name == "content": - default = DefaultValue.DEFAULT_CONTENT - return self.__attributes.get(attr_name, (default, None))[0] def __getitem__(self, key: str) -> PExpr: return self.get_attribute_value(key) - def create_label_var_pair( + def get_var( self, attr_name: str, atomic_unit: AtomicUnit, - labeled_script: LabeledUnitBlock, - ) -> Tuple[int, str]: + ) -> Tuple[str, int]: attr = self.get_attribute(attr_name) - if attr is not None: - label = labeled_script.get_label(attr) - else: + if attr is None: # Creates sketched attribute - if attr_name == "state" and isinstance( - DefaultValue.DEFAULT_STATE.const, PStr - ): # HACK + if attr_name == "state": # HACK attr = Attribute( - attr_name, DefaultValue.DEFAULT_STATE.const.value, False + attr_name, + String( + UNDEF, + ElementInfo.get_sketched(), + ), + ElementInfo.get_sketched(), ) else: - attr = Attribute(attr_name, PEUndef(), False) # type: ignore + attr = Attribute( + attr_name, + Null(ElementInfo.get_sketched()), + ElementInfo.get_sketched(), + ) - attr.line, attr.column = ( - DeltaPCompiler._sketched, - DeltaPCompiler._sketched, - ) - DeltaPCompiler._sketched -= 1 - labeled_script.add_sketch_location(atomic_unit, attr) + label = self.__compiler._sketched + self.__compiler._sketched -= 1 self.add_attribute(attr) - label = labeled_script.add_label(attr_name, attr, sketched=True) + else: + # HACK: Allows to fix unsupported expressions + attr_value = self.get_attribute_value(attr_name) + if attr_value == PEUnsupported(): + if self.__compiler._labeled_script.has_label(attr.value): + label = self.__compiler._labeled_script.get_label(attr.value) + else: + label = self.__compiler._labeled_script.add_label( + f"literal-{self.__compiler._literal}", attr.value + ) + self.__attributes[attr_name] = ( + PRLet( + f"literal-{label}", + PEUndef(), + label, + ), + attr, + ) + self.__compiler._literal += 1 + + label = self.__compiler._labeled_script.get_label(attr) + + self.__compiler._labeled_script.add_location(atomic_unit, attr) + self.__compiler._labeled_script.add_location(attr, attr.value) + + return ( + self.__compiler._get_attribute_name( + attr.name, attr, atomic_unit.type, self.__tech + ), + label, + ) + + def _get_scope_name(self, name: str): + return ":dejavu:".join(self.scope + [name]) + + def _get_attribute_name( + self, + attr_name: str, + attribute: Attribute, + au_type: str, + tech: Tech, + ) -> str: + return self._get_scope_name(attr_name + "_" + str(hash(attribute))) + + def __has_var(self, id: str) -> bool: + scopes = id.split(":dejavu:") + while True: + if ":dejavu:".join(scopes) in self.vars: + return True + if len(scopes) == 1: + break + scopes.pop(-2) - return label, labeled_script.get_var(label) + return False - @staticmethod - def _compile_expr(expr: Optional[str], tech: Tech) -> Optional[PExpr]: - # FIXME to fix this I need to extend GLITCH's IR - if expr is None: - return None - if isinstance(expr, PEUndef): - return expr + def __get_prlet(self, expr: Expr, value: PExpr) -> PRLet: + if self._labeled_script.has_label(expr): + label = self._labeled_script.get_label(expr) + else: + literal_name = f"literal-{self._literal}" + label = self._labeled_script.add_label(literal_name, expr) + self._literal += 1 - return PEConst(PStr(expr)) + return PRLet( + f"literal-{label}", + value, + label, + ) + + def _compile_expr(self, expr: Expr) -> PExpr: + def binary_op(op: Type[PBinOp], left: Expr, right: Expr) -> PExpr: + left_expr = self._compile_expr(left) + right_expr = self._compile_expr(right) + if isinstance(left_expr, PEUnsupported) or isinstance( + right_expr, PEUnsupported + ): + return PEUnsupported() + return PEBinOP( + op(), + left_expr, + right_expr, + ) + + if isinstance(expr, String): + value = PEConst(PStr(expr.value)) + return self.__get_prlet(expr, value) + elif isinstance(expr, (Integer, Float, Complex)): + value = PEConst(PStr(str(expr.value))) + return self.__get_prlet(expr, value) + elif isinstance(expr, Boolean): + return self.__get_prlet(expr, PEConst(PBool(expr.value))) + elif isinstance(expr, Null): + return self.__get_prlet(expr, PEUndef()) + elif isinstance(expr, Not): + return PEUnOP(PNot(), self._compile_expr(expr.expr)) + elif isinstance(expr, Minus): + return PEUnOP(PNeg(), self._compile_expr(expr.expr)) + elif isinstance(expr, Or): + return binary_op(POr, expr.left, expr.right) + elif isinstance(expr, And): + return binary_op(PAnd, expr.left, expr.right) + elif isinstance(expr, Sum): + return binary_op(PAdd, expr.left, expr.right) + elif isinstance(expr, Equal): + return binary_op(PEq, expr.left, expr.right) + elif isinstance(expr, NotEqual): + return PEUnOP(PNot(), binary_op(PEq, expr.left, expr.right)) + elif isinstance(expr, LessThan): + return binary_op(PLt, expr.left, expr.right) + elif isinstance(expr, LessThanOrEqual): + return PEBinOP( + POr(), + binary_op(PLt, expr.left, expr.right), + binary_op(PEq, expr.left, expr.right), + ) + elif isinstance(expr, GreaterThan): + return binary_op(PGt, expr.left, expr.right) + elif isinstance(expr, GreaterThanOrEqual): + return PEBinOP( + POr(), + binary_op(PGt, expr.left, expr.right), + binary_op(PEq, expr.left, expr.right), + ) + elif isinstance(expr, Subtract): + return binary_op(PSub, expr.left, expr.right) + elif isinstance(expr, Multiply): + return binary_op(PMultiply, expr.left, expr.right) + elif isinstance(expr, Divide): + return binary_op(PDivide, expr.left, expr.right) + elif isinstance(expr, Modulo): + return binary_op(PMod, expr.left, expr.right) + elif isinstance(expr, Power): + return binary_op(PPower, expr.left, expr.right) + elif isinstance(expr, VariableReference) and self.__has_var( + self._get_scope_name(expr.value) + ): + return PEVar(self._get_scope_name(expr.value)) + elif isinstance(expr, VariableReference): # undefined variable + return PEUnsupported() + else: + # TODO: Unsupported + logging.warning(f"Unsupported expression, got {expr}") + return PEUnsupported() + + def _check_seen_resource(self, path: PExpr): + # HACK: avoids some problems with duplicate atomic units + # (it only considers the last one defined) + path_str = PStatement.get_str( + path, {}, ignore_vars=True # HACK: avoids having to get the vars + ) + if path_str is not None: + if (self.scope, path_str) in self.seen_resources: + return True + self.seen_resources.append((self.scope.copy(), path_str)) + return False + + def _handle_attr( + self, atomic_unit: AtomicUnit, attributes: _Attributes, path: PExpr, attr: str + ): + attr_var, label = attributes.get_var(attr, atomic_unit) + self.vars.add(attr_var) + statement = PLet( + attr_var, + attributes[attr], + label, + PAttr(path, attr, PEVar(attr_var)), + ) + return statement - @staticmethod def __handle_file( + self, atomic_unit: AtomicUnit, - attributes: __Attributes, - labeled_script: LabeledUnitBlock, + attributes: _Attributes, ) -> PStatement: path = attributes["path"] + path_attr = attributes.get_attribute("path") + if path_attr is not None: + self._labeled_script.add_location(atomic_unit, path_attr) + self._labeled_script.add_location(path_attr, path_attr.value) # The path may be defined as the name of the atomic unit if path == PEUndef(): - path = PEConst(PStr(atomic_unit.name)) # type: ignore + path = self._compile_expr(atomic_unit.name) + self._labeled_script.add_location(atomic_unit, atomic_unit.name) - state_label, state_var = attributes.create_label_var_pair( - "state", atomic_unit, labeled_script - ) - statement = PLet( - state_var, - attributes["state"], - state_label, - PIf( - PEBinOP(PEq(), PEVar(state_var), PEConst(PStr("present"))), - PCreate(path), - PIf( - PEBinOP(PEq(), PEVar(state_var), PEConst(PStr("absent"))), - PRm(path), - PIf( - PEBinOP(PEq(), PEVar(state_var), PEConst(PStr("directory"))), - PMkdir(path), - PSkip(), - ), - ), - ), - ) + if self._check_seen_resource(path): + return PSkip() - content_label, content_var = attributes.create_label_var_pair( - "content", atomic_unit, labeled_script - ) + statement = self._handle_attr(atomic_unit, attributes, path, "state") statement = PSeq( - statement, - PLet( - content_var, - attributes["content"], - content_label, - PWrite(path, PEVar(content_var)), - ), - ) - - owner_label, owner_var = attributes.create_label_var_pair( - "owner", atomic_unit, labeled_script + statement, self._handle_attr(atomic_unit, attributes, path, "content") ) statement = PSeq( - statement, - PLet( - owner_var, - attributes["owner"], - owner_label, - PChown(path, PEVar(owner_var)), - ), - ) - - mode_label, mode_var = attributes.create_label_var_pair( - "mode", atomic_unit, labeled_script + statement, self._handle_attr(atomic_unit, attributes, path, "owner") ) statement = PSeq( - statement, - PLet( - mode_var, - attributes["mode"], - mode_label, - PChmod(path, PEVar(mode_var)), - ), + statement, self._handle_attr(atomic_unit, attributes, path, "mode") ) return statement - @staticmethod + def __handle_defined_type( + self, + atomic_unit: AtomicUnit, + ): + definition = self._labeled_script.env.get_definition(atomic_unit.type) + self.scope += [f"defined_resource{str(random.randint(0, 28021904))}"] + + defined_attributes: Dict[str, Tuple[Attribute, bool]] = {} + for attr in atomic_unit.attributes: + defined_attributes[attr.name] = (attr, False) + + for attr in definition.attributes: + if attr.name not in defined_attributes: + defined_attributes[attr.name] = (attr, True) + + for name, attr in list(defined_attributes.items()): + new_name = self._get_scope_name(name) + self.vars.add(new_name) + defined_attributes[new_name] = attr + defined_attributes.pop(name) + + statement = self.__handle_unit_block(definition) + + # The scope is popped here since it allows variable references + # compiled from the attributes' values to have the outside scope + self.scope.pop() + + for name, t in defined_attributes.items(): + attr, in_ub = t + value = self._compile_expr(attr.value) + self._labeled_script.add_location(attr, attr.value) + if in_ub: + self._labeled_script.add_location(definition, attr) + else: + self._labeled_script.add_location(atomic_unit, attr) + statement = PLet( + name, + value, + self._labeled_script.get_label(attr), + statement, + ) + + return statement + def __handle_atomic_unit( - statement: PStatement, + self, atomic_unit: AtomicUnit, - tech: Tech, - labeled_script: LabeledUnitBlock, ) -> PStatement: - attributes: DeltaPCompiler.__Attributes = DeltaPCompiler.__Attributes( - atomic_unit.type, tech + statement = PSkip() + tech = self._labeled_script.tech + attributes: DeltaPCompiler._Attributes = DeltaPCompiler._Attributes( + self, atomic_unit.type, tech ) - if attributes.au_type == "file": + + if atomic_unit.type == "file": for attribute in atomic_unit.attributes: attributes.add_attribute(attribute) statement = PSeq( statement, - DeltaPCompiler.__handle_file(atomic_unit, attributes, labeled_script), + self.__handle_file(atomic_unit, attributes), ) + elif atomic_unit.type in DeltaPCompiler.AU_TYPE_ATTRIBUTES: + for attribute in atomic_unit.attributes: + attributes.add_attribute(attribute) + statement = DeltaPCompiler._AtomicUnitCompiler( + atomic_unit.type, + self, + DeltaPCompiler.AU_TYPE_ATTRIBUTES[atomic_unit.type], + ).compile(atomic_unit, attributes) + # Defined type + elif self._labeled_script.env.has_definition(atomic_unit.type): + statement = PSeq(self.__handle_defined_type(atomic_unit), PSkip()) + return statement - @staticmethod def __handle_conditional( - conditional: ConditionalStatement, tech: Tech, labeled_script: LabeledUnitBlock + self, + conditional: ConditionalStatement, ) -> PStatement: + self.scope += [f"condition{str(random.randint(0, 28021904))}"] body = PSkip() for stat in conditional.statements: - if isinstance(stat, AtomicUnit): - body = DeltaPCompiler.__handle_atomic_unit( - body, stat, tech, labeled_script - ) - elif isinstance(stat, ConditionalStatement): - body = PSeq( - body, - DeltaPCompiler.__handle_conditional(stat, tech, labeled_script), - ) + body = PSeq(body, self.__handle_code_element(stat)) + self.scope.pop() else_statement = PSkip() if conditional.else_statement is not None: - else_statement = DeltaPCompiler.__handle_conditional( - conditional.else_statement, tech, labeled_script - ) + else_statement = self.__handle_conditional(conditional.else_statement) - DeltaPCompiler._condition += 1 + self._condition += 1 return PIf( # FIXME: This creates a placeholder since we will branch every time # There are cases that we can infer the value of the condition # The creation of these variables should be done in the solver - PEVar(f"dejavu-condition-{DeltaPCompiler._condition}"), + PEVar(f"dejavu-condition-{self._condition}"), body, else_statement, ) - @staticmethod - def compile(labeled_script: LabeledUnitBlock, tech: Tech) -> PStatement: - statement = PSkip() - script = labeled_script.script + def __handle_variable( + self, + variable: Variable, + ) -> PStatement: + name = self._get_scope_name(variable.name) + self.vars.add(name) + statement = PLet( + name, + self._compile_expr(variable.value), + self._labeled_script.get_label(variable), + PSkip(), + ) + self._labeled_script.add_location(variable, variable.value) + return statement - # TODO: Handle variables - # TODO: Handle scopes - # TODO: The statements will not be in the correct order + def __handle_unit_block( + self, + unit_block: UnitBlock, + ) -> PStatement: + compiled = PSkip() + statements: List[CodeElement] = ( + unit_block.statements + + unit_block.atomic_units + + unit_block.variables + + unit_block.unit_blocks + ) + statements.sort(key=lambda x: (x.line, x.column), reverse=True) - for stat in script.statements: - if isinstance(stat, ConditionalStatement): - statement = PSeq( - statement, - DeltaPCompiler.__handle_conditional(stat, tech, labeled_script), - ) + # If we do not do this here, since we iterate over the statements + # in reverse, the variables will not be defined when expressions are + # compiled + for variable in unit_block.variables: + self.vars.add(self._get_scope_name(variable.name)) - for atomic_unit in script.atomic_units: - statement = DeltaPCompiler.__handle_atomic_unit( - statement, atomic_unit, tech, labeled_script - ) + for statement in statements: + self._labeled_script.add_location(unit_block, statement) + new_compiled = self.__handle_code_element(statement) + if isinstance(new_compiled, PLet): + new_compiled.body = compiled + compiled = PSeq(new_compiled, PSkip()) + else: + compiled = PSeq(new_compiled, compiled) - return statement + return compiled + + def __handle_code_element(self, code_element: CodeElement) -> PStatement: + if isinstance(code_element, AtomicUnit): + return self.__handle_atomic_unit(code_element) + elif isinstance(code_element, ConditionalStatement): + return self.__handle_conditional(code_element) + elif isinstance(code_element, Variable): + return self.__handle_variable(code_element) + elif ( + isinstance(code_element, UnitBlock) + and code_element.type == UnitBlockType.definition + ): + return PSkip() + elif isinstance(code_element, UnitBlock): + return self.__handle_unit_block(code_element) + else: + logging.warning(f"Unsupported code element, got {code_element}") + return PSkip() + + def compile(self) -> PStatement: + script = self._labeled_script.script + # TODO: Handle scopes + return self.__handle_unit_block(script) diff --git a/glitch/repair/interactive/compiler/environment.py b/glitch/repair/interactive/compiler/environment.py new file mode 100644 index 00000000..a9ce75c4 --- /dev/null +++ b/glitch/repair/interactive/compiler/environment.py @@ -0,0 +1,21 @@ +from typing import Dict +from glitch.repr.inter import * + + +class DefinedAtomicUnitEnv: + def __init__(self, unit_block: UnitBlock) -> None: + self.__atomic_units: Dict[str, UnitBlock] = {} + self.__collect_definitions(unit_block) + + def __collect_definitions(self, unit_block: UnitBlock) -> None: + for ub in unit_block.unit_blocks: + if ub.type == UnitBlockType.definition: + assert ub.name is not None + self.__atomic_units[ub.name] = ub + self.__collect_definitions(ub) + + def get_definition(self, name: str) -> UnitBlock: + return self.__atomic_units[name] + + def has_definition(self, name: str) -> bool: + return name in self.__atomic_units diff --git a/glitch/repair/interactive/compiler/labeler.py b/glitch/repair/interactive/compiler/labeler.py index da39e6ab..3cdc13a9 100644 --- a/glitch/repair/interactive/compiler/labeler.py +++ b/glitch/repair/interactive/compiler/labeler.py @@ -1,7 +1,7 @@ from typing import Dict from glitch.repr.inter import * -from glitch.repair.interactive.compiler.names_database import NamesDatabase from glitch.tech import Tech +from glitch.repair.interactive.compiler.environment import DefinedAtomicUnitEnv class LabeledUnitBlock: @@ -17,11 +17,12 @@ def __init__(self, script: UnitBlock, tech: Tech) -> None: """ self.script = script self.tech: Tech = tech + self.env: DefinedAtomicUnitEnv = DefinedAtomicUnitEnv(script) self.__label = 0 self.__label_to_var: Dict[int, str] = {} self.__codeelement_to_label: Dict[CodeElement, int] = {} self.__label_to_codeelement: Dict[int, CodeElement] = {} - self.__sketch_location: Dict[CodeElement, CodeElement] = {} + self.__location: Dict[CodeElement, CodeElement] = {} def add_label( self, name: str, codeelement: CodeElement, sketched: bool = False @@ -46,16 +47,17 @@ def add_label( self.__label += 1 return self.__label - 1 - def add_sketch_location( - self, sketch_location: CodeElement, codeelement: CodeElement - ) -> None: - """Defines where a sketched code element is defined in the script. + def add_location(self, location: CodeElement, codeelement: CodeElement) -> None: + """Defines where a code element is defined in the script. Args: - sketch_location (CodeElement): The code element where the sketched code element is defined. - codeelement (CodeElement): The sketched code element. + sketch_location (CodeElement): The code element where the code element is defined. + codeelement (CodeElement): The code element. """ - self.__sketch_location[codeelement] = sketch_location + if isinstance(codeelement, BinaryOperation): + self.add_location(location, codeelement.left) + self.add_location(location, codeelement.right) + self.__location[codeelement] = location def get_label(self, codeelement: CodeElement) -> int: """Returns the label of the code element. @@ -68,6 +70,17 @@ def get_label(self, codeelement: CodeElement) -> int: """ return self.__codeelement_to_label[codeelement] + def has_label(self, codeelement: CodeElement) -> bool: + """Returns whether the code element has a label. + + Args: + codeelement (CodeElement): The code element. + + Returns: + bool: Whether the code element has a label. + """ + return codeelement in self.__codeelement_to_label + def get_codeelement(self, label: int) -> CodeElement: """Returns the code element with the given label. @@ -98,7 +111,7 @@ def get_var(self, label: int) -> str: """ return self.__label_to_var[label] - def get_sketch_location(self, codeelement: CodeElement) -> CodeElement: + def get_location(self, codeelement: CodeElement) -> CodeElement: """Returns the location where the sketched code element is defined. Args: @@ -107,7 +120,7 @@ def get_sketch_location(self, codeelement: CodeElement) -> CodeElement: Returns: CodeElement: The location where the sketched code element is defined. """ - return self.__sketch_location[codeelement] + return self.__location[codeelement] class GLITCHLabeler: @@ -122,9 +135,7 @@ def label_attribute( atomic_unit (AtomicUnit): The attribute's atomic unit. attribute (Attribute): The attribute. """ - type = NamesDatabase.get_au_type(atomic_unit.type, labeled.tech) - name = NamesDatabase.get_attr_name(attribute.name, type, labeled.tech) - labeled.add_label(name, attribute) # type: ignore + labeled.add_label(atomic_unit.name, attribute) # type: ignore @staticmethod def label_atomic_unit(labeled: LabeledUnitBlock, atomic_unit: AtomicUnit) -> None: @@ -158,16 +169,46 @@ def label_conditional( conditional (ConditionalStatement): The conditional statement. """ for statement in conditional.statements: - if isinstance(statement, AtomicUnit): - GLITCHLabeler.label_atomic_unit(labeled, statement) - elif isinstance(statement, ConditionalStatement): - GLITCHLabeler.label_conditional(labeled, statement) - elif isinstance(statement, Variable): - GLITCHLabeler.label_variable(labeled, statement) - + GLITCHLabeler.label_statement(labeled, statement) if conditional.else_statement is not None: GLITCHLabeler.label_conditional(labeled, conditional.else_statement) + @staticmethod + def label_unit_block(labeled: LabeledUnitBlock, unit_block: UnitBlock) -> None: + """Labels a unit block. + + Args: + labeled (LabeledUnitBlock): The labeled script. + unit_block (UnitBlock): The unit block. + """ + for statement in unit_block.statements: + GLITCHLabeler.label_statement(labeled, statement) + for atomic_unit in unit_block.atomic_units: + GLITCHLabeler.label_atomic_unit(labeled, atomic_unit) + for variable in unit_block.variables: + GLITCHLabeler.label_variable(labeled, variable) + for unit_block in unit_block.unit_blocks: + GLITCHLabeler.label_unit_block(labeled, unit_block) + for attribute in unit_block.attributes: + labeled.add_label(attribute.name, attribute) + + @staticmethod + def label_statement(labeled: LabeledUnitBlock, statement: CodeElement) -> None: + """Labels a statement. + + Args: + labeled (LabeledUnitBlock): The labeled script. + statement (CodeElement): The statement. + """ + if isinstance(statement, AtomicUnit): + GLITCHLabeler.label_atomic_unit(labeled, statement) + elif isinstance(statement, ConditionalStatement): + GLITCHLabeler.label_conditional(labeled, statement) + elif isinstance(statement, Variable): + GLITCHLabeler.label_variable(labeled, statement) + elif isinstance(statement, UnitBlock): + GLITCHLabeler.label_unit_block(labeled, statement) + @staticmethod def label(script: UnitBlock, tech: Tech) -> LabeledUnitBlock: """Labels a script. @@ -180,15 +221,5 @@ def label(script: UnitBlock, tech: Tech) -> LabeledUnitBlock: LabeledUnitBlock: The labeled script. """ labeled = LabeledUnitBlock(script, tech) - - for statement in script.statements: - if isinstance(statement, ConditionalStatement): - GLITCHLabeler.label_conditional(labeled, statement) - - for atomic_unit in script.atomic_units: - GLITCHLabeler.label_atomic_unit(labeled, atomic_unit) - - for variable in script.variables: - GLITCHLabeler.label_variable(labeled, variable) - + GLITCHLabeler.label_unit_block(labeled, script) return labeled diff --git a/glitch/repair/interactive/compiler/names_database.py b/glitch/repair/interactive/compiler/names_database.py index 29577110..b4f09a70 100644 --- a/glitch/repair/interactive/compiler/names_database.py +++ b/glitch/repair/interactive/compiler/names_database.py @@ -1,5 +1,6 @@ from glitch.tech import Tech -from typing import Optional +from glitch.repr.inter import * +from typing import Tuple class NamesDatabase: @@ -17,8 +18,28 @@ def get_au_type(type: str, tech: Tech) -> str: match type, tech: case "file", Tech.puppet | Tech.chef | Tech.ansible: return "file" + case "directory", Tech.chef: + return "file" + case "link", Tech.chef: + return "file" case "ansible.builtin.file", Tech.ansible: return "file" + case "ansible.builtin.user", Tech.ansible: + return "user" + case "ansible.builtin.package", Tech.ansible: + return "package" + case "ansible.builtin.service", Tech.ansible: + return "service" + case "ansible.builtin.yum" | "yum", Tech.ansible: + return "package" + case "ansible.builtin.apt" | "apt", Tech.ansible: + return "package" + case "amazon.aws.s3_bucket", Tech.ansible: + return "aws_s3_bucket" + case "amazon.aws.ec2_instance", Tech.ansible: + return "aws_instance" + case "amazon.aws.iam_role", Tech.ansible: + return "aws_iam_role" case _: pass return type @@ -35,6 +56,7 @@ def reverse_attr_name(name: str, au_type: str, tech: Tech) -> str: Returns: str: The technology-specific name of the attribute. """ + au_type = NamesDatabase.get_au_type(au_type, tech) match name, au_type, tech: case "path", "file", Tech.puppet | Tech.chef | Tech.ansible: return "path" @@ -44,10 +66,20 @@ def reverse_attr_name(name: str, au_type: str, tech: Tech) -> str: return "mode" case "content", "file", Tech.puppet | Tech.chef | Tech.ansible: return "content" + case "state", "file" | "user" | "package" | "service", Tech.chef: + return "action" case "state", "file", Tech.ansible: return "state" - case "state", "file", Tech.puppet: + case "state", "file" | "user" | "package" | "service", Tech.puppet: return "ensure" + case "enabled", "service", Tech.puppet: + return "enable" + case "enabled", "service", Tech.chef: + return "action" + case "name", "aws_s3_bucket", Tech.terraform: + return "bucket" + case "assume_role_policy", "aws_iam_role", Tech.ansible: + return "assume_role_policy_document" case _: pass return name @@ -68,12 +100,46 @@ def reverse_attr_value(value: str, attr_name: str, au_type: str, tech: Tech) -> match value, attr_name, au_type, tech: case "present", "state", "file", Tech.ansible: return "file" + case "present", "state", "user", Tech.puppet: + return "present" + case "present", "state", "file" | "user", Tech.chef: + return ":create" + case "absent", "state", "file" | "user", Tech.chef: + return ":delete" + case "present", "state", "package", Tech.chef: + return ":install" + case "absent", "state", "package", Tech.chef: + return ":remove" + case "latest", "state", "package", Tech.chef: + return ":upgrade" + case "purged", "state", "package", Tech.chef: + return ":purge" + case "nothing", "state", "package", Tech.chef: + return ":nothing" + case "reconfig", "state", "package", Tech.chef: + return ":reconfig" + case "start", "state", "service", Tech.puppet: + return "running" + case "stop", "state", "service", Tech.puppet: + return "stopped" + case "start", "state", "service", Tech.chef: + return ":start" + case "stop", "state", "service", Tech.chef: + return ":stop" + case "start", "state", "service", Tech.ansible: + return "started" + case "stop", "state", "service", Tech.ansible: + return "stopped" + case "true", "enabled", "service", Tech.chef: + return ":enable" + case "false", "enabled", "service", Tech.chef: + return ":disable" case _: pass return value @staticmethod - def get_attr_name(name: str, au_type: str, tech: Tech) -> str: + def __get_attr_name(name: str, au_type: str, tech: Tech) -> str: """Returns the generic name of the attribute with the given name, atomic unit type and tech. Args: @@ -95,19 +161,25 @@ def get_attr_name(name: str, au_type: str, tech: Tech) -> str: return "mode" case "content", "file", Tech.puppet | Tech.chef | Tech.ansible: return "content" - case "ensure", "file", Tech.puppet: + case "ensure", "file" | "user" | "package" | "service", Tech.puppet: return "state" - case "state", "file", Tech.ansible: + case "state", "file" | "user" | "package" | "service", Tech.ansible: return "state" + case "action", "file" | "user" | "package" | "service", Tech.chef: + return "state" + case "enable", "service", Tech.puppet: + return "enabled" + case "bucket", "aws_s3_bucket", Tech.terraform: + return "name" + case "assume_role_policy_document", "aws_iam_role", Tech.ansible: + return "assume_role_policy" case _: pass return name @staticmethod - def get_attr_value( - value: str, name: str, au_type: str, tech: Tech - ) -> Optional[str]: + def __get_attr_value(value: Expr, name: str, au_type: str, tech: Tech) -> Expr: """Returns the generic value of the attribute with the given value, name, atomic unit type and tech. @@ -120,12 +192,159 @@ def get_attr_value( Returns: str: The generic value of the attribute. """ - match value, name, au_type, tech: - case "file", "state", "file", Tech.puppet | Tech.ansible: - return "present" - case "touch", "state", "file", Tech.ansible: - return "present" - case _: - pass + v = None + if isinstance(value, (VariableReference, String)): + v = value.value + + if isinstance(value, Integer) and name == "mode": + return String( + oct(value.value).replace("o", ""), ElementInfo.from_code_element(value) + ) + + if v is not None: + match v, name, au_type, tech: + case ( + "present" | "directory" | "absent", + "state", + "file" | "user", + Tech.puppet, + ): + pass + case ( + "latest" | "present" | "absent" | "purged" | "disabled", + "state", + "package", + Tech.puppet, + ): + pass + case "installed", "state", "package", Tech.puppet: + v = "present" + case "running" | "true", "state", "service", Tech.puppet: + v = "start" + case "stopped" | "false", "state", "service", Tech.puppet: + v = "stop" + case ":start", "state", "service", Tech.chef: + v = "start" + case ":stop", "state", "service", Tech.chef: + v = "stop" + case "started", "state", "service", Tech.ansible: + v = "start" + case "stopped", "state", "service", Tech.ansible: + v = "stop" + case "file", "state", "file", Tech.puppet | Tech.ansible: + v = "present" + case "touch", "state", "file", Tech.ansible: + v = "present" + case ":create" | ":nothing", "state", "file" | "user", Tech.chef: + v = "present" + case ( + ":touch" | ":nothing" | ":create_if_missing", + "state", + "file", + Tech.chef, + ): + v = "present" + case ":upgrade", "state", "package", Tech.chef: + v = "latest" + case ":purge", "state", "package", Tech.chef: + v = "purged" + case ":reconfig", "state", "package", Tech.chef: + v = "reconfig" + case ":nothing", "state", "package", Tech.chef: + v = "nothing" + case ":delete", "state", "file" | "user", Tech.chef: + v = "absent" + case ":install", "state", "package", Tech.chef: + v = "present" + case ":remove", "state", "package", Tech.chef: + v = "absent" + case ":enable", "enabled", "service", Tech.chef: + v = "true" + case ":disable", "enabled", "service", Tech.chef: + v = "false" + case _: + return value + + return String(v, ElementInfo.from_code_element(value)) return value + + @staticmethod + def get_attr_pair( + value: Expr, attr_name: str, au_type: str, tech: Tech + ) -> Tuple[str, Expr]: + attr_name = NamesDatabase.__get_attr_name(attr_name, au_type, tech) + + v = None + if isinstance(value, (VariableReference, String)): + v = value.value + if v is not None: + match v, attr_name, au_type, tech: + case ":enable", "state", "service", Tech.chef: + return ( + "enabled", + String("true", ElementInfo.from_code_element(value)), + ) + case ":disable", "state", "service", Tech.chef: + return ( + "enabled", + String("false", ElementInfo.from_code_element(value)), + ) + case _: + pass + + return ( + attr_name, + NamesDatabase.__get_attr_value(value, attr_name, au_type, tech), + ) + + +class NormalizationVisitor: + def __init__(self, tech: Tech) -> None: + self.tech = tech + + def visit(self, element: CodeElement) -> None: + if isinstance(element, AtomicUnit): + self.visit_atomic_unit(element) + elif isinstance(element, UnitBlock): + self.visit_unit_block(element) + elif isinstance(element, ConditionalStatement): + self.visit_conditional_statement(element) + + if isinstance(element, Block): + self.visit_block(element) + + def visit_atomic_unit(self, element: AtomicUnit) -> None: + element.type = NamesDatabase.get_au_type(element.type, self.tech) + + # Since Terraform does not define a state, we add it manually + # FIXME: Probably should be in a better place + if self.tech == Tech.terraform: + element.attributes.insert( + 0, + Attribute( + "state", + # The element info should be unique + String("present", ElementInfo.get_sketched()), + ElementInfo.get_sketched(), + ), + ) + + for attr in element.attributes: + attr.name, attr.value = NamesDatabase.get_attr_pair( + attr.value, attr.name, element.type, self.tech + ) + + def visit_conditional_statement(self, element: ConditionalStatement) -> None: + if element.else_statement is not None: + self.visit(element.else_statement) + + def visit_unit_block(self, element: UnitBlock) -> None: + for ub in element.unit_blocks: + self.visit(ub) + for au in element.atomic_units: + self.visit(au) + + def visit_block(self, element: Block) -> None: + for child in element.statements: + self.visit(child) diff --git a/glitch/repair/interactive/compiler/template_database.py b/glitch/repair/interactive/compiler/template_database.py new file mode 100644 index 00000000..5a4fd451 --- /dev/null +++ b/glitch/repair/interactive/compiler/template_database.py @@ -0,0 +1,43 @@ +from glitch.tech import Tech +from glitch.repr.inter import * + + +class TemplateDatabase: + @staticmethod + def get_template(code_element: CodeElement, tech: Tech) -> str: + """Returns the template for the given code element on a given tech. + + Args: + type (CodeElement): The code element being considered. + tech (Tech): The tech being considered. + + Returns: + str: The template. + """ + if isinstance(code_element, Attribute) and tech == Tech.puppet: + return "{} => {},\n" + elif isinstance(code_element, Attribute) and tech == Tech.chef: + return "{} {}\n" + elif isinstance(code_element, Attribute) and tech == Tech.ansible: + return "{}: {}\n" + elif isinstance(code_element, Attribute) and tech == Tech.terraform: + return "{} = {}\n" + + raise NotImplementedError( + "Template not found for the given code element and tech." + ) + + @staticmethod + def get_template_for_multiline_string(tech: Tech) -> str: + """Returns the template for a multiline string on a given tech. + + Args: + tech (Tech): The tech being considered. + + Returns: + str: The template. + """ + if tech == Tech.terraform: + return "< str: + @staticmethod + def __get_var(id: str, vars: Dict[str, PExpr]) -> Optional[PExpr]: + scopes = id.split(":dejavu:") + while True: + if ":dejavu:".join(scopes) in vars: + return vars[":dejavu:".join(scopes)] + if len(scopes) == 1: + break + scopes.pop(-2) + + return None + + @staticmethod + def get_str( + expr: PExpr, vars: Dict[str, PExpr], ignore_vars: bool = False + ) -> Optional[str]: if isinstance(expr, PEConst) and isinstance(expr.const, PStr): return expr.const.value - elif isinstance(expr, PEVar): - return self.__get_str(vars[expr.id], vars) + elif isinstance(expr, PEConst) and isinstance(expr.const, PBool): + return str(expr.const.value).lower() + elif isinstance(expr, PEConst) and isinstance(expr.const, PNum): + return str(expr.const.value) + elif ignore_vars and isinstance(expr, PEVar): + return expr.id + elif ( + isinstance(expr, PEVar) and PStatement.__get_var(expr.id, vars) is not None + ): + res = PStatement.__get_var(expr.id, vars) + assert res is not None + return PStatement.get_str(res, vars, ignore_vars=ignore_vars) + elif isinstance(expr, PRLet): + return PStatement.get_str(expr.expr, vars, ignore_vars=ignore_vars) + elif isinstance(expr, PEBinOP) and isinstance(expr.op, PAdd): + lhs = PStatement.get_str(expr.lhs, vars, ignore_vars=ignore_vars) + rhs = PStatement.get_str(expr.rhs, vars, ignore_vars=ignore_vars) + if lhs is None or rhs is None: + return None + return lhs + rhs elif isinstance(expr, PEUndef): - return None # type: ignore - - raise RuntimeError(f"Unsupported expression, got {expr}") + return None + else: + logging.warning(f"Unsupported expression, got {expr}") + return None def __eval(self, expr: PExpr, vars: Dict[str, PExpr]) -> PExpr | None: if isinstance(expr, PEVar) and expr.id.startswith("dejavu-condition"): return expr - if isinstance(expr, PEVar): - return self.__eval(vars[expr.id], vars) + if isinstance(expr, PEVar) and self.__get_var(expr.id, vars) is not None: + res = self.__get_var(expr.id, vars) + assert res is not None + return self.__eval(res, vars) + elif isinstance(expr, PRLet): + return self.__eval(expr.expr, vars) elif isinstance(expr, PEUndef) or isinstance(expr, PEConst): # NOTE: it is an arbitrary string to represent an undefined value return expr @@ -164,28 +233,26 @@ def minimize(statement: "PStatement", considered_paths: List[str]) -> "PStatemen """ def minimize_aux( - statement: "PStatement", considered_paths: Sequence[PExpr] + statement: "PStatement", + considered_paths: List[str], + vars: Dict[str, PExpr], ) -> "PStatement": - # FIXME compile statement.path - if isinstance(statement, PMkdir) and statement.path in considered_paths: - return statement - elif isinstance(statement, PCreate) and statement.path in considered_paths: - return statement - elif isinstance(statement, PWrite) and statement.path in considered_paths: - return statement - elif isinstance(statement, PRm) and statement.path in considered_paths: - return statement - elif isinstance(statement, PCp) and ( - statement.src in considered_paths or statement.dst in considered_paths - ): - return statement - elif isinstance(statement, PChmod) and statement.path in considered_paths: - return statement - elif isinstance(statement, PChown) and statement.path in considered_paths: - return statement + if isinstance(statement, PAttr): + path = PStatement.get_str(statement.path, vars) + if path not in considered_paths: + return PSkip() + else: + return statement + elif isinstance(statement, PCp): + src = PStatement.get_str(statement.src, vars) + dst = PStatement.get_str(statement.dst, vars) + if src not in considered_paths and dst not in considered_paths: + return PSkip() + else: + return statement elif isinstance(statement, PSeq): - lhs = minimize_aux(statement.lhs, considered_paths) - rhs = minimize_aux(statement.rhs, considered_paths) + lhs = minimize_aux(statement.lhs, considered_paths, vars) + rhs = minimize_aux(statement.rhs, considered_paths, vars) if not isinstance(lhs, PSkip) and not isinstance(rhs, PSkip): return PSeq(lhs, rhs) elif isinstance(lhs, PSkip): @@ -193,8 +260,23 @@ def minimize_aux( elif isinstance(rhs, PSkip): return lhs elif isinstance(statement, PLet): - body = minimize_aux(statement.body, considered_paths) + vars[statement.id] = statement.expr + body = minimize_aux(statement.body, considered_paths, vars) if not isinstance(body, PSkip): + # Checks if the variable is used in the body + references = GetVarReferencesVisitor().visit(body) + for reference in references: + if ( + PStatement.__get_var( + reference.id, {statement.id: statement.expr} + ) + is not None + ): + break + else: + # The variable is not used in the body + return body + return PLet( statement.id, statement.expr, @@ -202,8 +284,8 @@ def minimize_aux( body, ) elif isinstance(statement, PIf): - cons = minimize_aux(statement.cons, considered_paths) - alt = minimize_aux(statement.alt, considered_paths) + cons = minimize_aux(statement.cons, considered_paths, vars) + alt = minimize_aux(statement.alt, considered_paths, vars) if not isinstance(cons, PSkip) or not isinstance(alt, PSkip): return PIf( statement.pred, @@ -213,53 +295,55 @@ def minimize_aux( return PSkip() - considered_paths_exprs: List[PEConst] = list( - map(lambda path: PEConst(const=PStr(value=path)), considered_paths) - ) - return minimize_aux(statement, considered_paths_exprs) + return minimize_aux(statement, considered_paths, {}) def to_filesystems( self, - fss: Union[FileSystemState, List[FileSystemState]] = [], + fss: Union[SystemState, List[SystemState]] = [], vars: Optional[Dict[str, PExpr]] = None, - ) -> List[FileSystemState]: - if isinstance(fss, FileSystemState): + ) -> List[SystemState]: + if isinstance(fss, SystemState): fss = [fss.copy()] elif fss == []: - fss = [FileSystemState()] + fss = [SystemState()] if vars is None: vars = {} - res_fss: List[FileSystemState] = [] + res_fss: List[SystemState] = [] for fs in fss: - get_str: Callable[[PExpr], str] = lambda expr: self.__get_str(expr, vars) + get_str: Callable[[PExpr], Optional[str]] = lambda expr: PStatement.get_str( + expr, vars + ) + + if isinstance(self, PAttr): + path = get_str(self.path) + if path is None: + res_fss.append(fs) + continue + else: + path = "" + + if path != "" and path not in fs.state: + fs.state[path] = State() if isinstance(self, PSkip): pass - elif isinstance(self, PMkdir): - fs.state[get_str(self.path)] = Dir(None, None) - elif isinstance(self, PCreate): - fs.state[get_str(self.path)] = File(None, None, None) - elif isinstance(self, PWrite): - path, content = get_str(self.path), get_str(self.content) - file = fs.state.get(path) - if isinstance(file, File): - file.content = content - elif isinstance(self, PRm): - fs.state[get_str(self.path)] = Nil() + elif isinstance(self, PAttr): + value = get_str(self.value) + if path != "" and value is not None: + fs.state[path].attrs[self.attr] = value + elif path != "": + fs.state[path].attrs[self.attr] = UNDEF elif isinstance(self, PCp): - fs.state[get_str(self.dst)] = fs.state[get_str(self.src)] - elif isinstance(self, PChmod): - path, mode = get_str(self.path), get_str(self.mode) - file = fs.state.get(path) - if isinstance(file, (File, Dir)): - file.mode = mode - elif isinstance(self, PChown): - path, owner = get_str(self.path), get_str(self.owner) - file = fs.state.get(path) - if isinstance(file, (File, Dir)): - file.owner = owner + try: + dst, src = get_str(self.dst), get_str(self.src) + if dst is None or src is None: + continue + fs.state[dst] = fs.state[src] + except ValueError: + logging.warning(f"Invalid path: {self.src} {self.dst}") + continue elif isinstance(self, PSeq): fss_lhs = self.lhs.to_filesystems(fs, vars) for fs_lhs in fss_lhs: @@ -299,24 +383,10 @@ class PSkip(PStatement): @dataclass -class PMkdir(PStatement): - path: PExpr - - -@dataclass -class PWrite(PStatement): - path: PExpr - content: PExpr - - -@dataclass -class PCreate(PStatement): - path: PExpr - - -@dataclass -class PRm(PStatement): +class PAttr(PStatement): path: PExpr + attr: str + value: PExpr @dataclass @@ -325,24 +395,6 @@ class PCp(PStatement): dst: PExpr -@dataclass -class PChmod(PStatement): - path: PExpr - mode: PExpr - - -@dataclass -class Chmod(PStatement): - path: PExpr - mode: PExpr - - -@dataclass -class PChown(PStatement): - path: PExpr - owner: PExpr - - @dataclass class PSeq(PStatement): lhs: PStatement @@ -350,15 +402,96 @@ class PSeq(PStatement): @dataclass -class PLet(PStatement): +class PLet(PExpr, PStatement): id: str expr: PExpr - label: Optional[int] + label: int body: PStatement +@dataclass +class PRLet(PExpr): + id: str + expr: PExpr + label: int + + @dataclass class PIf(PStatement): pred: PExpr cons: PStatement alt: PStatement + + +class TranverseDeltaPVisitor(ABC): + @abstractmethod + def visit(self, statement: PStatement | PExpr | PConst) -> Any: + if isinstance(statement, PEBinOP): + return self.visit_binop(statement) + elif isinstance(statement, PEUnOP): + return self.visit_unop(statement) + elif isinstance(statement, PAttr): + return self.visit_attr(statement) + elif isinstance(statement, PCp): + return self.visit_cp(statement) + elif isinstance(statement, PSeq): + return self.visit_seq(statement) + elif isinstance(statement, PLet): + return self.visit_let(statement) + elif isinstance(statement, PRLet): + return self.visit_rlet(statement) + elif isinstance(statement, PIf): + return self.visit_if(statement) + return None + + def visit_binop(self, binop: PEBinOP): + return self.visit(binop.lhs) + self.visit(binop.rhs) + + def visit_unop(self, unop: PEUnOP): + return self.visit(unop.operand) + + def visit_attr(self, stat: PAttr): + return self.visit(stat.path) + self.visit(stat.value) + + def visit_cp(self, stat: PCp): + return self.visit(stat.src) + self.visit(stat.dst) + + def visit_seq(self, stat: PSeq): + return self.visit(stat.lhs) + self.visit(stat.rhs) + + def visit_let(self, stat: PLet): + return self.visit(stat.expr) + self.visit(stat.body) + + def visit_rlet(self, stat: PRLet): + return self.visit(stat.expr) + + def visit_if(self, stat: PIf): + return self.visit(stat.pred) + self.visit(stat.cons) + self.visit(stat.alt) + + +class GetStringsVisitor(TranverseDeltaPVisitor): + def visit(self, statement: PStatement | PExpr | PConst) -> List[str]: + res = super().visit(statement) + if res is not None: + return res + elif isinstance(statement, PEConst): + return self.visit_const(statement) + return [] + + def visit_const(self, const: PEConst) -> List[str]: + if isinstance(const.const, PStr): + return [const.const.value] + return [] + + +class GetVarReferencesVisitor(TranverseDeltaPVisitor): + def visit(self, statement: PStatement | PExpr | PConst) -> List[PEVar]: + res = super().visit(statement) + if res is not None: + return res + elif isinstance(statement, PEVar): + return [statement] + return [] + + +from glitch.repair.interactive.values import UNDEF diff --git a/glitch/repair/interactive/filesystem.py b/glitch/repair/interactive/filesystem.py deleted file mode 100644 index d1145498..00000000 --- a/glitch/repair/interactive/filesystem.py +++ /dev/null @@ -1,35 +0,0 @@ -from typing import Dict, Optional -from dataclasses import dataclass - - -class State: - def __str__(self) -> str: - return self.__class__.__name__.lower() - - -@dataclass -class File(State): - mode: Optional[str] - owner: Optional[str] - content: Optional[str] - - -@dataclass -class Dir(State): - mode: Optional[str] - owner: Optional[str] - - -@dataclass -class Nil(State): - pass - - -class FileSystemState: - def __init__(self) -> None: - self.state: Dict[str, State] = {} - - def copy(self): - fs = FileSystemState() - fs.state = self.state.copy() - return fs diff --git a/glitch/repair/interactive/main.py b/glitch/repair/interactive/main.py new file mode 100644 index 00000000..b8f8c7d2 --- /dev/null +++ b/glitch/repair/interactive/main.py @@ -0,0 +1,76 @@ +import difflib +import subprocess + +from copy import deepcopy +from typing import List +from glitch.tech import Tech +from glitch.parsers.parser import Parser +from glitch.repr.inter import UnitBlock, UnitBlockType +from tempfile import NamedTemporaryFile + +from glitch.repair.interactive.tracer.tracer import STrace +from glitch.repair.interactive.compiler.labeler import GLITCHLabeler +from glitch.repair.interactive.compiler.compiler import DeltaPCompiler +from glitch.repair.interactive.tracer.transform import ( + get_affected_paths, + get_file_system_state, +) +from glitch.repair.interactive.solver import PatchSolver, PatchApplier +from glitch.repair.interactive.compiler.names_database import NormalizationVisitor +from glitch.repair.interactive.delta_p import PStatement + + +def run_infrafix(path: str, pid: str, parser: Parser, type: UnitBlockType, tech: Tech): + inter: UnitBlock | None = parser.parse_file(path, type) + assert inter is not None + NormalizationVisitor(tech).visit(inter) + labeled_script = GLITCHLabeler.label(inter, tech) + statement = DeltaPCompiler(labeled_script).compile() + + syscalls = STrace(pid).run() + workdir = subprocess.check_output([f"pwdx {pid}"], shell=True) + workdir = workdir.decode("utf-8").strip().split(": ")[1] + sys_affected_paths = list(get_affected_paths(workdir, syscalls)) + + for i, path in enumerate(sys_affected_paths): + print(f"{i}: {path}") + indexes = input( + "Enter the indexes for the paths you wish to consider (separated by comma): " + ) + path_indexes = list(map(int, indexes.split(","))) + affected_paths: List[str] = [sys_affected_paths[i] for i in path_indexes] + statement = PStatement.minimize(statement, list(set(affected_paths))) + + filesystem_state = get_file_system_state(set(affected_paths)) + + solver = PatchSolver(statement, filesystem_state) + patches = solver.solve() + assert patches is not None + + with open(labeled_script.script.path) as f: + original_file = f.read() + + for i, patch in enumerate(patches): + copy_labeled_script = deepcopy(labeled_script) + with NamedTemporaryFile(mode="w+") as f: + f.write(original_file) + f.flush() + copy_labeled_script.script.path = f.name + PatchApplier(solver).apply_patch(patch, copy_labeled_script) + + f.seek(0, 0) + patch_file = f.read() + patch = difflib.unified_diff( + original_file.splitlines(), patch_file.splitlines() + ) + print("=" * 20 + f" Patch {i} " + "=" * 20) + print( + "".join( + [line + "\n" if not line.endswith("\n") else line for line in patch] + ) + ) + print("=" * 49) + p = int(input("Enter the patch number you wish to apply: ")) + + PatchApplier(solver).apply_patch(patches[p], labeled_script) + print(f"Patch applied to file {labeled_script.script.path} successfully!") diff --git a/glitch/repair/interactive/solver.py b/glitch/repair/interactive/solver.py index 50dc8816..8330e156 100644 --- a/glitch/repair/interactive/solver.py +++ b/glitch/repair/interactive/solver.py @@ -1,209 +1,278 @@ +import z3 +import re +import subprocess +import tempfile import time +import random +import string +import logging +import glitch.repr.inter as inter from copy import deepcopy -from typing import List, Callable, Tuple, Any +from typing import List, Callable, Tuple, Any, Literal from z3 import ( Solver, - sat, If, - StringVal, IntVal, - String, + BoolVal, Bool, And, Not, Int, Or, Sum, - ModelRef, + Concat, + StringVal, + SeqRef, Context, ExprRef, ) -from glitch.repair.interactive.filesystem import FileSystemState -from glitch.repair.interactive.filesystem import * +from glitch.repair.interactive.system import SystemState +from glitch.repair.interactive.system import * from glitch.repair.interactive.delta_p import * -from glitch.repair.interactive.values import DefaultValue, UNDEF +from glitch.repair.interactive.values import UNDEF, UNSUPPORTED from glitch.repair.interactive.compiler.labeler import LabeledUnitBlock -from glitch.repr.inter import Attribute, AtomicUnit, CodeElement, Block -from glitch.repair.interactive.compiler.labeler import GLITCHLabeler +from glitch.repr.inter import ( + Attribute, + AtomicUnit, + CodeElement, + ElementInfo, + Variable, + UnitBlock, +) from glitch.repair.interactive.compiler.names_database import NamesDatabase - -Fun = Callable[[ExprRef], ExprRef] +from glitch.repair.interactive.compiler.template_database import TemplateDatabase +from glitch.tech import Tech class PatchSolver: - @dataclass - class __Funs: - state_fun: Fun - contents_fun: Fun - mode_fun: Fun - owner_fun: Fun - def __init__( self, statement: PStatement, - filesystem: FileSystemState, + filesystem: SystemState, timeout: int = 180, + memory_limit: int = 1024 * 1024, ctx: Optional[Context] = None, + debug: bool = False, ) -> None: - # FIXME: the filesystem in here should be generated from - # checking the affected paths in statement - self.solver = Solver(ctx=ctx) + if ctx is None: + self.ctx = z3.Context() + self.debug = debug + self.memory_limit = memory_limit + self.constraints: List[ExprRef] = [] self.timeout = timeout self.statement = statement - self.sum_var = Int("sum") + self.sum_var = Int("sum", ctx=self.ctx) self.unchanged: Dict[int, ExprRef] = {} self.vars: Dict[str, ExprRef] = {} self.holes: Dict[str, ExprRef] = {} - # FIXME: check the defaults - self.__funs = PatchSolver.__Funs( - lambda p: StringVal(UNDEF), - lambda p: self.__compile_expr(DefaultValue.DEFAULT_CONTENT), - lambda p: self.__compile_expr(DefaultValue.DEFAULT_MODE), - lambda p: self.__compile_expr(DefaultValue.DEFAULT_OWNER), - ) + self.possible_strings = GetStringsVisitor().visit(statement) + for path, state in filesystem.state.items(): + self.possible_strings.append(path) + for key, value in state.attrs.items(): + self.possible_strings.append(key) + self.possible_strings.append(value) + self.possible_strings += [ + UNDEF, + UNSUPPORTED, + "", + "directory", + ] + for i in range(len(self.possible_strings)): + self.possible_strings += self.possible_strings[i].split(":") + self.possible_strings = list(set(self.possible_strings)) - # TODO?: We might want to use the default file system state to update - # the funs - # default_fs = self.__get_default_fs() + # FIXME: check the defaults + self.__funs: Dict[str, Callable[[ExprRef], ExprRef]] = {} - labels = self.__collect_labels(statement) + labels = list(set(self.__collect_labels(statement))) for label in labels: - self.unchanged[label] = Int(f"unchanged-{label}") + self.unchanged[label] = Int(f"unchanged-{label}", ctx=self.ctx) constraints, self.__funs = self.__generate_soft_constraints( self.statement, self.__funs ) - for constraint in constraints: - self.solver.add(constraint) + self.constraints += constraints self.__generate_hard_constraints(filesystem) - self.solver.add(Sum(list(self.unchanged.values())) == self.sum_var) + self.constraints.append(Sum(list(self.unchanged.values())) == self.sum_var) + + def __get_var(self, id: str) -> Optional[ExprRef]: + scopes = id.split(":dejavu:") + while True: + if ":dejavu:".join(scopes) in self.vars: + return self.vars[":dejavu:".join(scopes)] + if len(scopes) == 1: + break + scopes.pop(-2) + + return None + + def __const_string(self, name: str) -> ExprRef: + var = z3.String(name, ctx=self.ctx) + self.constraints.append(Or(*[var == s for s in self.possible_strings])) + return var def __collect_labels(self, statement: PStatement | PExpr) -> List[int]: if isinstance(statement, PSeq): return self.__collect_labels(statement.lhs) + self.__collect_labels( statement.rhs ) + elif isinstance(statement, PAttr): + return self.__collect_labels(statement.path) + self.__collect_labels( + statement.value + ) elif isinstance(statement, PIf): return ( self.__collect_labels(statement.pred) + self.__collect_labels(statement.cons) + self.__collect_labels(statement.alt) ) - elif isinstance(statement, PLet) and isinstance(statement.label, int): - return [statement.label] + self.__collect_labels(statement.body) + elif isinstance(statement, PRLet): + return [statement.label] + elif isinstance(statement, PLet): + return self.__collect_labels(statement.body) + self.__collect_labels( + statement.expr + ) + elif isinstance(statement, PEBinOP): + return self.__collect_labels(statement.lhs) + self.__collect_labels( + statement.rhs + ) + return [] + + def __collect_vars(self, statement: PStatement | PExpr) -> List[str]: + if isinstance(statement, PSeq): + return self.__collect_vars(statement.lhs) + self.__collect_vars( + statement.rhs + ) + elif isinstance(statement, PAttr): + return self.__collect_vars(statement.path) + self.__collect_vars( + statement.value + ) + elif isinstance(statement, PIf): + return ( + self.__collect_vars(statement.pred) + + self.__collect_vars(statement.cons) + + self.__collect_vars(statement.alt) + ) + elif isinstance(statement, PLet): + return ( + [statement.id] + + self.__collect_vars(statement.body) + + self.__collect_vars(statement.expr) + ) return [] - def __compile_expr(self, expr: PExpr) -> ExprRef: + def __compile_expr(self, expr: PExpr) -> Tuple[ExprRef, List[ExprRef]]: + constraints: List[ExprRef] = [] + if isinstance(expr, PEConst) and isinstance(expr.const, PStr): - return StringVal(expr.const.value) + return StringVal(expr.const.value, ctx=self.ctx), constraints + elif isinstance(expr, PEConst) and isinstance(expr.const, PBool): + return StringVal(str(expr.const.value).lower(), ctx=self.ctx), constraints + # TODO: add scope handling. elif isinstance(expr, PEVar) and expr.id.startswith("dejavu-condition-"): - self.vars[expr.id] = Bool(expr.id) - return self.vars[expr.id] + self.vars[expr.id] = Bool(expr.id, ctx=self.ctx) + return self.vars[expr.id], constraints + elif isinstance(expr, PEVar) and self.__get_var(expr.id) is not None: + var = self.__get_var(expr.id) + assert var is not None + return var, constraints elif isinstance(expr, PEVar): - self.vars[expr.id] = String(expr.id) - return self.vars[expr.id] + self.vars[expr.id] = self.__const_string(expr.id) + return self.vars[expr.id], constraints elif isinstance(expr, PEUndef): # NOTE: it is an arbitrary string to represent an undefined value - return StringVal(UNDEF) + return StringVal(UNDEF, ctx=self.ctx), constraints + elif isinstance(expr, PRLet): + if expr.id in self.vars: + return self.vars[expr.id], constraints + constraints, _ = self.__generate_soft_constraints(expr, self.__funs) + return self.vars[expr.id], constraints elif isinstance(expr, PEBinOP) and isinstance(expr.op, PEq): - return self.__compile_expr(expr.lhs) == self.__compile_expr(expr.rhs) + lhs, lhs_constraints = self.__compile_expr(expr.lhs) + rhs, rhs_constraints = self.__compile_expr(expr.rhs) + return lhs == rhs, lhs_constraints + rhs_constraints + elif isinstance(expr, PEBinOP) and isinstance(expr.op, PAdd): + lhs, lhs_constraints = self.__compile_expr(expr.lhs) + rhs, rhs_constraints = self.__compile_expr(expr.rhs) + if (isinstance(lhs, SeqRef) and lhs.as_string() == UNSUPPORTED) or ( + isinstance(rhs, SeqRef) and rhs.as_string() == UNSUPPORTED + ): + return ( + StringVal(UNSUPPORTED, ctx=self.ctx), + lhs_constraints + rhs_constraints, + ) + return Concat(lhs, rhs), lhs_constraints + rhs_constraints - raise ValueError(f"Not supported {expr}") + logging.warning(f"Unsupported expression: {expr}") + return StringVal(UNSUPPORTED, ctx=self.ctx), constraints - def __generate_hard_constraints(self, filesystem: FileSystemState) -> None: + def __generate_hard_constraints(self, filesystem: SystemState) -> None: for path, state in filesystem.state.items(): - self.solver.add( - self.__funs.state_fun(StringVal(path)) == StringVal(str(state)) - ) - content, mode, owner = UNDEF, UNDEF, UNDEF - - if isinstance(state, File): - content = UNDEF if state.content is None else state.content - if isinstance(state, File) or isinstance(state, Dir): - mode = UNDEF if state.mode is None else state.mode - owner = UNDEF if state.owner is None else state.owner - - self.solver.add( - self.__funs.contents_fun(StringVal(path)) == StringVal(content) - ) - self.solver.add(self.__funs.mode_fun(StringVal(path)) == StringVal(mode)) - self.solver.add(self.__funs.owner_fun(StringVal(path)) == StringVal(owner)) + for key, value in state.attrs.items(): + self.constraints.append( + self.__funs[key](StringVal(path, ctx=self.ctx)) + == StringVal(value, ctx=self.ctx) + ) - def __generate_soft_constraints(self, statement: PStatement, funs: __Funs) -> Tuple[ + def __generate_soft_constraints( + self, + statement: PStatement | PExpr, + funs: Dict[str, Callable[[ExprRef], ExprRef]], + ) -> Tuple[ List[ExprRef], - __Funs, + Dict[str, Callable[[ExprRef], ExprRef]], ]: # Avoids infinite recursion + previous_funs = deepcopy(funs) funs = deepcopy(funs) - # NOTE: For now it doesn't make sense to update the funs for the - # default values because the else will always be the default value - previous_state_fun = funs.state_fun - previous_contents_fun = funs.contents_fun - previous_mode_fun = funs.mode_fun - previous_owner_fun = funs.owner_fun constraints: List[ExprRef] = [] - if isinstance(statement, PMkdir): - path = self.__compile_expr(statement.path) - funs.state_fun = lambda p: If( - p == path, StringVal("dir"), previous_state_fun(p) - ) - elif isinstance(statement, PCreate): - path = self.__compile_expr(statement.path) - funs.state_fun = lambda p: If( - p == path, StringVal("file"), previous_state_fun(p) - ) - elif isinstance(statement, PWrite): - path = self.__compile_expr(statement.path) - funs.contents_fun = lambda p: If( - p == path, - self.__compile_expr(statement.content), - previous_contents_fun(p), - ) - elif isinstance(statement, PRm): - path = self.__compile_expr(statement.path) - funs.state_fun = lambda p: If( - p == path, StringVal("nil"), previous_state_fun(p) + if isinstance(statement, PAttr): + path, constraints = self.__compile_expr(statement.path) + value, value_constraints = self.__compile_expr(statement.value) + constraints += value_constraints + + if statement.attr not in previous_funs: + previous_funs[statement.attr] = lambda p: StringVal(UNDEF, ctx=self.ctx) + funs[statement.attr] = lambda p: If( + p == path, value, previous_funs[statement.attr](p), ctx=self.ctx ) elif isinstance(statement, PCp): - dst, src = self.__compile_expr(statement.dst), self.__compile_expr( - statement.src - ) - funs.state_fun = lambda p: If( - p == dst, previous_state_fun(src), previous_state_fun(p) - ) - funs.contents_fun = lambda p: If( + src, src_constraints = self.__compile_expr(statement.src) + constraints += src_constraints + dst, dest_constraints = self.__compile_expr(statement.dst) + constraints += dest_constraints + + funs["state"] = lambda p: If( p == dst, - previous_contents_fun(src), - previous_contents_fun(p), + previous_funs["state"](src), + previous_funs["state"](p), + ctx=self.ctx, ) - funs.mode_fun = lambda p: If( - p == dst, previous_mode_fun(src), previous_mode_fun(p) - ) - funs.owner_fun = lambda p: If( - p == dst, previous_owner_fun(src), previous_owner_fun(p) + funs["content"] = lambda p: If( + p == dst, + previous_funs["content"](src), + previous_funs["content"](p), + ctx=self.ctx, ) - elif isinstance(statement, PChmod): - path = self.__compile_expr(statement.path) - funs.mode_fun = lambda p: If( - p == path, - self.__compile_expr(statement.mode), - previous_mode_fun(p), + funs["mode"] = lambda p: If( + p == dst, + previous_funs["mode"](src), + previous_funs["mode"](p), + ctx=self.ctx, ) - elif isinstance(statement, PChown): - path = self.__compile_expr(statement.path) - funs.owner_fun = lambda p: If( - p == path, - self.__compile_expr(statement.owner), - previous_owner_fun(p), + funs["owner"] = lambda p: If( + p == dst, + previous_funs["owner"](src), + previous_funs["owner"](p), + ctx=self.ctx, ) elif isinstance(statement, PSeq): - self.__generate_soft_constraints(statement.lhs, funs) lhs_constraints, funs = self.__generate_soft_constraints( statement.lhs, funs ) @@ -212,23 +281,42 @@ def __generate_soft_constraints(self, statement: PStatement, funs: __Funs) -> Tu statement.rhs, funs ) constraints += rhs_constraints - elif isinstance(statement, PLet): - hole, var = String(f"loc-{statement.label}"), String(statement.id) + elif isinstance(statement, PRLet): + hole, var = self.__const_string( + f"loc-{statement.label}" + ), self.__const_string(statement.id) self.holes[f"loc-{statement.label}"] = hole self.vars[statement.id] = var - unchanged = self.unchanged[statement.label] # type: ignore - constraints.append( - Or( # type: ignore - And(unchanged == 1, var == self.__compile_expr(statement.expr)), # type: ignore - And(unchanged == 0, var == hole), # type: ignore - ) # type: ignore + unchanged = self.unchanged[statement.label] + value, constraints = self.__compile_expr(statement.expr) + self.constraints.append( + Or( + And(unchanged == 1, var == value), + And(unchanged == 0, var == hole), + ) + ) + elif isinstance(statement, PLet): + if statement.id in self.vars: + var = self.vars[statement.id] + else: + var = z3.String(statement.id, ctx=self.ctx) + self.vars[statement.id] = var + hole = z3.String(f"loc-{statement.id}-{statement.label}", ctx=self.ctx) + self.holes[f"loc-{statement.label}"] = hole + + value, constraints = self.__compile_expr(statement.expr) + constraints.append(var == value) + constraints.append(var == hole) + expr_constraints, funs = self.__generate_soft_constraints( + statement.expr, funs ) + constraints += expr_constraints body_constraints, funs = self.__generate_soft_constraints( statement.body, funs ) constraints += body_constraints elif isinstance(statement, PIf): - condition = self.__compile_expr(statement.pred) + condition, constraints = self.__compile_expr(statement.pred) cons_constraints, cons_funs = self.__generate_soft_constraints( statement.cons, funs @@ -237,147 +325,545 @@ def __generate_soft_constraints(self, statement: PStatement, funs: __Funs) -> Tu statement.alt, funs ) - funs.state_fun = lambda p: If( - condition, cons_funs.state_fun(p), alt_funs.state_fun(p) - ) - funs.contents_fun = lambda p: If( - condition, cons_funs.contents_fun(p), alt_funs.contents_fun(p) - ) - funs.mode_fun = lambda p: If( - condition, cons_funs.mode_fun(p), alt_funs.mode_fun(p) - ) - funs.owner_fun = lambda p: If( - condition, cons_funs.owner_fun(p), alt_funs.owner_fun(p) - ) + keys = list(funs.keys()) + list(cons_funs.keys()) + list(alt_funs.keys()) + for key in keys: + if key not in cons_funs: + cons_funs[key] = lambda p: StringVal(UNDEF, ctx=self.ctx) + if key not in alt_funs: + alt_funs[key] = lambda p: StringVal(UNDEF, ctx=self.ctx) + cons = cons_funs[key] + alt = alt_funs[key] + + # The default attributes are required due to Python's deep binding + funs[key] = lambda p, cons=cons, alt=alt: If( + condition, cons(p), alt(p), ctx=self.ctx + ) + + # It allows to fix variables in the unchosen branch + labels_cons = self.__collect_labels(statement.cons) + labels_alt = self.__collect_labels(statement.alt) + + unchanged = True + for label in set(labels_cons) - set(labels_alt): + unchanged = And(unchanged, self.unchanged[label] == 1) + self.constraints.append(Or(condition, unchanged)) - for label in self.__collect_labels(statement.cons): - self.solver.add(Or(condition, self.unchanged[label] == 1)) - for label in self.__collect_labels(statement.alt): - self.solver.add(Or(Not(condition), self.unchanged[label] == 1)) + unchanged = True + for label in set(labels_alt) - set(labels_cons): + unchanged = And(unchanged, self.unchanged[label] == 1) + self.constraints.append(Or(Not(condition), unchanged)) + + vars_cons = self.__collect_vars(statement.cons) + vars_alt = self.__collect_vars(statement.alt) + + fixed_vars = True + for var in set(vars_cons) - set(vars_alt): + fixed_vars = And(fixed_vars, self.vars[var] == "") + self.constraints.append(Or(condition, fixed_vars)) + + fixed_vars = True + for var in set(vars_alt) - set(vars_cons): + fixed_vars = And(fixed_vars, self.vars[var] == "") + self.constraints.append(Or(Not(condition), fixed_vars)) - # NOTE: This works because the only constraints created should - # always be added. Its kinda of an HACK for constraint in cons_constraints: - # constraints.append(Or(Not(condition), And(condition, constraint))) - constraints.append(constraint) + constraints.append(Or(Not(condition), And(condition, constraint))) for constraint in alt_constraints: - constraints.append(constraint) - # constraints.append(Or(condition, And(Not(condition), constraint))) + constraints.append(Or(condition, And(Not(condition), constraint))) return constraints, funs - def solve(self) -> Optional[List[ModelRef]]: - models: List[ModelRef] = [] + def __decode_smtlib2_string(self, string: str) -> str: + """ + Converts SMTLIB2-style Unicode escape sequences in a string to their respective characters. - start = time.time() - elapsed = 0 + Parameters: + string (str): The input string containing SMTLIB2-style Unicode escapes. + + Returns: + str: The decoded string. + """ - while True and elapsed < self.timeout: + def unicode_replacer(match) -> str: # type: ignore + hex_value = match.group(1) # type: ignore + return chr(int(hex_value, 16)) # type: ignore + + # Replace all matches in the string + return re.sub(r"\\u\{([0-9A-Fa-f]+)\}", unicode_replacer, string) # type: ignore + + def __parse_z3_output(self, z3_output: str) -> Dict[str, Any]: + define_fun_pattern = re.compile( + r"\(define-fun (\S+) \(\) (\S+)\n\s+(.+?)\)", re.DOTALL + ) + parsed_data: Dict[str, Any] = {} + + for match in define_fun_pattern.finditer(z3_output): + name = match.group(1) + if name.startswith("|") and name.endswith("|"): + name = name[1:-1] + datatype = match.group(2) + value = match.group(3).strip() + + if datatype == "String": + # Replace Z3 double-escaped quotes (e.g., "") with single quotes (") + value = value.replace('""', '"') + value = value.strip('"') + elif datatype == "Int": + value = int(value) + elif datatype == "Bool" and value == "true": + value = True + elif datatype == "Bool" and value == "false": + value = False + + parsed_data[name] = value + + return parsed_data + + def __add_named_to_assertions(self, smtlib_code: str) -> Tuple[str, Dict[str, str]]: + def generate_random_name(length: int = 10): + return "A" + "".join( + random.choices(string.ascii_letters + string.digits, k=length) + ) + + result: List[str] = [] + names_to_assertions: Dict[str, str] = {} + stack: List[int] = [] + start_index = None + inside_assert = False + i = 0 + + while i < len(smtlib_code): + char = smtlib_code[i] + + if char == "(": + stack.append(i) + if not inside_assert and smtlib_code[i : i + 7] == "(assert": + start_index = i + inside_assert = True + + elif char == ")": + stack.pop() + if inside_assert and not stack and start_index is not None: + assert_content = smtlib_code[start_index + 7 : i].strip() + random_name = generate_random_name() + names_to_assertions[random_name] = assert_content + transformed_assert = ( + f"(assert (! {assert_content} :named {random_name}))" + ) + result.append(transformed_assert) + start_index = None + inside_assert = False + i += 1 + continue + + if not inside_assert: + result.append(char) + i += 1 + + return "".join(result), names_to_assertions + + def __run_z3(self, smt2: str, timeout: int) -> str: + temp = tempfile.NamedTemporaryFile(mode="w+") + temp.write(smt2) + temp.flush() + result = subprocess.run( + f"ulimit -v {self.memory_limit} && timeout {timeout} z3 -smt2 -model {temp.name}", + shell=True, + text=True, + capture_output=True, + ) + if result.returncode == 124: + raise TimeoutError("Solver timed out") + elif result.returncode in [137, 139]: + raise MemoryError("Solver ran out of memory") + temp.close() + return result.stdout + + def __run_solver(self, timeout: int) -> Tuple[bool, Optional[Dict[str, Any]]]: + solver = Solver(ctx=self.ctx) + for constraint in self.constraints: + solver.add(constraint) + + output = self.__run_z3(solver.to_smt2(), timeout) + result = output.split("\n")[0] + if result.strip() == "sat": + model = output.split("\n", 1)[1] + model = self.__decode_smtlib2_string(model) + res: Dict[str, Any] = self.__parse_z3_output(model) + + return True, res + elif result.strip() == "unsat" and self.debug: + smt2, names_to_assertions = self.__add_named_to_assertions(solver.to_smt2()) + smt2 = ( + "(set-option :produce-unsat-cores true)\n" + smt2 + "\n(get-unsat-core)" + ) + output = self.__run_z3(smt2, timeout) + unsat_core = output.split("\n", 1)[1].strip()[1:-1].split(" ") + print("=== Unsat core: ===") + for name in unsat_core: + print(names_to_assertions[name]) + print("===================") + + return False, None + + def __get_solver_value(self, value: Any) -> ExprRef | bool: + if isinstance(value, str): + return StringVal(value, ctx=self.ctx) + elif isinstance(value, int): + return IntVal(value, ctx=self.ctx) + elif isinstance(value, bool): + return BoolVal(value, ctx=self.ctx) + raise ValueError(f"Unsupported value type: {type(value)}") + + def solve(self) -> List[Dict[str, Any]]: + models: List[Dict[str, Any]] = [] + + start = time.time() + while True and time.time() - start < self.timeout: lo, hi = 0, len(self.unchanged) + 1 model = None - while lo < hi and elapsed < self.timeout: + while lo < hi and time.time() - start < self.timeout: mid = (lo + hi) // 2 - self.solver.push() - self.solver.add(self.sum_var >= IntVal(mid)) - if self.solver.check() == sat: + self.constraints.append(self.sum_var >= IntVal(mid, ctx=self.ctx)) + sat, o_model = self.__run_solver( + self.timeout - int(time.time() - start) + ) + if sat: lo = mid + 1 - model = self.solver.model() - self.solver.pop() - elapsed = time.time() - start + model = o_model + self.constraints.pop() continue else: hi = mid - self.solver.pop() - elapsed = time.time() - start - elapsed = time.time() - start + self.constraints.pop() + if model is None: break models.append(model) # Removes conditional variables that were not used - dvars = filter(lambda v: model[v] is not None, self.vars.values()) # type: ignore - self.solver.add(Not(And([v == model[v] for v in dvars]))) + dvars = list( + filter(lambda v: str(v) in model, self.vars.values()) # type: ignore + ) + + self.constraints.append( + Not(And([v == self.__get_solver_value(model[str(v)]) for v in dvars])) + ) + + if time.time() - start >= self.timeout: + raise TimeoutError("Solver timed out") - if elapsed >= self.timeout: - return None return models - @staticmethod - def __find_atomic_unit( - labeled_script: LabeledUnitBlock, attribute: Attribute - ) -> Optional[AtomicUnit]: - def aux_find_atomic_unit(code_element: CodeElement) -> Optional[AtomicUnit]: - if ( - isinstance(code_element, AtomicUnit) - and attribute in code_element.attributes - ): - return code_element - elif isinstance(code_element, Block): - for statement in code_element.statements: - result = aux_find_atomic_unit(statement) - if result is not None: - return result - return None - - code_elements = ( - labeled_script.script.statements + labeled_script.script.atomic_units - ) - for code_element in code_elements: - result = aux_find_atomic_unit(code_element) - if result is not None: - return result - raise ValueError(f"Attribute {attribute} not found in the script") +@dataclass +class PatchChange: + value: str + codeelement: CodeElement + type: Literal["add_sketch", "delete", "modify"] + info: ElementInfo + + +class PatchApplier: + def __init__(self, solver: PatchSolver) -> None: + self.unchanged = solver.unchanged + self.holes = solver.holes # TODO: improve way to identify sketch def __is_sketch(self, codeelement: CodeElement) -> bool: return codeelement.line < 0 and codeelement.column < 0 - def apply_patch( - self, model_ref: ModelRef, labeled_script: LabeledUnitBlock + def __add_sketch_attribute( + self, + labeled_script: LabeledUnitBlock, + attribute: Attribute, + atomic_unit: AtomicUnit, + value: str, + tech: Tech, ) -> None: + # FIXME: There should be a way to change the way we apply the patch + # according to the technology + if tech == Tech.terraform and attribute.name == "state": + return + + name, _ = NamesDatabase.get_attr_pair( + inter.String(value, ElementInfo(-1, -1, -1, -1, "")), + attribute.name, + atomic_unit.type, + tech, + ) + # FIXME + is_string = name not in ["state", "enabled"] and value not in ["true", "false"] + atomic_unit.attributes.append(attribute) + + path = labeled_script.script.path + + old_value = attribute.value + assert old_value is not None + attribute.value = inter.String( + value, + ElementInfo.from_code_element(old_value), + ) # FIXME + + with open(path, "r") as f: + lines = f.readlines() + + last_attribute = None + for attr in atomic_unit.attributes: + if not self.__is_sketch(attr): + last_attribute = attr + if last_attribute is None: + line = atomic_unit.line + 1 + col = 2 + else: + line = last_attribute.line + 1 + col = len(lines[line - 2]) - len(lines[line - 2].lstrip()) + attribute.line = line + new_line = TemplateDatabase.get_template(attribute, tech) + if tech == Tech.terraform: + value = value if not is_string else f'"{value}"' + else: + value = value if not is_string else f"'{value}'" + new_line = col * " " + new_line.format(attribute.name, value) + lines.insert(line - 1, new_line) + if not lines[line - 2].endswith("\n"): + lines[line - 2] = lines[line - 2] + "\n" + with open(path, "w") as f: + f.writelines(lines) + + labeled_script.add_label(attribute.name, attribute) + + def __delete_code_element(self, labeled_script: LabeledUnitBlock, ce: CodeElement): + path = labeled_script.script.path + with open(path, "r") as f: + lines = f.readlines() + + line = ce.line - 1 + lines[line] = lines[line][: ce.column - 1] + lines[line][ce.end_column :] + if lines[line].strip() == "": + lines.pop(line) + + with open(path, "w") as f: + f.writelines(lines) + + def __delete_attribute( + self, + labeled_script: LabeledUnitBlock, + ce: AtomicUnit | UnitBlock, + attribute: Attribute, + ) -> None: + if attribute in ce.attributes: + ce.attributes.remove(attribute) + self.__delete_code_element(labeled_script, attribute) + + def __delete_variable( + self, + labeled_script: LabeledUnitBlock, + ce: UnitBlock, + variable: Variable, + ): + if variable in ce.variables: + ce.variables.remove(variable) + self.__delete_code_element(labeled_script, variable) + + def __modify_codeelement( + self, + labeled_script: LabeledUnitBlock, + codeelement: CodeElement, + value: str, + tech: Tech, + ): + with open(labeled_script.script.path, "r") as f: + lines = f.readlines() + + old_line = lines[codeelement.line - 1] + start = codeelement.column - 1 + + if codeelement.line != codeelement.end_line: + # Cute the other lines + lines = lines[: codeelement.line] + lines[codeelement.end_line :] + end = len(old_line) - 1 + else: + end = codeelement.end_column - 1 + + if ( + value not in ["true", "false"] # FIXME + and ( + old_line[start:end].startswith('"') + or codeelement.code.startswith('"') + ) + and ( + old_line[start:end].endswith('"') or codeelement.code.endswith('"') + ) + ): + value = f'"{value}"' + elif ( + value not in ["true", "false"] # FIXME + and ( + old_line[start:end].startswith("'") + or codeelement.code.startswith("'") + ) + and ( + old_line[start:end].endswith("'") or codeelement.code.endswith("'") + ) + ): + value = f"'{value}'" + elif len(value.split("\n")) > 1: + value = TemplateDatabase.get_template_for_multiline_string(tech).format( + value + ) + + if old_line[end - 1] == "\n": + value = f"{value}\n" + new_line = old_line[:start] + value + old_line[end:] + lines[codeelement.line - 1] = new_line + + with open(labeled_script.script.path, "w") as f: + f.writelines(lines) + + def get_changes( + self, model_ref: Dict[str, Any], labeled_script: LabeledUnitBlock + ) -> List[PatchChange]: changed: List[Tuple[int, Any]] = [] for label, unchanged in self.unchanged.items(): - if model_ref[unchanged] == 0: # type: ignore - hole = self.holes[f"loc-{label}"] - changed.append((label, model_ref[hole])) + if model_ref[str(unchanged)] == 0: # type: ignore + changed.append((label, model_ref[f"loc-{label}"])) + + # Track attributes that became undefined + for hole in self.holes: + var = str(self.holes[hole]) + if model_ref[var] == UNDEF: # type: ignore + if hole.rsplit("-", 1)[0].endswith("-"): # Avoid sketches + continue + label = int(hole.rsplit("-", 1)[-1]) + if label not in self.unchanged: # Make sure it is not a literal + changed.append((label, model_ref[var])) + + changes: List[PatchChange] = [] for change in changed: label, value = change - value = value.as_string() + value = value codeelement = labeled_script.get_codeelement(label) - if not isinstance(codeelement, Attribute): + + assert isinstance(codeelement, (inter.Expr, inter.KeyValue)) + if not isinstance(codeelement, (inter.String, inter.Null, inter.KeyValue)): + # HACK: This allows to fix unsupported expressions + codeelement = inter.String( + value, ElementInfo.from_code_element(codeelement) + ) + codeelement.code = "''" + + info = ElementInfo.from_code_element(codeelement) + if value == UNDEF: + changes.append(PatchChange(value, codeelement, "delete", info)) continue - if self.__is_sketch(codeelement): - atomic_unit = labeled_script.get_sketch_location(codeelement) - if not isinstance(atomic_unit, AtomicUnit): - raise RuntimeError("Atomic unit not found") + kv = labeled_script.get_location(codeelement) + if not self.__is_sketch(codeelement) or isinstance(kv, Variable): + changes.append(PatchChange(value, codeelement, "modify", info)) + elif self.__is_sketch(codeelement) and isinstance(kv, Attribute): + au = labeled_script.get_location(kv) + assert isinstance(au, AtomicUnit) + if len(au.attributes) == 0: + info = ElementInfo.from_code_element(au) + else: + info = ElementInfo.from_code_element(au.attributes[-1]) + changes.append(PatchChange(value, codeelement, "add_sketch", info)) - atomic_unit_type = NamesDatabase.get_au_type( - atomic_unit.type, labeled_script.tech - ) - name = NamesDatabase.reverse_attr_name( - codeelement.name, atomic_unit_type, labeled_script.tech - ) - codeelement.name = name - atomic_unit.attributes.append(codeelement) - # Remove sketch label and add regular label - labeled_script.remove_label(codeelement) - GLITCHLabeler.label_attribute(labeled_script, atomic_unit, codeelement) - else: - atomic_unit = PatchSolver.__find_atomic_unit( - labeled_script, codeelement - ) + # The sort is necessary to avoid problems in the textual changes + changes.sort(key=lambda x: (x.info.line, x.info.column), reverse=True) + return changes + + def reverse_changes( + self, labeled_script: LabeledUnitBlock, changes: List[PatchChange] + ) -> None: + def reverse(value: str, attr: str, loc_loc: AtomicUnit) -> Tuple[str, str]: + attr_name = NamesDatabase.reverse_attr_name( + attr, loc_loc.type, labeled_script.tech + ) + value = NamesDatabase.reverse_attr_value( + value, + attr, + loc_loc.type, + labeled_script.tech, + ) + return attr_name, value + + for change in changes: + if isinstance(change.codeelement, (inter.String, inter.Null)): + loc = labeled_script.get_location(change.codeelement) + if isinstance(loc, Attribute): + loc_loc = labeled_script.get_location(loc) + if isinstance(loc_loc, AtomicUnit): + loc.name, change.value = reverse( + change.value, loc.name, loc_loc + ) + elif isinstance(loc, AtomicUnit): + # Only for paths in the name + _, change.value = reverse(change.value, "path", loc) + loc.name = inter.String(change.value, change.info) - # Remove attributes that are not defined - if value == UNDEF and isinstance(atomic_unit, AtomicUnit): - atomic_unit.attributes.remove(codeelement) - labeled_script.remove_label(codeelement) - elif isinstance(atomic_unit, AtomicUnit): - codeelement.value = NamesDatabase.reverse_attr_value( - value, codeelement.name, atomic_unit.type, labeled_script.tech + def apply_patch( + self, model_ref: Dict[str, Any], labeled_script: LabeledUnitBlock + ) -> None: + changed_elements = self.get_changes(model_ref, labeled_script) + self.reverse_changes(labeled_script, changed_elements) + deleted_kvs: List[inter.KeyValue] = [] + + for ce in changed_elements: + # Deleted Elements + if ce.type == "delete": + if isinstance(ce.codeelement, inter.KeyValue): + loc = ce.codeelement + else: + loc = labeled_script.get_location(ce.codeelement) + assert isinstance(loc, inter.KeyValue) + + loc_loc = labeled_script.get_location(loc) + assert isinstance(loc_loc, (AtomicUnit, UnitBlock)) + + if loc not in deleted_kvs: + if isinstance(loc, Attribute): + self.__delete_attribute(labeled_script, loc_loc, loc) + elif isinstance(loc, Variable): + assert isinstance(loc_loc, UnitBlock) + self.__delete_variable(labeled_script, loc_loc, loc) + deleted_kvs.append(loc) + # Modified elements + elif ce.type == "modify": + loc = labeled_script.get_location(ce.codeelement) + if isinstance(ce.codeelement, inter.Null): + ce.codeelement = inter.String(ce.value, ce.info) + # HACK: Allows to add quotes in the modify_codeelement func + ce.codeelement.code = '""' + assert isinstance(ce.codeelement, inter.String) + + if isinstance(loc, Attribute): + ce.codeelement.value = ce.value + loc.value = ce.codeelement + self.__modify_codeelement( + labeled_script, ce.codeelement, ce.value, labeled_script.tech + ) + elif isinstance(loc, Variable): + ce.codeelement.value = ce.value + self.__modify_codeelement( + labeled_script, ce.codeelement, ce.value, labeled_script.tech + ) + elif isinstance(loc, AtomicUnit): + ce.codeelement.value = ce.value + loc.name = ce.codeelement + self.__modify_codeelement( + labeled_script, ce.codeelement, ce.value, labeled_script.tech + ) + elif ce.type == "add_sketch": + loc = labeled_script.get_location(ce.codeelement) + assert isinstance(loc, Attribute) + loc_loc = labeled_script.get_location(loc) + assert isinstance(loc_loc, AtomicUnit) + self.__add_sketch_attribute( + labeled_script, loc, loc_loc, ce.value, labeled_script.tech ) diff --git a/glitch/repair/interactive/system.py b/glitch/repair/interactive/system.py new file mode 100644 index 00000000..651fbb3f --- /dev/null +++ b/glitch/repair/interactive/system.py @@ -0,0 +1,24 @@ +from typing import Dict + + +class State: + def __init__(self) -> None: + self.attrs: Dict[str, str] = {} + + def __eq__(self, value: object) -> bool: + if not isinstance(value, State): + return False + return self.attrs == value.attrs + + def __repr__(self) -> str: + return str(self.attrs) + + +class SystemState: + def __init__(self) -> None: + self.state: Dict[str, State] = {} + + def copy(self): + fs = SystemState() + fs.state = self.state.copy() + return fs diff --git a/glitch/repair/interactive/tracer/model.py b/glitch/repair/interactive/tracer/model.py index d41abbed..334f411b 100644 --- a/glitch/repair/interactive/tracer/model.py +++ b/glitch/repair/interactive/tracer/model.py @@ -34,6 +34,9 @@ def get_syscall_with_type(syscall: Syscall) -> Syscall: "unlink": "SUnlink", "unlinkat": "SUnlinkAt", "chdir": "SChdir", + "fchmodat": "SFchmodAt", + "chmod": "SChmod", + "fchownat": "FChownAt", } if syscall.cmd in verbs: return globals()[verbs[syscall.cmd]]( @@ -194,3 +197,45 @@ def path(self) -> str: @property def flags(self) -> Union[List[UnlinkFlag], str]: return self.args[2] + + +class SFchmodAt(Syscall): + @property + def dirfd(self) -> str: + return self.args[0] + + @property + def path(self) -> str: + return self.args[1] + + @property + def mode(self) -> str: + return self.args[2] + + +class SChmod(Syscall): + @property + def path(self) -> str: + return self.args[0] + + @property + def mode(self) -> str: + return self.args[1] + + +class FChownAt(Syscall): + @property + def dirfd(self) -> str: + return self.args[0] + + @property + def path(self) -> str: + return self.args[1] + + @property + def owner(self) -> str: + return self.args[2] + + @property + def group(self) -> str: + return self.args[3] diff --git a/glitch/repair/interactive/tracer/parser.py b/glitch/repair/interactive/tracer/parser.py index 47106f43..106b2fee 100644 --- a/glitch/repair/interactive/tracer/parser.py +++ b/glitch/repair/interactive/tracer/parser.py @@ -60,7 +60,7 @@ class UnlinkFlag(Enum): AT_REMOVEDIR = 0 -def parse_tracer_output(tracer_output: str, debug: bool = False) -> Syscall: +def parse_tracer_output(tracer_output: str, debug: bool = False) -> Syscall | None: # Tokens defined as functions preserve order def t_ADDRESS(t: LexToken): r"0[xX][0-9a-fA-F]+" @@ -147,7 +147,7 @@ def t_COMMENT(t: LexToken) -> None: # Ignore comments def t_ANY_error(t: LexToken) -> None: - logging.error(f"Illegal character {t.value[0]!r}.") + logging.debug(f"Illegal character {t.value[0]!r}.") t.lexer.skip(1) lexer = lex() @@ -291,8 +291,9 @@ def p_unlink_flags(p: YaccProduction) -> None: r"unlink_flags : unlink_flags PIPE UNLINK_FLAG" p[0] = p[1] + [p[3]] - def p_error(p: YaccProduction) -> None: - logging.error(f"Syntax error at {p.value!r}") + def p_error(p: YaccProduction | None) -> None: + if p is not None: + logging.debug(f"Syntax error at {p.value!r}") # Build the parser parser = yacc() diff --git a/glitch/repair/interactive/tracer/tracer.py b/glitch/repair/interactive/tracer/tracer.py index faabb34d..d46e1a4a 100644 --- a/glitch/repair/interactive/tracer/tracer.py +++ b/glitch/repair/interactive/tracer/tracer.py @@ -42,10 +42,16 @@ def run(self) -> List[Syscall]: # type: ignore or "--- SIG" in line ): continue - syscall = get_syscall_with_type(parse_tracer_output(line)) + + parsed_syscall = parse_tracer_output(line) + if parsed_syscall is None: + continue + syscall = get_syscall_with_type(parsed_syscall) + if "/bin/synth-glitch" in syscall.args: # TODO don't break and just update break self.syscalls.append(syscall) + proc.kill() return self.syscalls diff --git a/glitch/repair/interactive/tracer/transform.py b/glitch/repair/interactive/tracer/transform.py index 4314cf3c..cebc48bf 100644 --- a/glitch/repair/interactive/tracer/transform.py +++ b/glitch/repair/interactive/tracer/transform.py @@ -3,7 +3,8 @@ from typing import Set, Callable from glitch.repair.interactive.tracer.model import * -from glitch.repair.interactive.filesystem import * +from glitch.repair.interactive.system import * +from glitch.repair.interactive.values import UNDEF def get_affected_paths(workdir: str, syscalls: List[Syscall]) -> Set[str]: @@ -43,32 +44,44 @@ def abspath(workdir: str, path: str): elif isinstance(syscall, SRename): paths.add(abspath(workdir, syscall.src)) paths.add(abspath(workdir, syscall.dst)) - elif isinstance(syscall, (SUnlink, SUnlinkAt, SRmdir, SMkdir, SMkdirAt)): + elif isinstance( + syscall, + (SUnlink, SUnlinkAt, SRmdir, SMkdir, SMkdirAt, SChmod, SFchmodAt, FChownAt), + ): paths.add(abspath(workdir, syscall.path)) elif isinstance(syscall, SChdir): workdir = syscall.path + paths = {path for path in paths if path != "/dev/null"} return paths -def get_file_system_state(files: Set[str]) -> FileSystemState: +def get_file_system_state(files: Set[str]) -> SystemState: """Get the file system state from the given set of files. Args: files: A set of files. Returns: - FileSystemState: The file system state. + SystemState: The file system state. """ - fs = FileSystemState() + fs = SystemState() get_owner: Callable[[str], str] = lambda f: getpwuid(os.stat(f).st_uid).pw_name - get_mode: Callable[[str], str] = lambda f: oct(os.stat(f).st_mode & 0o777)[2:] + get_mode: Callable[[str], str] = lambda f: "0" + oct(os.stat(f).st_mode & 0o777)[2:] for file in files: if not os.path.exists(file): - fs.state[file] = Nil() + fs.state[file] = State() + fs.state[file].attrs["state"] = "absent" + fs.state[file].attrs["mode"] = UNDEF + fs.state[file].attrs["owner"] = UNDEF + fs.state[file].attrs["content"] = UNDEF elif os.path.isdir(file): - fs.state[file] = Dir(get_mode(file), get_owner(file)) + fs.state[file] = State() + fs.state[file].attrs["state"] = "directory" + fs.state[file].attrs["mode"] = get_mode(file) + fs.state[file].attrs["owner"] = get_owner(file) + fs.state[file].attrs["content"] = UNDEF elif os.path.isfile(file): with open(file, "rb") as f: bytes = f.read() @@ -76,6 +89,10 @@ def get_file_system_state(files: Set[str]) -> FileSystemState: content = bytes.decode("utf-8") except UnicodeDecodeError: content = bytes.hex() - fs.state[file] = File(get_mode(file), get_owner(file), content) + fs.state[file] = State() + fs.state[file].attrs["state"] = "present" + fs.state[file].attrs["mode"] = get_mode(file) + fs.state[file].attrs["owner"] = get_owner(file) + fs.state[file].attrs["content"] = content return fs diff --git a/glitch/repair/interactive/values.py b/glitch/repair/interactive/values.py index 83d4d93a..a6f32213 100644 --- a/glitch/repair/interactive/values.py +++ b/glitch/repair/interactive/values.py @@ -1,10 +1,4 @@ from glitch.repair.interactive.delta_p import * -UNDEF = "glitch-undef" - - -class DefaultValue: - DEFAULT_MODE = PEConst(PStr("644")) - DEFAULT_OWNER = PEConst(PStr("root")) - DEFAULT_STATE = PEConst(PStr("present")) - DEFAULT_CONTENT = PEUndef() +UNDEF = "glitch-undef" # type: ignore +UNSUPPORTED = "glitch-unsupported" diff --git a/glitch/repr/inter.py b/glitch/repr/inter.py index 1dcf74d0..2028f88b 100644 --- a/glitch/repr/inter.py +++ b/glitch/repr/inter.py @@ -1,16 +1,65 @@ from abc import ABC from enum import Enum -from typing import List, Union, Dict, Any +from dataclasses import dataclass +from typing import List, Union, Dict, Any, ClassVar + +UNDEFINED_POSITION = -33550336 + + +@dataclass +class ElementInfo: + line: int + column: int + end_line: int + end_column: int + code: str + sketched: ClassVar[int] = -1 + + @staticmethod + def from_code_element(element: "CodeElement") -> "ElementInfo": + return ElementInfo( + element.line, + element.column, + element.end_line, + element.end_column, + element.code, + ) + + @staticmethod + def get_sketched() -> "ElementInfo": + info = ElementInfo( + ElementInfo.sketched, + ElementInfo.sketched, + ElementInfo.sketched, + ElementInfo.sketched, + "", + ) + ElementInfo.sketched -= 1 + return info class CodeElement(ABC): - def __init__(self) -> None: - self.line: int = -1 - self.column: int = -1 - self.code: str = "" + def __init__(self, info: ElementInfo | None = None) -> None: + if info is not None: + self.line: int = info.line + self.column: int = info.column + self.end_line: int = info.end_line + self.end_column: int = info.end_column + self.code: str = info.code + else: + self.line: int = UNDEFINED_POSITION + self.column: int = UNDEFINED_POSITION + self.end_line: int = UNDEFINED_POSITION + self.end_column: int = UNDEFINED_POSITION + self.code: str = "" def __hash__(self) -> int: - return hash(self.line) * hash(self.column) + return ( + hash(self.line) + * hash(self.column) + * hash(self.end_line) + * hash(self.end_column) + ) def __eq__(self, o: object) -> bool: if not isinstance(o, CodeElement): @@ -25,18 +74,377 @@ def as_dict(self) -> Dict[str, Any]: "ir_type": self.__class__.__name__, "line": self.line, "column": self.column, + "end_line": self.end_line, + "end_column": self.end_column, "code": self.code, } +# TODO: as dict for expr and values +@dataclass +class Expr(CodeElement, ABC): + def __init__(self, info: ElementInfo) -> None: + super().__init__(info) + + def __hash__(self) -> int: + return ( + hash(self.line) + * hash(self.column) + * hash(self.end_line) + * hash(self.end_column) + ) + + def __eq__(self, o: object) -> bool: + if not isinstance(o, CodeElement): + return False + return self.line == o.line and self.column == o.column + + +@dataclass +class Value(Expr, ABC): + def __init__(self, info: ElementInfo, value: Any) -> None: + super().__init__(info) + self.value = value + + def __eq__(self, o: object) -> bool: + if not isinstance(o, CodeElement): + return False + return ( + self.line == o.line + and self.column == o.column + and self.end_line == o.end_line + and self.end_column == o.end_column + ) + + def __hash__(self) -> int: + return ( + hash(self.line) + * hash(self.column) + * hash(self.end_line) + * hash(self.end_column) + ) + + def as_dict(self) -> Dict[str, Any]: + return { + **super().as_dict(), + "value": ( + self.value + if not isinstance(self.value, CodeElement) + else self.value.as_dict() + ), + } + + +class String(Value): + def __init__(self, value: str, info: ElementInfo) -> None: + super().__init__(info, value) + + +class Integer(Value): + def __init__(self, value: int, info: ElementInfo) -> None: + super().__init__(info, value) + + +class Complex(Value): + def __init__(self, value: complex, info: ElementInfo) -> None: + super().__init__(info, value) + + +class Float(Value): + def __init__(self, value: float, info: ElementInfo) -> None: + super().__init__(info, value) + + +class Boolean(Value): + def __init__(self, value: bool, info: ElementInfo) -> None: + super().__init__(info, value) + + +class Null(Value): + def __init__(self, info: ElementInfo | None = None) -> None: + if info is None: + # Let's hope there are no files with 2**32 lines lol + info = ElementInfo(2**32, 2**32, 2**32, 2**32, "") + super().__init__(info, None) + + +class Undef(Value): + def __init__(self, info: ElementInfo | None = None) -> None: + if info is None: + # Let's hope there are no files with 2**32 lines lol + info = ElementInfo(2**32, 2**32, 2**32, 2**32, "") + super().__init__(info, None) + + +class Hash(Value): + def __init__(self, value: Dict[Expr, Expr], info: ElementInfo) -> None: + super().__init__(info, value) + + def as_dict(self) -> Dict[str, Any]: + return { + **super().as_dict(), + "value": [ + {"key": k.as_dict(), "value": v.as_dict()} + for k, v in self.value.items() + ], + } + + +class Array(Value): + def __init__(self, value: List[Expr], info: ElementInfo) -> None: + super().__init__(info, value) + + def as_dict(self) -> Dict[str, Any]: + return { + **super().as_dict(), + "value": [v.as_dict() for v in self.value], + } + + +class VariableReference(Value): + def __init__(self, value: str, info: ElementInfo) -> None: + super().__init__(info, value) + + +class FunctionCall(Expr): + def __init__(self, name: str, args: List[Expr], info: ElementInfo) -> None: + super().__init__(info) + self.name: str = name + self.args: List[Expr] = args + + def as_dict(self) -> Dict[str, Any]: + return { + **super().as_dict(), + "name": self.name, + "args": [a.as_dict() for a in self.args], + } + + +class MethodCall(Expr): + def __init__( + self, receiver: Expr, method: str, args: List[Expr], info: ElementInfo + ) -> None: + super().__init__(info) + self.receiver: Expr = receiver + self.method: str = method + self.args: List[Expr] = args + + def as_dict(self) -> Dict[str, Any]: + return { + **super().as_dict(), + "receiver": self.receiver.as_dict(), + "method": self.method, + "args": [a.as_dict() for a in self.args], + } + + +class BlockExpr(Expr): + def __init__(self, info: ElementInfo) -> None: + super().__init__(info) + self.statements: List[CodeElement] = [] + + def add_statement(self, statement: CodeElement) -> None: + self.statements.append(statement) + + def as_dict(self) -> Dict[str, Any]: + return { + **super().as_dict(), + "statements": [s.as_dict() for s in self.statements], + } + + +# This is only used in Chef, and should be removed soon +class AddArgs(Value): + def __init__(self, value: List[Expr], info: ElementInfo) -> None: + super().__init__(info, value) + + def as_dict(self) -> Dict[str, Any]: + return { + **super().as_dict(), + "value": [v.as_dict() for v in self.value], + } + + +class UnaryOperation(Expr, ABC): + def __init__(self, info: ElementInfo, expr: Expr) -> None: + super().__init__(info) + self.expr = expr + + def as_dict(self) -> Dict[str, Any]: + return { + **super().as_dict(), + "expr": self.expr.as_dict(), + } + + +class Not(UnaryOperation): + def __init__(self, info: ElementInfo, expr: Expr) -> None: + super().__init__(info, expr) + + +class Minus(UnaryOperation): + def __init__(self, info: ElementInfo, expr: Expr) -> None: + super().__init__(info, expr) + + +class BinaryOperation(Expr, ABC): + def __init__(self, info: ElementInfo, left: Expr, right: Expr) -> None: + super().__init__(info) + self.left = left + self.right = right + + def __eq__(self, o: object) -> bool: + if not isinstance(o, CodeElement): + return False + return ( + self.line == o.line + and self.column == o.column + and self.end_line == o.end_line + and self.end_column == o.end_column + ) + + def __hash__(self) -> int: + return ( + hash(self.line) + * hash(self.column) + * hash(self.end_line) + * hash(self.end_column) + ) + + def as_dict(self) -> Dict[str, Any]: + return { + **super().as_dict(), + "left": self.left.as_dict(), + "right": self.right.as_dict(), + "type": self.__class__.__name__.lower(), + } + + +class Or(BinaryOperation): + def __init__(self, info: ElementInfo, left: Expr, right: Expr) -> None: + super().__init__(info, left, right) + + +class And(BinaryOperation): + def __init__(self, info: ElementInfo, left: Expr, right: Expr) -> None: + super().__init__(info, left, right) + + +class Sum(BinaryOperation): + def __init__(self, info: ElementInfo, left: Expr, right: Expr) -> None: + super().__init__(info, left, right) + + +class Equal(BinaryOperation): + def __init__(self, info: ElementInfo, left: Expr, right: Expr) -> None: + super().__init__(info, left, right) + + +class NotEqual(BinaryOperation): + def __init__(self, info: ElementInfo, left: Expr, right: Expr) -> None: + super().__init__(info, left, right) + + +class LessThan(BinaryOperation): + def __init__(self, info: ElementInfo, left: Expr, right: Expr) -> None: + super().__init__(info, left, right) + + +class LessThanOrEqual(BinaryOperation): + def __init__(self, info: ElementInfo, left: Expr, right: Expr) -> None: + super().__init__(info, left, right) + + +class GreaterThan(BinaryOperation): + def __init__(self, info: ElementInfo, left: Expr, right: Expr) -> None: + super().__init__(info, left, right) + + +class GreaterThanOrEqual(BinaryOperation): + def __init__(self, info: ElementInfo, left: Expr, right: Expr) -> None: + super().__init__(info, left, right) + + +class In(BinaryOperation): + def __init__(self, info: ElementInfo, left: Expr, right: Expr) -> None: + super().__init__(info, left, right) + + +class Subtract(BinaryOperation): + def __init__(self, info: ElementInfo, left: Expr, right: Expr) -> None: + super().__init__(info, left, right) + + +class Multiply(BinaryOperation): + def __init__(self, info: ElementInfo, left: Expr, right: Expr) -> None: + super().__init__(info, left, right) + + +class Divide(BinaryOperation): + def __init__(self, info: ElementInfo, left: Expr, right: Expr) -> None: + super().__init__(info, left, right) + + +class Modulo(BinaryOperation): + def __init__(self, info: ElementInfo, left: Expr, right: Expr) -> None: + super().__init__(info, left, right) + + +class Power(BinaryOperation): + def __init__(self, info: ElementInfo, left: Expr, right: Expr) -> None: + super().__init__(info, left, right) + + +class RightShift(BinaryOperation): + def __init__(self, info: ElementInfo, left: Expr, right: Expr) -> None: + super().__init__(info, left, right) + + +class LeftShift(BinaryOperation): + def __init__(self, info: ElementInfo, left: Expr, right: Expr) -> None: + super().__init__(info, left, right) + + +class Access(BinaryOperation): + def __init__(self, info: ElementInfo, left: Expr, right: Expr) -> None: + super().__init__(info, left, right) + + +class BitwiseAnd(BinaryOperation): + def __init__(self, info: ElementInfo, left: Expr, right: Expr) -> None: + super().__init__(info, left, right) + + +class BitwiseOr(BinaryOperation): + def __init__(self, info: ElementInfo, left: Expr, right: Expr) -> None: + super().__init__(info, left, right) + + +class BitwiseXor(BinaryOperation): + def __init__(self, info: ElementInfo, left: Expr, right: Expr) -> None: + super().__init__(info, left, right) + + +class Assign(BinaryOperation): + def __init__(self, info: ElementInfo, left: Expr, right: Expr) -> None: + super().__init__(info, left, right) + + class Block(CodeElement): def __init__(self) -> None: - super().__init__() + CodeElement.__init__(self) self.statements: List[CodeElement] = [] - def add_statement(self, statement: "ConditionalStatement") -> None: + def add_statement(self, statement: CodeElement) -> None: self.statements.append(statement) + def set_element_info(self, info: ElementInfo) -> None: + self.line = info.line + self.column = info.column + self.end_line = info.end_line + self.end_column = info.end_column + self.code = info.code + @staticmethod def __as_dict_statement( stat: Dict[str, Any] | List[Any] | CodeElement | str, @@ -61,21 +469,26 @@ def as_dict(self) -> Dict[str, Any]: } -class ConditionalStatement(Block): +class ConditionalStatement(Block, Expr): class ConditionType(Enum): IF = 1 SWITCH = 2 def __init__( self, - condition: str, + condition: Expr, type: "ConditionalStatement.ConditionType", is_default: bool = False, + is_top: bool = False, + info: ElementInfo | None = None, ) -> None: - super().__init__() - self.condition: str = condition + Block.__init__(self) + if info is not None: + Expr.__init__(self, info) + self.condition: Expr = condition self.else_statement: ConditionalStatement | None = None self.is_default = is_default + self.is_top = is_top self.type = type def __repr__(self) -> str: @@ -84,9 +497,10 @@ def __repr__(self) -> str: def as_dict(self) -> Dict[str, Any]: return { **super().as_dict(), - "condition": self.condition, + "condition": self.condition.as_dict(), "type": self.type.name, "is_default": self.is_default, + "is_top": self.is_top, "else_statement": ( self.else_statement.as_dict() if self.else_statement else None ), @@ -94,8 +508,8 @@ def as_dict(self) -> Dict[str, Any]: class Comment(CodeElement): - def __init__(self, content: str) -> None: - super().__init__() + def __init__(self, content: str, info: ElementInfo | None = None) -> None: + super().__init__(info) self.content: str = content def __repr__(self) -> str: @@ -109,75 +523,73 @@ def as_dict(self) -> Dict[str, Any]: class KeyValue(CodeElement): - def __init__(self, name: str, value: str | None, has_variable: bool) -> None: + def __init__(self, name: str, value: Expr, info: ElementInfo) -> None: + super().__init__(info) self.name: str = name - self.value: str | None = value - self.has_variable: bool = has_variable - self.keyvalues: List[KeyValue] = [] + self.value: Expr = value def __repr__(self) -> str: - value = repr(self.value).split("\n")[0] - if value == "None": - return f"{self.name}:{value}:{self.keyvalues}" - else: - return f"{self.name}:{value}" + return self.code def as_dict(self) -> Dict[str, Any]: return { **super().as_dict(), "name": self.name, - # FIXME: In Puppet code, the value can be a ConditionalStatement or a dict. - # The types need to be fixed. - "value": self.value if not isinstance(self.value, CodeElement) else self.value.as_dict(), # type: ignore - "has_variable": self.has_variable, - "keyvalues": [kv.as_dict() for kv in self.keyvalues], + "value": self.value.as_dict(), } class Variable(KeyValue): - def __init__(self, name: str, value: str | None, has_variable: bool) -> None: - super().__init__(name, value, has_variable) + def __init__(self, name: str, value: Expr, info: ElementInfo) -> None: + super().__init__(name, value, info) class Attribute(KeyValue): - def __init__(self, name: str, value: str | None, has_variable: bool) -> None: - super().__init__(name, value, has_variable) + def __init__(self, name: str, value: Expr, info: ElementInfo) -> None: + super().__init__(name, value, info) class AtomicUnit(Block): - def __init__(self, name: str | None, type: str) -> None: + def __init__(self, name: Expr, type: str) -> None: super().__init__() - self.name: str | None = name + self.name: Expr = name self.type: str = type - self.attributes: list[Attribute] = [] + self.attributes: List[Attribute] = [] def add_attribute(self, a: Attribute) -> None: self.attributes.append(a) def __repr__(self) -> str: - return f"{self.name} {self.type}" + if isinstance(self.name, String): + name_str = self.name.value + elif hasattr(self.name, "code"): + name_str = self.name.code + else: + name_str = str(self.name) + + return f"{name_str} {self.type}" def as_dict(self) -> Dict[str, Any]: return { **super().as_dict(), - "name": self.name, + "name": self.name.as_dict(), "type": self.type, "attributes": [a.as_dict() for a in self.attributes], } class Dependency(CodeElement): - def __init__(self, name: str) -> None: + def __init__(self, names: List[str]) -> None: super().__init__() - self.name: str = name + self.names: List[str] = names def __repr__(self) -> str: - return self.name + return ",".join(self.names) def as_dict(self) -> Dict[str, Any]: return { **super().as_dict(), - "name": self.name, + "names": self.names, } @@ -186,6 +598,8 @@ class UnitBlockType(str, Enum): tasks = "tasks" vars = "vars" block = "block" + function = "function" + definition = "definition" unknown = "unknown" @@ -224,7 +638,7 @@ def add_attribute(self, a: Attribute) -> None: self.attributes.append(a) def as_dict(self) -> Dict[str, Any]: - return { + result = { **super().as_dict(), "dependencies": [d.as_dict() for d in self.dependencies], "comments": [c.as_dict() for c in self.comments], @@ -237,6 +651,18 @@ def as_dict(self) -> Dict[str, Any]: "type": self.type, } + lines = -1 + if self.path != "": + with open(self.path, "r") as f: + try: + lines = sum(1 for _ in f) + except Exception: + pass + if lines != -1: + result["lines"] = lines + + return result + class File: def __init__(self, name: str) -> None: diff --git a/glitch/tech.py b/glitch/tech.py index ad321319..6a3a5aac 100644 --- a/glitch/tech.py +++ b/glitch/tech.py @@ -11,5 +11,4 @@ def __init__(self, tech: str, extensions: List[str]): chef = "chef", ["rb"] puppet = "puppet", ["pp"] terraform = "terraform", ["tf"] - docker = "docker", ["Dockerfile"] gha = "github-actions", ["yml", "yaml"] diff --git a/glitch/tests/design/docker/files/avoid_comments.Dockerfile b/glitch/tests/design/docker/files/avoid_comments.Dockerfile deleted file mode 100644 index 670e5ec5..00000000 --- a/glitch/tests/design/docker/files/avoid_comments.Dockerfile +++ /dev/null @@ -1,5 +0,0 @@ -# A ubuntu container -FROM ubuntu:20.04 -USER ubuntu - -CMD ['echo', "Hello"] \ No newline at end of file diff --git a/glitch/tests/design/docker/files/duplicate_block.Dockerfile b/glitch/tests/design/docker/files/duplicate_block.Dockerfile deleted file mode 100644 index a6df387c..00000000 --- a/glitch/tests/design/docker/files/duplicate_block.Dockerfile +++ /dev/null @@ -1,15 +0,0 @@ -FROM ubuntu:20.04 -USER ubuntu - -RUN wget https://repo.continuum.io/archive/Anaconda3-5.0.1-Linux-x86_64.sh && \ - gpg Anaconda3-5.0.1-Linux-x86_64.sh && \ - sh Anaconda3-5.0.1-Linux-x86_64.sh -CMD ['echo', "Hello"] - -FROM ubuntu:20.04 -USER ubuntu - -RUN wget https://repo.continuum.io/archive/Anaconda3-5.0.1-Linux-x86_64.sh && \ - gpg Anaconda3-5.0.1-Linux-x86_64.sh && \ - sh Anaconda3-5.0.1-Linux-x86_64.sh -CMD ['echo', "Hello"] diff --git a/glitch/tests/design/docker/files/improper_alignment.Dockerfile b/glitch/tests/design/docker/files/improper_alignment.Dockerfile deleted file mode 100644 index 184513e8..00000000 --- a/glitch/tests/design/docker/files/improper_alignment.Dockerfile +++ /dev/null @@ -1,5 +0,0 @@ -FROM ubuntu:20.04 -USER ubuntu - -RUN foo bar --a b \ - --c d diff --git a/glitch/tests/design/docker/files/long_statement.Dockerfile b/glitch/tests/design/docker/files/long_statement.Dockerfile deleted file mode 100644 index dbf898e5..00000000 --- a/glitch/tests/design/docker/files/long_statement.Dockerfile +++ /dev/null @@ -1,6 +0,0 @@ -FROM ubuntu:20.04 -USER ubuntu - -RUN cURL -X POST -d '{"name": "cURL", "type": "article"}' -H 'Accept-Encoding: application/json' -H 'Authorization: Bearer token' https://www.curl_blog.com/posts - -CMD ['echo', "Hello"] \ No newline at end of file diff --git a/glitch/tests/design/docker/files/too_many_variables.Dockerfile b/glitch/tests/design/docker/files/too_many_variables.Dockerfile deleted file mode 100644 index 5913b51f..00000000 --- a/glitch/tests/design/docker/files/too_many_variables.Dockerfile +++ /dev/null @@ -1,10 +0,0 @@ -FROM ubuntu:20.04 -USER ubuntu - -ARG aqua_admin_password -ARG aqua_sso_client_secret -ARG aqua_sso_client -ARG aqua_scanner_username -ARG aqua_scanner_password -ARG aqua_operator_username -ARG aqua_operator_passowrd \ No newline at end of file diff --git a/glitch/tests/design/docker/test_design.py b/glitch/tests/design/docker/test_design.py deleted file mode 100644 index c8a32794..00000000 --- a/glitch/tests/design/docker/test_design.py +++ /dev/null @@ -1,76 +0,0 @@ -import unittest - -from glitch.analysis.design.visitor import DesignVisitor -from glitch.parsers.docker import DockerParser -from glitch.tech import Tech - - -class TestDesign(unittest.TestCase): - def __help_test(self, path, n_errors: int, codes, lines) -> None: - parser = DockerParser() - inter = parser.parse(path, "script", False) - analysis = DesignVisitor(Tech.docker) - analysis.config("configs/default.ini") - errors = list( - filter( - lambda e: e.code.startswith("design_") - or e.code.startswith("implementation_"), - set(analysis.check(inter)), - ) - ) - errors = sorted(errors, key=lambda e: (e.path, e.line, e.code)) - self.assertEqual(len(errors), n_errors) - for i in range(n_errors): - self.assertEqual(errors[i].code, codes[i]) - self.assertEqual(errors[i].line, lines[i]) - - def test_docker_long_statement(self) -> None: - self.__help_test( - "tests/design/docker/files/long_statement.Dockerfile", - 1, - ["implementation_long_statement"], - [4], - ) - - def test_docker_improper_alignment(self) -> None: - # TODO: Fix smell, due to docker parsing method the attributes are not - # detected in differents lines, making it impossible to trigger alignment - pass - # self.__help_test( - # "tests/design/docker/files/improper_alignment.Dockerfile", - # 1, - # [ - # "implementation_improper_alignment" - # ], [1] - # ) - - def test_docker_duplicate_block(self) -> None: - self.__help_test( - "tests/design/docker/files/duplicate_block.Dockerfile", - 2, - [ - "design_duplicate_block", - "design_duplicate_block", - ], - [1, 9], - ) - - def test_docker_avoid_comments(self) -> None: - self.__help_test( - "tests/design/docker/files/avoid_comments.Dockerfile", - 1, - [ - "design_avoid_comments", - ], - [1], - ) - - def test_docker_too_many_variables(self) -> None: - self.__help_test( - "tests/design/docker/files/too_many_variables.Dockerfile", - 1, - [ - "implementation_too_many_variables", - ], - [1], - ) diff --git a/glitch/tests/design/gha/test_design.py b/glitch/tests/design/gha/test_design.py deleted file mode 100644 index 96549586..00000000 --- a/glitch/tests/design/gha/test_design.py +++ /dev/null @@ -1,45 +0,0 @@ -import unittest - -from glitch.analysis.design.visitor import DesignVisitor -from glitch.analysis.rules import Error -from glitch.parsers.gha import GithubActionsParser -from glitch.tech import Tech -from glitch.repr.inter import UnitBlockType -from typing import List - - -class TestDesign(unittest.TestCase): - def __help_test( - self, path: str, n_errors: int, codes: List[str], lines: List[int] - ) -> None: - parser = GithubActionsParser() - inter = parser.parse(path, UnitBlockType.script, False) - assert inter is not None - analysis = DesignVisitor(Tech.gha) - analysis.config("configs/default.ini") - errors: List[Error] = list(set(analysis.check(inter))) - errors = list( - filter( - lambda e: e.code.startswith("design_") - or e.code.startswith("implementation_"), - errors, - ) - ) - errors = sorted(errors, key=lambda e: (e.path, e.line, e.code)) - self.assertEqual(len(errors), n_errors) - for i in range(n_errors): - self.assertEqual(errors[i].path, path) - self.assertEqual(errors[i].code, codes[i]) - self.assertEqual(errors[i].line, lines[i]) - - # NOTE: This test also verifies if the paths of errors in inner Unit Blocks - # are correctly reported. - def test_gha_too_many_variables(self) -> None: - self.__help_test( - "tests/design/gha/files/too_many_variables.yml", - 1, - [ - "implementation_too_many_variables", - ], - [10], - ) diff --git a/glitch/tests/hierarchical/test_parsers.py b/glitch/tests/hierarchical/test_parsers.py deleted file mode 100644 index f9c0bd1b..00000000 --- a/glitch/tests/hierarchical/test_parsers.py +++ /dev/null @@ -1,55 +0,0 @@ -import unittest -from glitch.parsers.ansible import AnsibleParser -from glitch.parsers.chef import ChefParser -from glitch.parsers.puppet import PuppetParser - - -class TestAnsible(unittest.TestCase): - def __test_parse_vars(self, path, vars) -> None: - with open(path, "r") as file: - unitblock = AnsibleParser._AnsibleParser__parse_vars_file( - self, "test", file - ) - self.assertEqual(str(unitblock.variables), vars) - file.close() - - def __test_parse_attributes(self, path, attributes) -> None: - with open(path, "r") as file: - unitblock = AnsibleParser._AnsibleParser__parse_playbook(self, "test", file) - play = unitblock.unit_blocks[0] - self.assertEqual(str(play.attributes), attributes) - file.close() - - def test_hierarchichal_vars(self) -> None: - vars = "[test[0]:None:[test1[0]:\"['1', '2']\"], test[1]:\"['3', '4']\", test:\"['x', 'y', '23']\", test2[0]:\"['2', '5', '6']\", vars:None:[factorial_of:'5', factorial_value:'1']]" - self.__test_parse_vars("tests/hierarchical/ansible/vars.yml", vars) - - def test_hierarchical_attributes(self) -> None: - attributes = "[hosts:'localhost', debug:None:[msg:'The factorial of 5 is {{ factorial_value }}', seq[0]:None:[test:'something'], seq:\"['y', 'z']\", hash:None:[test1:'1', test2:'2']]]" - self.__test_parse_attributes( - "tests/hierarchical/ansible/attributes.yml", attributes - ) - - -class TestPuppet(unittest.TestCase): - def __test_parse_vars(self, path, vars) -> None: - unitblock = PuppetParser().parse_file(path, None) - self.assertEqual(str(unitblock.variables), vars) - - def test_hierarchical_vars(self) -> None: - vars = "[$my_hash:None:[key1:None:[test1:'1', test2:'2'], key2:'value2', key3:'value3', key4:None:[key5:'value5']], $configdir:'${boxen::config::configdir}/php', $datadir:'${boxen::config::datadir}/php', $pluginsdir:'${root}/plugins', $cachedir:'${php::config::datadir}/cache', $extensioncachedir:'${php::config::datadir}/cache/extensions']" - self.__test_parse_vars("tests/hierarchical/puppet/vars.pp", vars) - - -class TestChef(unittest.TestCase): - def __test_parse_vars(self, path, vars) -> None: - unitblock = ChefParser().parse_file(path, None) - self.assertEqual(str(unitblock.variables), vars) - - def test_hierarchical_vars(self) -> None: - vars = "[grades:None:[Jane Doe:'10', Jim Doe:'6'], default:None:[zabbix:None:[database:None:[password:''], test:None:[name:'something']]]]" - self.__test_parse_vars("tests/hierarchical/chef/vars.rb", vars) - - -if __name__ == "__main__": - unittest.main() diff --git a/glitch/tests/parser/gha/test_parser.py b/glitch/tests/parser/gha/test_parser.py deleted file mode 100644 index 31e50e1a..00000000 --- a/glitch/tests/parser/gha/test_parser.py +++ /dev/null @@ -1,154 +0,0 @@ -import unittest - -from glitch.parsers.gha import GithubActionsParser -from glitch.repr.inter import * - - -class TestGithubActionsParser(unittest.TestCase): - def test_gha_valid_workflow(self) -> None: - """ - run commands - with - runs-on - """ - p = GithubActionsParser() - ir = p.parse_file( - "tests/parser/gha/files/valid_workflow.yml", UnitBlockType.script - ) - - assert ir is not None - assert isinstance(ir, UnitBlock) - assert ir.type == UnitBlockType.script - assert ir.name == "Run Python Tests" - - assert len(ir.attributes) == 1 - assert ir.attributes[0].name == "on" - assert ir.attributes[0].value is None - assert len(ir.attributes[0].keyvalues) == 2 - - assert ir.attributes[0].keyvalues[0].name == "push" - assert ir.attributes[0].keyvalues[0].value is None - assert len(ir.attributes[0].keyvalues[0].keyvalues) == 1 - assert ir.attributes[0].keyvalues[0].keyvalues[0].name == "branches" - assert ir.attributes[0].keyvalues[0].keyvalues[0].value == ["main"] - assert ( - ir.attributes[0].keyvalues[0].keyvalues[0].code - == " branches:\n - main\n " - ) - - assert ir.attributes[0].keyvalues[1].name == "pull_request" - assert ir.attributes[0].keyvalues[1].value is None - assert len(ir.attributes[0].keyvalues[1].keyvalues) == 1 - assert ir.attributes[0].keyvalues[1].keyvalues[0].name == "branches" - assert ir.attributes[0].keyvalues[1].keyvalues[0].value == ["main"] - - assert len(ir.unit_blocks) == 1 - assert ir.unit_blocks[0].type == UnitBlockType.block - assert ir.unit_blocks[0].name == "build" - - assert len(ir.unit_blocks[0].attributes) == 1 - assert ir.unit_blocks[0].attributes[0].name == "runs-on" - assert ir.unit_blocks[0].attributes[0].value == "ubuntu-latest" - - assert len(ir.unit_blocks[0].atomic_units) == 5 - - assert ir.unit_blocks[0].atomic_units[0].name == "" - assert ir.unit_blocks[0].atomic_units[0].type == "actions/checkout@v3" - - assert ir.unit_blocks[0].atomic_units[1].name == "" - assert ir.unit_blocks[0].atomic_units[1].type == "ruby/setup-ruby@v1" - assert len(ir.unit_blocks[0].atomic_units[1].attributes) == 1 - assert ir.unit_blocks[0].atomic_units[1].attributes[0].name == "ruby-version" - assert ir.unit_blocks[0].atomic_units[1].attributes[0].value == "2.7.4" - assert not ir.unit_blocks[0].atomic_units[1].attributes[0].has_variable - - assert ir.unit_blocks[0].atomic_units[2].name == "Install Python 3" - assert ir.unit_blocks[0].atomic_units[2].type == "actions/setup-python@v4" - assert len(ir.unit_blocks[0].atomic_units[2].attributes) == 1 - assert ir.unit_blocks[0].atomic_units[2].attributes[0].name == "python-version" - assert ir.unit_blocks[0].atomic_units[2].attributes[0].value == "3.10.5" - assert not ir.unit_blocks[0].atomic_units[2].attributes[0].has_variable - - assert ir.unit_blocks[0].atomic_units[3].name == "Install dependencies" - assert ir.unit_blocks[0].atomic_units[3].type == "shell" - assert len(ir.unit_blocks[0].atomic_units[3].attributes) == 1 - assert ir.unit_blocks[0].atomic_units[3].attributes[0].name == "run" - assert ( - ir.unit_blocks[0].atomic_units[3].attributes[0].value - == "python -m pip install --upgrade pip\npython -m pip install -e .\n" - ) - assert not ir.unit_blocks[0].atomic_units[3].attributes[0].has_variable - - assert ir.unit_blocks[0].atomic_units[4].name == "Run tests with pytest" - assert ir.unit_blocks[0].atomic_units[4].type == "shell" - assert len(ir.unit_blocks[0].atomic_units[4].attributes) == 1 - assert ir.unit_blocks[0].atomic_units[4].attributes[0].name == "run" - assert ( - ir.unit_blocks[0].atomic_units[4].attributes[0].value - == "cd glitch\npython -m unittest discover tests" - ) - assert not ir.unit_blocks[0].atomic_units[4].attributes[0].has_variable - - def test_gha_valid_workflow_2(self) -> None: - """ - comments - env (global) - has_variable - defaults (job) - """ - p = GithubActionsParser() - ir = p.parse_file( - "tests/parser/gha/files/valid_workflow_2.yml", UnitBlockType.script - ) - assert ir is not None - assert isinstance(ir, UnitBlock) - assert ir.type == UnitBlockType.script - - assert len(ir.variables) == 1 - assert isinstance(ir.variables[0], Variable) - assert ir.variables[0].name == "build" - assert ir.variables[0].value == "${{ github.workspace }}/build" - assert ir.variables[0].has_variable - - assert len(ir.unit_blocks) == 1 - - assert len(ir.unit_blocks[0].variables) == 1 - assert isinstance(ir.unit_blocks[0].variables[0], Variable) - assert ir.unit_blocks[0].variables[0].name == "run" - assert ir.unit_blocks[0].variables[0].value is None - assert not ir.unit_blocks[0].variables[0].has_variable - assert len(ir.unit_blocks[0].variables[0].keyvalues) == 1 - assert ir.unit_blocks[0].variables[0].keyvalues[0].name == "shell" - assert ir.unit_blocks[0].variables[0].keyvalues[0].value == "powershell" - - assert len(ir.unit_blocks[0].atomic_units) == 4 - assert ir.unit_blocks[0].atomic_units[1].name == "Configure CMake" - assert ir.unit_blocks[0].atomic_units[1].type == "shell" - assert len(ir.unit_blocks[0].atomic_units[1].attributes) == 1 - assert ir.unit_blocks[0].atomic_units[1].attributes[0].name == "run" - assert ( - ir.unit_blocks[0].atomic_units[1].attributes[0].value - == "cmake -B ${{ env.build }}" - ) - assert ir.unit_blocks[0].atomic_units[1].attributes[0].has_variable - - assert len(ir.comments) == 24 - - assert ( - ir.comments[0].content - == "# https://github.com/actions/starter-workflows/blob/main/code-scanning/msvc.yml" - ) - assert ir.comments[0].line == 1 - - assert ir.comments[9].content == "# for actions/checkout to fetch code" - assert ir.comments[9].line == 31 - - def test_gha_index_out_of_range(self) -> None: - """ - This file previously gave an index out of range even though it is valid. - """ - p = GithubActionsParser() - ir = p.parse_file( - "tests/parser/gha/files/index_out_of_range.yml", UnitBlockType.script - ) - assert ir is not None diff --git a/glitch/tests/parser/puppet/test_parser.py b/glitch/tests/parser/puppet/test_parser.py deleted file mode 100644 index 626484ff..00000000 --- a/glitch/tests/parser/puppet/test_parser.py +++ /dev/null @@ -1,21 +0,0 @@ -import unittest -from glitch.parsers.puppet import PuppetParser -from glitch.repr.inter import * - - -class TestPuppetParser(unittest.TestCase): - def test_puppet_parser_if(self) -> None: - unit_block = PuppetParser().parse_file("tests/parser/puppet/files/if.pp", None) - assert len(unit_block.statements) == 1 - assert isinstance(unit_block.statements[0], ConditionalStatement) - # FIXME: the expression should not be a string and or at least should be - # equal to the script - assert unit_block.statements[0].condition == "$x==absent" - assert len(unit_block.statements[0].statements) == 1 - assert isinstance(unit_block.statements[0].statements[0], AtomicUnit) - assert unit_block.statements[0].else_statement is not None - assert isinstance(unit_block.statements[0].else_statement, ConditionalStatement) - assert len(unit_block.statements[0].else_statement.statements) == 1 - assert isinstance( - unit_block.statements[0].else_statement.statements[0], AtomicUnit - ) diff --git a/glitch/tests/parser/terraform/files/value_has_variable.tf b/glitch/tests/parser/terraform/files/value_has_variable.tf deleted file mode 100644 index 8caf184d..00000000 --- a/glitch/tests/parser/terraform/files/value_has_variable.tf +++ /dev/null @@ -1,11 +0,0 @@ -resource "google_bigquery_dataset" "dataset" { - access { - user_by_email = google_service_account.bqowner.email - } - test = "${var.value1}" -} - -resource "google_service_account" "bqowner" { - account_id = "bqowner" - email = "email.com" -} \ No newline at end of file diff --git a/glitch/tests/parser/terraform/test_parser.py b/glitch/tests/parser/terraform/test_parser.py deleted file mode 100644 index 97fd9f94..00000000 --- a/glitch/tests/parser/terraform/test_parser.py +++ /dev/null @@ -1,64 +0,0 @@ -import unittest -from glitch.parsers.terraform import TerraformParser -from typing import Sequence - - -class TestTerraform(unittest.TestCase): - def __help_test(self, path, attributes) -> None: - unitblock = TerraformParser().parse_file(path, None) - au = unitblock.atomic_units[0] - self.assertEqual(str(au.attributes), attributes) - - def __help_test_comments(self, path, comments: Sequence[str]) -> None: - unitblock = TerraformParser().parse_file(path, None) - self.assertEqual(str(unitblock.comments), comments) - - def test_terraform_null_value(self) -> None: - attributes = "[account_id:'']" - self.__help_test( - "tests/parser/terraform/files/null_value_assign.tf", attributes - ) - - def test_terraform_empty_string(self) -> None: - attributes = "[account_id:'']" - self.__help_test( - "tests/parser/terraform/files/empty_string_assign.tf", attributes - ) - - def test_terraform_boolean_value(self) -> None: - attributes = "[account_id:'True']" - self.__help_test( - "tests/parser/terraform/files/boolean_value_assign.tf", attributes - ) - - def test_terraform_multiline_string(self) -> None: - attributes = "[user_data:' #!/bin/bash\\n sudo apt-get update\\n sudo apt-get install -y apache2\\n sudo systemctl start apache2']" - self.__help_test( - "tests/parser/terraform/files/multiline_string_assign.tf", attributes - ) - - def test_terraform_value_has_variable(self) -> None: - attributes = "[access:None:[user_by_email:'${google_service_account.bqowner.email}'], test:'${var.value1}']" - self.__help_test( - "tests/parser/terraform/files/value_has_variable.tf", attributes - ) - - def test_terraform_dict_value(self) -> None: - attributes = "[labels:None:[env:'default']]" - self.__help_test( - "tests/parser/terraform/files/dict_value_assign.tf", attributes - ) - - def test_terraform_list_value(self) -> None: - attributes = "[keys[0]:'value1', keys[1][0]:'1', keys[1][1]:None:[key2:'value2'], keys[2]:None:[key3:'value3']]" - self.__help_test( - "tests/parser/terraform/files/list_value_assign.tf", attributes - ) - - def test_terraform_dynamic_block(self) -> None: - attributes = "[dynamic.setting:None:[content:None:[namespace:'${setting.value[\"namespace\"]}']]]" - self.__help_test("tests/parser/terraform/files/dynamic_block.tf", attributes) - - def test_terraform_comments(self) -> None: - comments = "[#comment1\n, //comment2\n, /*comment3\n default_table_expiration_ms = 3600000\n \n finish comment3 */, #comment4\n, #comment5\n, #comment inside dict\n, //comment2 inside dict\n]" - self.__help_test_comments("tests/parser/terraform/files/comments.tf", comments) diff --git a/glitch/tests/repair/interactive/delta_p/delta_p_puppet_scripts.py b/glitch/tests/repair/interactive/delta_p/delta_p_puppet_scripts.py deleted file mode 100644 index 01fa84ff..00000000 --- a/glitch/tests/repair/interactive/delta_p/delta_p_puppet_scripts.py +++ /dev/null @@ -1,426 +0,0 @@ -from glitch.repair.interactive.delta_p import * - -delta_p_puppet = PSeq( - lhs=PSkip(), - rhs=PSeq( - lhs=PSeq( - lhs=PSeq( - lhs=PLet( - id="state-2", - expr=PEConst(const=PStr(value="present")), - label=2, - body=PIf( - pred=PEBinOP( - op=PEq(), - lhs=PEVar(id="state-2"), - rhs=PEConst(const=PStr(value="present")), - ), - cons=PCreate( - path=PEConst( - const=PStr( - value="/var/www/customers/public_html/index.php" - ) - ) - ), - alt=PIf( - pred=PEBinOP( - op=PEq(), - lhs=PEVar(id="state-2"), - rhs=PEConst(const=PStr(value="absent")), - ), - cons=PRm( - path=PEConst( - const=PStr( - value="/var/www/customers/public_html/index.php" - ) - ) - ), - alt=PIf( - pred=PEBinOP( - op=PEq(), - lhs=PEVar(id="state-2"), - rhs=PEConst(const=PStr(value="directory")), - ), - cons=PMkdir( - path=PEConst( - const=PStr( - value="/var/www/customers/public_html/index.php" - ) - ) - ), - alt=PSkip(), - ), - ), - ), - ), - rhs=PLet( - id="content-1", - expr=PEConst( - const=PStr( - value="

Hello World

" - ) - ), - label=1, - body=PWrite( - path=PEConst( - const=PStr(value="/var/www/customers/public_html/index.php") - ), - content=PEVar(id="content-1"), - ), - ), - ), - rhs=PLet( - id="owner-4", - expr=PEConst(const=PStr(value="web_admin")), - label=4, - body=PChown( - path=PEConst( - const=PStr(value="/var/www/customers/public_html/index.php") - ), - owner=PEVar(id="owner-4"), - ), - ), - ), - rhs=PLet( - id="mode-3", - expr=PEConst(const=PStr(value="0755")), - label=3, - body=PChmod( - path=PEConst( - const=PStr(value="/var/www/customers/public_html/index.php") - ), - mode=PEVar(id="mode-3"), - ), - ), - ), -) - - -delta_p_puppet_2 = PSeq( - lhs=PSkip(), - rhs=PSeq( - lhs=PSeq( - lhs=PSeq( - lhs=PLet( - id="state-0", - expr=PEConst(const=PStr(value="absent")), - label=0, - body=PIf( - pred=PEBinOP( - op=PEq(), - lhs=PEVar(id="state-0"), - rhs=PEConst(const=PStr(value="present")), - ), - cons=PCreate( - path=PEConst(const=PStr(value="/usr/sbin/policy-rc.d")) - ), - alt=PIf( - pred=PEBinOP( - op=PEq(), - lhs=PEVar(id="state-0"), - rhs=PEConst(const=PStr(value="absent")), - ), - cons=PRm( - path=PEConst(const=PStr(value="/usr/sbin/policy-rc.d")) - ), - alt=PIf( - pred=PEBinOP( - op=PEq(), - lhs=PEVar(id="state-0"), - rhs=PEConst(const=PStr(value="directory")), - ), - cons=PMkdir( - path=PEConst( - const=PStr(value="/usr/sbin/policy-rc.d") - ) - ), - alt=PSkip(), - ), - ), - ), - ), - rhs=PLet( - id="sketched-content-1", - expr=PEUndef(), - label=1, - body=PWrite( - path=PEConst(const=PStr(value="/usr/sbin/policy-rc.d")), - content=PEVar(id="sketched-content-1"), - ), - ), - ), - rhs=PLet( - id="sketched-owner-2", - expr=PEUndef(), - label=2, - body=PChown( - path=PEConst(const=PStr(value="/usr/sbin/policy-rc.d")), - owner=PEVar(id="sketched-owner-2"), - ), - ), - ), - rhs=PLet( - id="sketched-mode-3", - expr=PEUndef(), - label=3, - body=PChmod( - path=PEConst(const=PStr(value="/usr/sbin/policy-rc.d")), - mode=PEVar(id="sketched-mode-3"), - ), - ), - ), -) - - -delta_p_puppet_if = PSeq( - lhs=PSkip(), - rhs=PIf( - pred=PEVar(id="dejavu-condition-2"), - cons=PSeq( - lhs=PSkip(), - rhs=PSeq( - lhs=PSeq( - lhs=PSeq( - lhs=PLet( - id="state-0", - expr=PEConst(const=PStr(value="absent")), - label=0, - body=PIf( - pred=PEBinOP( - op=PEq(), - lhs=PEVar(id="state-0"), - rhs=PEConst(const=PStr(value="present")), - ), - cons=PCreate( - path=PEConst( - const=PStr(value="/usr/sbin/policy-rc.d") - ) - ), - alt=PIf( - pred=PEBinOP( - op=PEq(), - lhs=PEVar(id="state-0"), - rhs=PEConst(const=PStr(value="absent")), - ), - cons=PRm( - path=PEConst( - const=PStr(value="/usr/sbin/policy-rc.d") - ) - ), - alt=PIf( - pred=PEBinOP( - op=PEq(), - lhs=PEVar(id="state-0"), - rhs=PEConst(const=PStr(value="directory")), - ), - cons=PMkdir( - path=PEConst( - const=PStr( - value="/usr/sbin/policy-rc.d" - ) - ) - ), - alt=PSkip(), - ), - ), - ), - ), - rhs=PLet( - id="sketched-content-2", - expr=PEUndef(), - label=2, - body=PWrite( - path=PEConst(const=PStr(value="/usr/sbin/policy-rc.d")), - content=PEVar(id="sketched-content-2"), - ), - ), - ), - rhs=PLet( - id="sketched-owner-3", - expr=PEUndef(), - label=3, - body=PChown( - path=PEConst(const=PStr(value="/usr/sbin/policy-rc.d")), - owner=PEVar(id="sketched-owner-3"), - ), - ), - ), - rhs=PLet( - id="sketched-mode-4", - expr=PEUndef(), - label=4, - body=PChmod( - path=PEConst(const=PStr(value="/usr/sbin/policy-rc.d")), - mode=PEVar(id="sketched-mode-4"), - ), - ), - ), - ), - alt=PIf( - pred=PEVar(id="dejavu-condition-1"), - cons=PSeq( - lhs=PSkip(), - rhs=PSeq( - lhs=PSeq( - lhs=PSeq( - lhs=PLet( - id="state-1", - expr=PEConst(const=PStr(value="present")), - label=1, - body=PIf( - pred=PEBinOP( - op=PEq(), - lhs=PEVar(id="state-1"), - rhs=PEConst(const=PStr(value="present")), - ), - cons=PCreate( - path=PEConst( - const=PStr(value="/usr/sbin/policy-rc.d") - ) - ), - alt=PIf( - pred=PEBinOP( - op=PEq(), - lhs=PEVar(id="state-1"), - rhs=PEConst(const=PStr(value="absent")), - ), - cons=PRm( - path=PEConst( - const=PStr( - value="/usr/sbin/policy-rc.d" - ) - ) - ), - alt=PIf( - pred=PEBinOP( - op=PEq(), - lhs=PEVar(id="state-1"), - rhs=PEConst( - const=PStr(value="directory") - ), - ), - cons=PMkdir( - path=PEConst( - const=PStr( - value="/usr/sbin/policy-rc.d" - ) - ) - ), - alt=PSkip(), - ), - ), - ), - ), - rhs=PLet( - id="sketched-content-5", - expr=PEUndef(), - label=5, - body=PWrite( - path=PEConst( - const=PStr(value="/usr/sbin/policy-rc.d") - ), - content=PEVar(id="sketched-content-5"), - ), - ), - ), - rhs=PLet( - id="sketched-owner-6", - expr=PEUndef(), - label=6, - body=PChown( - path=PEConst(const=PStr(value="/usr/sbin/policy-rc.d")), - owner=PEVar(id="sketched-owner-6"), - ), - ), - ), - rhs=PLet( - id="sketched-mode-7", - expr=PEUndef(), - label=7, - body=PChmod( - path=PEConst(const=PStr(value="/usr/sbin/policy-rc.d")), - mode=PEVar(id="sketched-mode-7"), - ), - ), - ), - ), - alt=PSkip(), - ), - ), -) - - -delta_p_puppet_default_state = PSeq( - lhs=PSkip(), - rhs=PSeq( - lhs=PSeq( - lhs=PSeq( - lhs=PLet( - id="sketched-state-4", - expr=PEConst(const=PStr(value="present")), - label=4, - body=PIf( - pred=PEBinOP( - op=PEq(), - lhs=PEVar(id="sketched-state-4"), - rhs=PEConst(const=PStr(value="present")), - ), - cons=PCreate( - path=PEConst(const=PStr(value="/root/.ssh/config")) - ), - alt=PIf( - pred=PEBinOP( - op=PEq(), - lhs=PEVar(id="sketched-state-4"), - rhs=PEConst(const=PStr(value="absent")), - ), - cons=PRm( - path=PEConst(const=PStr(value="/root/.ssh/config")) - ), - alt=PIf( - pred=PEBinOP( - op=PEq(), - lhs=PEVar(id="sketched-state-4"), - rhs=PEConst(const=PStr(value="directory")), - ), - cons=PMkdir( - path=PEConst(const=PStr(value="/root/.ssh/config")) - ), - alt=PSkip(), - ), - ), - ), - ), - rhs=PLet( - id="content-0", - expr=PEConst( - const=PStr(value="template('fuel/root_ssh_config.erb')") - ), - label=0, - body=PWrite( - path=PEConst(const=PStr(value="/root/.ssh/config")), - content=PEVar(id="content-0"), - ), - ), - ), - rhs=PLet( - id="owner-1", - expr=PEConst(const=PStr(value="root")), - label=1, - body=PChown( - path=PEConst(const=PStr(value="/root/.ssh/config")), - owner=PEVar(id="owner-1"), - ), - ), - ), - rhs=PLet( - id="mode-3", - expr=PEConst(const=PStr(value="0600")), - label=3, - body=PChmod( - path=PEConst(const=PStr(value="/root/.ssh/config")), - mode=PEVar(id="mode-3"), - ), - ), - ), -) diff --git a/glitch/tests/repair/interactive/test_delta_p.py b/glitch/tests/repair/interactive/test_delta_p.py deleted file mode 100644 index 2a58bc73..00000000 --- a/glitch/tests/repair/interactive/test_delta_p.py +++ /dev/null @@ -1,141 +0,0 @@ -from glitch.parsers.puppet import PuppetParser -from glitch.repair.interactive.compiler.compiler import DeltaPCompiler -from glitch.repair.interactive.compiler.labeler import GLITCHLabeler -from glitch.repair.interactive.delta_p import * -from glitch.repr.inter import UnitBlockType -from glitch.tech import Tech -from glitch.tests.repair.interactive.delta_p.delta_p_puppet_scripts import * -from tempfile import NamedTemporaryFile - - -def test_delta_p_compiler_puppet() -> None: - puppet_script = """ - file { '/var/www/customers/public_html/index.php': - path => '/var/www/customers/public_html/index.php', - content => '

Hello World

', - ensure => present, - mode => '0755', - owner => 'web_admin' - } - """ - - with NamedTemporaryFile() as f: - f.write(puppet_script.encode()) - f.flush() - puppet_parser = PuppetParser().parse_file(f.name, UnitBlockType.script) - labeled_script = GLITCHLabeler.label(puppet_parser, Tech.puppet) - - # Check labels - i = 0 - for atomic_unit in labeled_script.script.atomic_units: - for attribute in atomic_unit.attributes: - assert labeled_script.get_label(attribute) == i - i += 1 - - statement = DeltaPCompiler.compile(labeled_script, Tech.puppet) - - assert statement == delta_p_puppet - - -def test_delta_p_compiler_puppet_2() -> None: - puppet_script = """ - file {'/usr/sbin/policy-rc.d': - ensure => absent, - } - """ - - with NamedTemporaryFile() as f: - f.write(puppet_script.encode()) - f.flush() - puppet_parser = PuppetParser().parse_file(f.name, UnitBlockType.script) - labeled_script = GLITCHLabeler.label(puppet_parser, Tech.puppet) - - # Check labels - i = 0 - for atomic_unit in labeled_script.script.atomic_units: - for attribute in atomic_unit.attributes: - assert labeled_script.get_label(attribute) == i - i += 1 - - statement = DeltaPCompiler.compile(labeled_script, Tech.puppet) - - assert statement == delta_p_puppet_2 - - -def test_delta_p_compiler_puppet_if() -> None: - puppet_script = """ -if $x == 'absent' { - file {'/usr/sbin/policy-rc.d': - ensure => absent, - } -} else { - file {'/usr/sbin/policy-rc.d': - ensure => present, - } -} -""" - - with NamedTemporaryFile() as f: - f.write(puppet_script.encode()) - f.flush() - ir = PuppetParser().parse_file(f.name, UnitBlockType.script) - labeled_script = GLITCHLabeler.label(ir, Tech.puppet) - statement = DeltaPCompiler.compile(labeled_script, Tech.puppet) - - assert statement == delta_p_puppet_if - - -def test_delta_p_compiler_puppet_default_state() -> None: - puppet_script = """ -file { '/root/.ssh/config': - content => template('fuel/root_ssh_config.erb'), - owner => 'root', - group => 'root', - mode => '0600', -} -""" - - with NamedTemporaryFile() as f: - f.write(puppet_script.encode()) - f.flush() - ir = PuppetParser().parse_file(f.name, UnitBlockType.script) - labeled_script = GLITCHLabeler.label(ir, Tech.puppet) - statement = DeltaPCompiler.compile(labeled_script, Tech.puppet) - assert statement == delta_p_puppet_default_state - - -def test_delta_p_to_filesystems() -> None: - statement = delta_p_puppet - fss = statement.to_filesystems() - assert len(fss) == 1 - assert fss[0].state == { - "/var/www/customers/public_html/index.php": File( - "0755", "web_admin", "

Hello World

" - ) - } - - -def test_delta_p_to_filesystems_2() -> None: - statement = delta_p_puppet_2 - fss = statement.to_filesystems() - assert len(fss) == 1 - assert fss[0].state == {"/usr/sbin/policy-rc.d": Nil()} - - -def test_delta_p_to_filesystems_if() -> None: - statement = delta_p_puppet_if - fss = statement.to_filesystems() - assert len(fss) == 2 - assert fss[0].state == {"/usr/sbin/policy-rc.d": Nil()} - assert fss[1].state == {"/usr/sbin/policy-rc.d": File(None, None, None)} - - -def test_delta_p_to_filesystems_default_state() -> None: - statement = delta_p_puppet_default_state - fss = statement.to_filesystems() - assert len(fss) == 1 - assert fss[0].state == { - "/root/.ssh/config": File( - "0600", "root", "template('fuel/root_ssh_config.erb')" - ) - } diff --git a/glitch/tests/repair/interactive/test_delta_p_minimize.py b/glitch/tests/repair/interactive/test_delta_p_minimize.py deleted file mode 100644 index 47477d02..00000000 --- a/glitch/tests/repair/interactive/test_delta_p_minimize.py +++ /dev/null @@ -1,54 +0,0 @@ -from glitch.repair.interactive.delta_p import * - - -def test_delta_p_minimize_let() -> None: - statement = PLet( - "x", - "test1", - 1, - PCreate(PEConst(const=PStr(value="test23456"))), - ) - - minimized = PStatement.minimize(statement, ["test1"]) - assert isinstance(minimized, PSkip) - - -def test_delta_p_minimize_seq() -> None: - statement = PSeq( - PCreate(PEConst(const=PStr(value="test1"))), - PCreate(PEConst(const=PStr(value="test2"))), - ) - - minimized = PStatement.minimize(statement, ["test1"]) - assert isinstance(minimized, PCreate) - assert minimized.path == PEConst(const=PStr(value="test1")) - - minimized = PStatement.minimize(statement, ["test2"]) - assert isinstance(minimized, PCreate) - assert minimized.path == PEConst(const=PStr(value="test2")) - - minimized = PStatement.minimize(statement, ["test3"]) - assert isinstance(minimized, PSkip) - - -def test_delta_p_minimize_if() -> None: - statement = PIf( - PBool(True), - PCreate(PEConst(const=PStr(value="test2"))), - PCreate(PEConst(const=PStr(value="test3"))), - ) - - minimized = PStatement.minimize(statement, ["test2"]) - assert isinstance(minimized, PIf) - assert minimized == PIf( - PBool(True), PCreate(PEConst(const=PStr(value="test2"))), PSkip() - ) - - minimized = PStatement.minimize(statement, ["test3"]) - assert isinstance(minimized, PIf) - assert minimized == PIf( - PBool(True), PSkip(), PCreate(PEConst(const=PStr(value="test3"))) - ) - - minimized = PStatement.minimize(statement, ["test1"]) - assert isinstance(minimized, PSkip) diff --git a/glitch/tests/repair/interactive/test_patch_solver.py b/glitch/tests/repair/interactive/test_patch_solver.py deleted file mode 100644 index a8e7b23e..00000000 --- a/glitch/tests/repair/interactive/test_patch_solver.py +++ /dev/null @@ -1,295 +0,0 @@ -from z3 import ModelRef -from tempfile import NamedTemporaryFile - -from glitch.repair.interactive.delta_p import * -from glitch.repair.interactive.solver import PatchSolver -from glitch.repair.interactive.values import UNDEF -from glitch.parsers.puppet import PuppetParser -from glitch.parsers.ansible import AnsibleParser -from glitch.parsers.parser import Parser -from glitch.repair.interactive.compiler.labeler import GLITCHLabeler -from glitch.repair.interactive.compiler.compiler import DeltaPCompiler -from glitch.repr.inter import UnitBlockType -from glitch.tech import Tech - - -puppet_script_1 = """ -file { '/var/www/customers/public_html/index.php': - path => '/var/www/customers/public_html/index.php', - content => '

Hello World

', - ensure => present, - mode => '0755', - owner => 'web_admin' -} -""" - -puppet_script_2 = """ - file { '/etc/icinga2/conf.d/test.conf': - ensure => file, - tag => 'icinga2::config::file', - } -""" - -puppet_script_3 = """ -file { 'test1': - ensure => file, -} - -file { 'test2': - ensure => file, -} -""" - -puppet_script_4 = """ -if $x == 'absent' { - file {'/usr/sbin/policy-rc.d': - ensure => absent, - } -} else { - file {'/usr/sbin/policy-rc.d': - ensure => present, - } -} -""" - -puppet_script_5 = """ -file { '/etc/dhcp/dhclient-enter-hooks': - content => template('fuel/dhclient-enter-hooks.erb'), - owner => 'root', - group => 'root', - mode => '0755', -} -""" - -ansible_script_1 = """ ---- -- ansible.builtin.file: - path: "/var/www/customers/public_html/index.php" - state: file - owner: "web_admin" - mode: '0755' -""" - -labeled_script = None -statement = None - - -def setup_patch_solver( - script: str, - parser: Parser, - script_type: UnitBlockType, - tech: Tech, -) -> None: - global labeled_script, statement - DeltaPCompiler._condition = 0 - with NamedTemporaryFile() as f: - f.write(script.encode()) - f.flush() - parsed_file = parser.parse_file(f.name, script_type) - labeled_script = GLITCHLabeler.label(parsed_file, tech) - statement = DeltaPCompiler.compile(labeled_script, tech) - - -def patch_solver_apply( - solver: PatchSolver, - model: ModelRef, - filesystem: FileSystemState, - tech: Tech, - n_filesystems: int = 1, -) -> None: - solver.apply_patch(model, labeled_script) - statement = DeltaPCompiler.compile(labeled_script, tech) - filesystems = statement.to_filesystems() - assert len(filesystems) == n_filesystems - assert any(fs.state == filesystem.state for fs in filesystems) - - -# TODO: Refactor tests - - -def test_patch_solver_if() -> None: - setup_patch_solver( - puppet_script_4, PuppetParser(), UnitBlockType.script, Tech.puppet - ) - filesystem = FileSystemState() - filesystem.state["/usr/sbin/policy-rc.d"] = File(None, None, None) - - solver = PatchSolver(statement, filesystem) - models = solver.solve() - assert len(models) == 2 - - assert models[0][solver.sum_var] == 8 - assert models[0][solver.vars["dejavu-condition-1"]] - assert not models[0][solver.vars["dejavu-condition-2"]] - assert models[0][solver.vars["state-0"]] == "absent" - assert models[0][solver.vars["state-1"]] == "present" - - assert models[1][solver.sum_var] == 7 - assert not models[1][solver.vars["dejavu-condition-1"]] - assert models[1][solver.vars["dejavu-condition-2"]] - assert models[1][solver.vars["state-0"]] == "present" - assert models[1][solver.vars["state-1"]] == "present" - - patch_solver_apply(solver, models[0], filesystem, Tech.puppet, n_filesystems=2) - - -def test_patch_solver_mode() -> None: - setup_patch_solver( - puppet_script_1, PuppetParser(), UnitBlockType.script, Tech.puppet - ) - filesystem = FileSystemState() - filesystem.state["/var/www/customers/public_html/index.php"] = File( - mode="0777", - owner="web_admin", - content="

Hello World

", - ) - - solver = PatchSolver(statement, filesystem) - models = solver.solve() - assert len(models) == 1 - model = models[0] - assert model[solver.sum_var] == 3 - assert model[solver.unchanged[1]] == 1 - assert model[solver.unchanged[2]] == 1 - assert model[solver.unchanged[3]] == 0 - assert model[solver.unchanged[4]] == 1 - assert ( - model[solver.vars["content-1"]] - == "

Hello World

" - ) - assert model[solver.vars["state-2"]] == "present" - assert model[solver.vars["mode-3"]] == "0777" - assert model[solver.vars["owner-4"]] == "web_admin" - patch_solver_apply(solver, model, filesystem, Tech.puppet) - - -def test_patch_solver_owner() -> None: - setup_patch_solver( - puppet_script_2, PuppetParser(), UnitBlockType.script, Tech.puppet - ) - filesystem = FileSystemState() - filesystem.state["/etc/icinga2/conf.d/test.conf"] = File(None, "new", None) - solver = PatchSolver(statement, filesystem) - models = solver.solve() - assert len(models) == 1 - model = models[0] - assert model[solver.sum_var] == 3 - assert model[solver.unchanged[0]] == 1 - assert model[solver.unchanged[2]] == 1 - assert model[solver.unchanged[3]] == 0 - assert model[solver.unchanged[4]] == 1 - assert model[solver.vars["state-0"]] == "present" - assert model[solver.vars["sketched-content-2"]] == UNDEF - assert model[solver.vars["sketched-owner-3"]] == "new" - assert model[solver.vars["sketched-mode-4"]] == UNDEF - - patch_solver_apply(solver, model, filesystem, Tech.puppet) - - -def test_patch_solver_two_files() -> None: - setup_patch_solver( - puppet_script_3, PuppetParser(), UnitBlockType.script, Tech.puppet - ) - filesystem = FileSystemState() - filesystem.state["test1"] = File(None, "new", None) - filesystem.state["test2"] = File("0666", None, None) - solver = PatchSolver(statement, filesystem) - models = solver.solve() - assert len(models) == 1 - model = models[0] - assert model[solver.sum_var] == 6 - - patch_solver_apply(solver, model, filesystem, Tech.puppet) - - -def test_patch_solver_delete_file() -> None: - setup_patch_solver( - puppet_script_1, PuppetParser(), UnitBlockType.script, Tech.puppet - ) - filesystem = FileSystemState() - filesystem.state["/var/www/customers/public_html/index.php"] = Nil() - - solver = PatchSolver(statement, filesystem) - models = solver.solve() - assert len(models) == 1 - model = models[0] - assert model[solver.sum_var] == 0 - assert model[solver.unchanged[1]] == 0 - assert model[solver.unchanged[2]] == 0 - assert model[solver.unchanged[3]] == 0 - assert model[solver.unchanged[4]] == 0 - assert model[solver.vars["content-1"]] == UNDEF - assert model[solver.vars["state-2"]] == "absent" - assert model[solver.vars["mode-3"]] == UNDEF - assert model[solver.vars["owner-4"]] == UNDEF - patch_solver_apply(solver, model, filesystem, Tech.puppet) - - -def test_patch_solver_remove_content() -> None: - setup_patch_solver( - puppet_script_1, PuppetParser(), UnitBlockType.script, Tech.puppet - ) - filesystem = FileSystemState() - filesystem.state["/var/www/customers/public_html/index.php"] = File( - mode="0755", owner="web_admin", content=None - ) - - solver = PatchSolver(statement, filesystem) - models = solver.solve() - assert len(models) == 1 - model = models[0] - assert model[solver.sum_var] == 3 - assert model[solver.unchanged[1]] == 0 - assert model[solver.unchanged[2]] == 1 - assert model[solver.unchanged[3]] == 1 - assert model[solver.unchanged[4]] == 1 - assert model[solver.vars["content-1"]] == UNDEF - assert model[solver.vars["state-2"]] == "present" - assert model[solver.vars["mode-3"]] == "0755" - assert model[solver.vars["owner-4"]] == "web_admin" - patch_solver_apply(solver, model, filesystem, Tech.puppet) - - -def test_patch_solver_mode_ansible() -> None: - setup_patch_solver( - ansible_script_1, AnsibleParser(), UnitBlockType.tasks, Tech.ansible - ) - filesystem = FileSystemState() - filesystem.state["/var/www/customers/public_html/index.php"] = File( - mode="0777", - owner="web_admin", - content=None, - ) - - solver = PatchSolver(statement, filesystem) - models = solver.solve() - assert len(models) == 1 - model = models[0] - assert model[solver.sum_var] == 3 - assert model[solver.unchanged[1]] == 1 - assert model[solver.unchanged[2]] == 1 - assert model[solver.unchanged[3]] == 0 - assert model[solver.unchanged[4]] == 1 - assert model[solver.vars["state-1"]] == "present" - assert model[solver.vars["owner-2"]] == "web_admin" - assert model[solver.vars["mode-3"]] == "0777" - assert model[solver.vars["sketched-content-4"]] == UNDEF - patch_solver_apply(solver, model, filesystem, Tech.ansible) - - -def test_patch_solver_new_attribute_difficult_name() -> None: - """ - This test requires the solver to create a new attribute "state". - However, the attribute "state" should be called "ensure" in Puppet, - so it is required to do the translation back. - """ - setup_patch_solver( - puppet_script_5, PuppetParser(), UnitBlockType.script, Tech.puppet - ) - filesystem = FileSystemState() - filesystem.state["/etc/dhcp/dhclient-enter-hooks"] = Nil() - - solver = PatchSolver(statement, filesystem) - models = solver.solve() - assert len(models) == 1 - patch_solver_apply(solver, models[0], filesystem, Tech.puppet) diff --git a/glitch/tests/security/chef/test_security.py b/glitch/tests/security/chef/test_security.py deleted file mode 100644 index d2d8e65a..00000000 --- a/glitch/tests/security/chef/test_security.py +++ /dev/null @@ -1,84 +0,0 @@ -import unittest - -from glitch.analysis.security import SecurityVisitor -from glitch.parsers.chef import ChefParser -from glitch.tech import Tech - - -class TestSecurity(unittest.TestCase): - def __help_test(self, path, n_errors: int, codes, lines) -> None: - parser = ChefParser() - inter = parser.parse(path, "script", False) - analysis = SecurityVisitor(Tech.chef) - analysis.config("configs/default.ini") - errors = list( - filter(lambda e: e.code.startswith("sec_"), set(analysis.check(inter))) - ) - errors = sorted(errors, key=lambda e: (e.path, e.line, e.code)) - self.assertEqual(len(errors), n_errors) - for i in range(n_errors): - self.assertEqual(errors[i].code, codes[i]) - self.assertEqual(errors[i].line, lines[i]) - - def test_chef_http(self) -> None: - self.__help_test("tests/security/chef/files/http.rb", 1, ["sec_https"], [3]) - - def test_chef_susp_comment(self) -> None: - self.__help_test("tests/security/chef/files/susp.rb", 1, ["sec_susp_comm"], [1]) - - def test_chef_def_admin(self) -> None: - self.__help_test( - "tests/security/chef/files/admin.rb", - 3, - ["sec_def_admin", "sec_hard_secr", "sec_hard_user"], - [8, 8, 8], - ) - - def test_chef_empt_pass(self) -> None: - self.__help_test( - "tests/security/chef/files/empty.rb", 1, ["sec_empty_pass"], [1] - ) - - def test_chef_weak_crypt(self) -> None: - self.__help_test( - "tests/security/chef/files/weak_crypt.rb", 1, ["sec_weak_crypt"], [4] - ) - - def test_chef_hard_secr(self) -> None: - self.__help_test( - "tests/security/chef/files/hard_secr.rb", - 2, - ["sec_hard_pass", "sec_hard_secr"], - [8, 8], - ) - - def test_chef_invalid_bind(self) -> None: - self.__help_test( - "tests/security/chef/files/inv_bind.rb", 1, ["sec_invalid_bind"], [7] - ) - - def test_chef_int_check(self) -> None: - self.__help_test( - "tests/security/chef/files/int_check.rb", 1, ["sec_no_int_check"], [1] - ) - - def test_chef_missing_default(self) -> None: - self.__help_test( - "tests/security/chef/files/missing_default.rb", - 1, - ["sec_no_default_switch"], - [2], - ) - - def test_chef_full_permission(self) -> None: - self.__help_test( - "tests/security/chef/files/full_permission.rb", - 1, - ["sec_full_permission_filesystem"], - [3], - ) - - def test_chef_obs_command(self) -> None: - self.__help_test( - "tests/security/chef/files/obs_command.rb", 1, ["sec_obsolete_command"], [2] - ) diff --git a/glitch/tests/security/docker/files/admin.Dockerfile b/glitch/tests/security/docker/files/admin.Dockerfile deleted file mode 100644 index c8a3878c..00000000 --- a/glitch/tests/security/docker/files/admin.Dockerfile +++ /dev/null @@ -1,4 +0,0 @@ -FROM python as builder -USER root - -FROM ubuntu as runtime \ No newline at end of file diff --git a/glitch/tests/security/docker/files/empty.Dockerfile b/glitch/tests/security/docker/files/empty.Dockerfile deleted file mode 100644 index a3fd40e5..00000000 --- a/glitch/tests/security/docker/files/empty.Dockerfile +++ /dev/null @@ -1,4 +0,0 @@ -FROM python as builder -USER builder - -ENV root_password "" \ No newline at end of file diff --git a/glitch/tests/security/docker/files/full_permission.Dockerfile b/glitch/tests/security/docker/files/full_permission.Dockerfile deleted file mode 100644 index 92bdbfd7..00000000 --- a/glitch/tests/security/docker/files/full_permission.Dockerfile +++ /dev/null @@ -1,3 +0,0 @@ -FROM python as builder -USER builder -RUN chmod a+xrw Anaconda3-5.0.1-Linux-x86_64.sh \ No newline at end of file diff --git a/glitch/tests/security/docker/files/hard_secr.Dockerfile b/glitch/tests/security/docker/files/hard_secr.Dockerfile deleted file mode 100644 index d6a63c04..00000000 --- a/glitch/tests/security/docker/files/hard_secr.Dockerfile +++ /dev/null @@ -1,3 +0,0 @@ -FROM postgres -USER builder -ENV POSTGRES_PASSWORD "password" \ No newline at end of file diff --git a/glitch/tests/security/docker/files/http.Dockerfile b/glitch/tests/security/docker/files/http.Dockerfile deleted file mode 100644 index 481a9cc8..00000000 --- a/glitch/tests/security/docker/files/http.Dockerfile +++ /dev/null @@ -1,5 +0,0 @@ -FROM python as builder -USER builder - -ARG URL -RUN curl http://$URL.com diff --git a/glitch/tests/security/docker/files/int_check.Dockerfile b/glitch/tests/security/docker/files/int_check.Dockerfile deleted file mode 100644 index 8913c358..00000000 --- a/glitch/tests/security/docker/files/int_check.Dockerfile +++ /dev/null @@ -1,6 +0,0 @@ -FROM ubuntu -USER ubuntu -RUN wget https://repo.continuum.io/archive/Anaconda3-5.0.1-Linux-x86_64.sh -RUN wget https://repo.continuum.io/archive/Anaconda3-5-Linux-x86_64.sh - -RUN gpg Anaconda3-5.0.1-Linux-x86_64.sh \ No newline at end of file diff --git a/glitch/tests/security/docker/files/inv_bind.Dockerfile b/glitch/tests/security/docker/files/inv_bind.Dockerfile deleted file mode 100644 index 7f20e100..00000000 --- a/glitch/tests/security/docker/files/inv_bind.Dockerfile +++ /dev/null @@ -1,4 +0,0 @@ -FROM ubuntu -USER ubuntu - -CMD ["uvicorn", "--host", "0.0.0.0", "main:app"] \ No newline at end of file diff --git a/glitch/tests/security/docker/files/non_off_image.Dockerfile b/glitch/tests/security/docker/files/non_off_image.Dockerfile deleted file mode 100644 index b0e35abb..00000000 --- a/glitch/tests/security/docker/files/non_off_image.Dockerfile +++ /dev/null @@ -1,2 +0,0 @@ -FROM quay.io/maksynbilenko/oracle-12c-base:latest -USER user \ No newline at end of file diff --git a/glitch/tests/security/docker/files/obs_command.Dockerfile b/glitch/tests/security/docker/files/obs_command.Dockerfile deleted file mode 100644 index 729e26ea..00000000 --- a/glitch/tests/security/docker/files/obs_command.Dockerfile +++ /dev/null @@ -1,4 +0,0 @@ -FROM ubuntu -USER ubuntu - -RUN pack foo bar \ No newline at end of file diff --git a/glitch/tests/security/docker/files/susp.Dockerfile b/glitch/tests/security/docker/files/susp.Dockerfile deleted file mode 100644 index 2497a350..00000000 --- a/glitch/tests/security/docker/files/susp.Dockerfile +++ /dev/null @@ -1,3 +0,0 @@ -FROM ubuntu -USER ubuntu -# TODO: Implement missing features \ No newline at end of file diff --git a/glitch/tests/security/docker/files/weak_crypt.Dockerfile b/glitch/tests/security/docker/files/weak_crypt.Dockerfile deleted file mode 100644 index f0c383e6..00000000 --- a/glitch/tests/security/docker/files/weak_crypt.Dockerfile +++ /dev/null @@ -1,8 +0,0 @@ -FROM ubuntu -USER ubuntu - -ARG USER -ARG PASS - -RUN useradd -h $USER -RUN usermod -p $(mkpasswd -H md5 $PASS) $USER diff --git a/glitch/tests/security/docker/test_security.py b/glitch/tests/security/docker/test_security.py deleted file mode 100644 index 33d90dee..00000000 --- a/glitch/tests/security/docker/test_security.py +++ /dev/null @@ -1,108 +0,0 @@ -import os -import unittest - -from glitch.analysis.security import SecurityVisitor -from glitch.parsers.docker import DockerParser -from glitch.repr.inter import UnitBlockType -from glitch.tech import Tech - - -class TestSecurity(unittest.TestCase): - def __help_test(self, path, n_errors: int, codes, lines) -> None: - parser = DockerParser() - inter = parser.parse(path, UnitBlockType.script, False) - analysis = SecurityVisitor(Tech.docker) - analysis.config("configs/default.ini") - errors = list( - filter(lambda e: e.code.startswith("sec_"), set(analysis.check(inter))) - ) - errors = sorted(errors, key=lambda e: (e.path, e.line, e.code)) - self.assertEqual(len(errors), n_errors) - for i in range(n_errors): - self.assertEqual(errors[i].code, codes[i]) - self.assertEqual(errors[i].line, lines[i]) - - def tearDown(self) -> None: - super().tearDown() - if os.path.exists("Dockerfile"): - os.remove("Dockerfile") - - def test_docker_admin(self) -> None: - self.__help_test( - "tests/security/docker/files/admin.Dockerfile", - 2, - ["sec_def_admin", "sec_def_admin"], - [2, 4], - ) - - def test_docker_empty(self) -> None: - self.__help_test( - "tests/security/docker/files/empty.Dockerfile", 1, ["sec_empty_pass"], [4] - ) - pass - - def test_docker_full_permission(self) -> None: - self.__help_test( - "tests/security/docker/files/full_permission.Dockerfile", - 1, - ["sec_full_permission_filesystem"], - [3], - ) - - def test_docker_hard_secret(self) -> None: - self.__help_test( - "tests/security/docker/files/hard_secr.Dockerfile", - 2, - ["sec_hard_pass", "sec_hard_secr"], - [3, 3], - ) - - def test_docker_http(self) -> None: - self.__help_test( - "tests/security/docker/files/http.Dockerfile", 1, ["sec_https"], [5] - ) - - def test_docker_int_check(self) -> None: - self.__help_test( - "tests/security/docker/files/int_check.Dockerfile", - 1, - ["sec_no_int_check"], - [4], - ) - - def test_docker_inv_bind(self) -> None: - self.__help_test( - "tests/security/docker/files/inv_bind.Dockerfile", - 1, - ["sec_invalid_bind"], - [4], - ) - - def test_docker_non_official_image(self) -> None: - self.__help_test( - "tests/security/docker/files/non_off_image.Dockerfile", - 1, - ["sec_non_official_image"], - [1], - ) - - def test_docker_obs_command(self) -> None: - self.__help_test( - "tests/security/docker/files/obs_command.Dockerfile", - 1, - ["sec_obsolete_command"], - [4], - ) - - def test_docker_susp(self) -> None: - self.__help_test( - "tests/security/docker/files/susp.Dockerfile", 1, ["sec_susp_comm"], [3] - ) - - def test_docker_weak_crypt(self) -> None: - self.__help_test( - "tests/security/docker/files/weak_crypt.Dockerfile", - 1, - ["sec_weak_crypt"], - [8], - ) diff --git a/glitch/tests/security/puppet/test_security.py b/glitch/tests/security/puppet/test_security.py deleted file mode 100644 index 918ebf17..00000000 --- a/glitch/tests/security/puppet/test_security.py +++ /dev/null @@ -1,89 +0,0 @@ -import unittest - -from glitch.analysis.security import SecurityVisitor -from glitch.parsers.puppet import PuppetParser -from glitch.tech import Tech - - -class TestSecurity(unittest.TestCase): - def __help_test(self, path, n_errors: int, codes, lines) -> None: - parser = PuppetParser() - inter = parser.parse(path, "script", False) - analysis = SecurityVisitor(Tech.puppet) - analysis.config("configs/default.ini") - errors = list( - filter(lambda e: e.code.startswith("sec_"), set(analysis.check(inter))) - ) - errors = sorted(errors, key=lambda e: (e.path, e.line, e.code)) - self.assertEqual(len(errors), n_errors) - for i in range(n_errors): - self.assertEqual(errors[i].code, codes[i]) - self.assertEqual(errors[i].line, lines[i]) - - def test_puppet_http(self) -> None: - self.__help_test("tests/security/puppet/files/http.pp", 1, ["sec_https"], [2]) - - def test_puppet_susp_comment(self) -> None: - self.__help_test( - "tests/security/puppet/files/susp.pp", 1, ["sec_susp_comm"], [19] - ) - - def test_puppet_def_admin(self) -> None: - self.__help_test( - "tests/security/puppet/files/admin.pp", - 3, - ["sec_def_admin", "sec_hard_secr", "sec_hard_user"], - [7, 7, 7], - ) - - def test_puppet_empt_pass(self) -> None: - self.__help_test( - "tests/security/puppet/files/empty.pp", 1, ["sec_empty_pass"], [1] - ) - - def test_puppet_weak_crypt(self) -> None: - self.__help_test( - "tests/security/puppet/files/weak_crypt.pp", 1, ["sec_weak_crypt"], [12] - ) - - def test_puppet_hard_secr(self) -> None: - self.__help_test( - "tests/security/puppet/files/hard_secr.pp", - 2, - ["sec_hard_pass", "sec_hard_secr"], - [2, 2], - ) - - def test_puppet_invalid_bind(self) -> None: - self.__help_test( - "tests/security/puppet/files/inv_bind.pp", 1, ["sec_invalid_bind"], [12] - ) - - def test_puppet_int_check(self) -> None: - self.__help_test( - "tests/security/puppet/files/int_check.pp", 1, ["sec_no_int_check"], [5] - ) - - def test_puppet_missing_default(self) -> None: - self.__help_test( - "tests/security/puppet/files/missing_default.pp", - 2, - ["sec_no_default_switch", "sec_no_default_switch"], - [2, 7], - ) - - def test_puppet_full_perm(self) -> None: - self.__help_test( - "tests/security/puppet/files/full_permission.pp", - 1, - ["sec_full_permission_filesystem"], - [4], - ) - - def test_puppet_obs_command(self) -> None: - self.__help_test( - "tests/security/puppet/files/obs_command.pp", - 1, - ["sec_obsolete_command"], - [2], - ) diff --git a/poetry.lock b/poetry.lock index 70963a15..b6a52e31 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.1.2 and should not be changed by hand. [[package]] name = "attrs" @@ -6,18 +6,19 @@ version = "25.3.0" description = "Classes Without Boilerplate" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "attrs-25.3.0-py3-none-any.whl", hash = "sha256:427318ce031701fea540783410126f03899a97ffc6f61596ad581ac2e40e3bc3"}, {file = "attrs-25.3.0.tar.gz", hash = "sha256:75d7cefc7fb576747b2c81b4442d4d4a1ce0900973527c011d1030fd3bf4af1b"}, ] [package.extras] -benchmark = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-codspeed", "pytest-mypy-plugins", "pytest-xdist[psutil]"] -cov = ["cloudpickle", "coverage[toml] (>=5.3)", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] -dev = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pre-commit-uv", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] +benchmark = ["cloudpickle ; platform_python_implementation == \"CPython\"", "hypothesis", "mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pympler", "pytest (>=4.3.0)", "pytest-codspeed", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pytest-xdist[psutil]"] +cov = ["cloudpickle ; platform_python_implementation == \"CPython\"", "coverage[toml] (>=5.3)", "hypothesis", "mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pytest-xdist[psutil]"] +dev = ["cloudpickle ; platform_python_implementation == \"CPython\"", "hypothesis", "mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pre-commit-uv", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pytest-xdist[psutil]"] docs = ["cogapp", "furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier"] -tests = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] -tests-mypy = ["mypy (>=1.11.1)", "pytest-mypy-plugins"] +tests = ["cloudpickle ; platform_python_implementation == \"CPython\"", "hypothesis", "mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pytest-xdist[psutil]"] +tests-mypy = ["mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\""] [[package]] name = "bashlex" @@ -25,20 +26,70 @@ version = "0.18" description = "Python parser for bash" optional = false python-versions = ">=2.7, !=3.0, !=3.1, !=3.2, !=3.3, !=3.4" +groups = ["main"] files = [ {file = "bashlex-0.18-py2.py3-none-any.whl", hash = "sha256:91d73a23a3e51711919c1c899083890cdecffc91d8c088942725ac13e9dcfffa"}, {file = "bashlex-0.18.tar.gz", hash = "sha256:5bb03a01c6d5676338c36fd1028009c8ad07e7d61d8a1ce3f513b7fff52796ee"}, ] +[[package]] +name = "black" +version = "25.9.0" +description = "The uncompromising code formatter." +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "black-25.9.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ce41ed2614b706fd55fd0b4a6909d06b5bab344ffbfadc6ef34ae50adba3d4f7"}, + {file = "black-25.9.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2ab0ce111ef026790e9b13bd216fa7bc48edd934ffc4cbf78808b235793cbc92"}, + {file = "black-25.9.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f96b6726d690c96c60ba682955199f8c39abc1ae0c3a494a9c62c0184049a713"}, + {file = "black-25.9.0-cp310-cp310-win_amd64.whl", hash = "sha256:d119957b37cc641596063cd7db2656c5be3752ac17877017b2ffcdb9dfc4d2b1"}, + {file = "black-25.9.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:456386fe87bad41b806d53c062e2974615825c7a52159cde7ccaeb0695fa28fa"}, + {file = "black-25.9.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a16b14a44c1af60a210d8da28e108e13e75a284bf21a9afa6b4571f96ab8bb9d"}, + {file = "black-25.9.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:aaf319612536d502fdd0e88ce52d8f1352b2c0a955cc2798f79eeca9d3af0608"}, + {file = "black-25.9.0-cp311-cp311-win_amd64.whl", hash = "sha256:c0372a93e16b3954208417bfe448e09b0de5cc721d521866cd9e0acac3c04a1f"}, + {file = "black-25.9.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:1b9dc70c21ef8b43248f1d86aedd2aaf75ae110b958a7909ad8463c4aa0880b0"}, + {file = "black-25.9.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8e46eecf65a095fa62e53245ae2795c90bdecabd53b50c448d0a8bcd0d2e74c4"}, + {file = "black-25.9.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9101ee58ddc2442199a25cb648d46ba22cd580b00ca4b44234a324e3ec7a0f7e"}, + {file = "black-25.9.0-cp312-cp312-win_amd64.whl", hash = "sha256:77e7060a00c5ec4b3367c55f39cf9b06e68965a4f2e61cecacd6d0d9b7ec945a"}, + {file = "black-25.9.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0172a012f725b792c358d57fe7b6b6e8e67375dd157f64fa7a3097b3ed3e2175"}, + {file = "black-25.9.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:3bec74ee60f8dfef564b573a96b8930f7b6a538e846123d5ad77ba14a8d7a64f"}, + {file = "black-25.9.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b756fc75871cb1bcac5499552d771822fd9db5a2bb8db2a7247936ca48f39831"}, + {file = "black-25.9.0-cp313-cp313-win_amd64.whl", hash = "sha256:846d58e3ce7879ec1ffe816bb9df6d006cd9590515ed5d17db14e17666b2b357"}, + {file = "black-25.9.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:ef69351df3c84485a8beb6f7b8f9721e2009e20ef80a8d619e2d1788b7816d47"}, + {file = "black-25.9.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e3c1f4cd5e93842774d9ee4ef6cd8d17790e65f44f7cdbaab5f2cf8ccf22a823"}, + {file = "black-25.9.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:154b06d618233fe468236ba1f0e40823d4eb08b26f5e9261526fde34916b9140"}, + {file = "black-25.9.0-cp39-cp39-win_amd64.whl", hash = "sha256:e593466de7b998374ea2585a471ba90553283fb9beefcfa430d84a2651ed5933"}, + {file = "black-25.9.0-py3-none-any.whl", hash = "sha256:474b34c1342cdc157d307b56c4c65bce916480c4a8f6551fdc6bf9b486a7c4ae"}, + {file = "black-25.9.0.tar.gz", hash = "sha256:0474bca9a0dd1b51791fcc507a4e02078a1c63f6d4e4ae5544b9848c7adfb619"}, +] + +[package.dependencies] +click = ">=8.0.0" +mypy-extensions = ">=0.4.3" +packaging = ">=22.0" +pathspec = ">=0.9.0" +platformdirs = ">=2" +pytokens = ">=0.1.10" +tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} +typing-extensions = {version = ">=4.0.1", markers = "python_version < \"3.11\""} + +[package.extras] +colorama = ["colorama (>=0.4.3)"] +d = ["aiohttp (>=3.10)"] +jupyter = ["ipython (>=7.8.0)", "tokenize-rt (>=3.2.0)"] +uvloop = ["uvloop (>=0.15.2)"] + [[package]] name = "certifi" -version = "2025.8.3" +version = "2024.6.2" description = "Python package for providing Mozilla's CA Bundle." optional = false -python-versions = ">=3.7" +python-versions = ">=3.6" +groups = ["main"] files = [ - {file = "certifi-2025.8.3-py3-none-any.whl", hash = "sha256:f6c12493cfb1b06ba2ff328595af9350c65d6644968e5d3a2ffd78699af217a5"}, - {file = "certifi-2025.8.3.tar.gz", hash = "sha256:e564105f78ded564e3ae7c923924435e1daa7463faeab5bb932bc53ffae63407"}, + {file = "certifi-2024.6.2-py3-none-any.whl", hash = "sha256:ddc6c8ce995e6987e7faf5e3f1b02b302836a0e5d98ece18392cb1a36c72ad56"}, + {file = "certifi-2024.6.2.tar.gz", hash = "sha256:3cd43f1c6fa7dedc5899d69d3ad0398fd018ad1a17fba83ddaf78aa46c747516"}, ] [[package]] @@ -47,6 +98,7 @@ version = "3.4.3" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." optional = false python-versions = ">=3.7" +groups = ["main"] files = [ {file = "charset_normalizer-3.4.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:fb7f67a1bfa6e40b438170ebdc8158b78dc465a5a67b6dde178a46987b244a72"}, {file = "charset_normalizer-3.4.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cc9370a2da1ac13f0153780040f465839e6cccb4a1e44810124b4e22483c93fe"}, @@ -135,6 +187,7 @@ version = "8.1.7" description = "Composable command line interface toolkit" optional = false python-versions = ">=3.7" +groups = ["main"] files = [ {file = "click-8.1.7-py3-none-any.whl", hash = "sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28"}, {file = "click-8.1.7.tar.gz", hash = "sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de"}, @@ -149,10 +202,12 @@ version = "0.4.6" description = "Cross-platform colored terminal text." optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +groups = ["main", "dev"] files = [ {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, ] +markers = {main = "platform_system == \"Windows\"", dev = "sys_platform == \"win32\""} [[package]] name = "configparser" @@ -160,6 +215,7 @@ version = "5.3.0" description = "Updated configparser from stdlib for earlier Pythons." optional = false python-versions = ">=3.7" +groups = ["main"] files = [ {file = "configparser-5.3.0-py3-none-any.whl", hash = "sha256:b065779fd93c6bf4cee42202fa4351b4bb842e96a3fb469440e484517a49b9fa"}, {file = "configparser-5.3.0.tar.gz", hash = "sha256:8be267824b541c09b08db124917f48ab525a6c3e837011f3130781a224c57090"}, @@ -167,18 +223,7 @@ files = [ [package.extras] docs = ["jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx"] -testing = ["flake8 (<5)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "types-backports"] - -[[package]] -name = "dockerfile-parse" -version = "2.0.0" -description = "Python library for Dockerfile manipulation" -optional = false -python-versions = ">=3.6" -files = [ - {file = "dockerfile-parse-2.0.0.tar.gz", hash = "sha256:21fe7d510642f2b61a999d45c3d9745f950e11fe6ba2497555b8f63782b78e45"}, - {file = "dockerfile_parse-2.0.0-py2.py3-none-any.whl", hash = "sha256:d8d9100f8255914378bc0524ffaad68ef043885b0fb40a6936b1bba458f40c3f"}, -] +testing = ["flake8 (<5)", "pytest (>=6)", "pytest-black (>=0.3.7) ; platform_python_implementation != \"PyPy\"", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1) ; platform_python_implementation != \"PyPy\"", "types-backports"] [[package]] name = "exceptiongroup" @@ -186,6 +231,8 @@ version = "1.3.0" description = "Backport of PEP 654 (exception groups)" optional = false python-versions = ">=3.7" +groups = ["dev"] +markers = "python_version == \"3.10\"" files = [ {file = "exceptiongroup-1.3.0-py3-none-any.whl", hash = "sha256:4d111e6e0c13d0644cad6ddaa7ed0261a0b36971f6d23e7ec9b4b9097da78a10"}, {file = "exceptiongroup-1.3.0.tar.gz", hash = "sha256:b241f5885f560bc56a59ee63ca4c6a8bfa46ae4ad651af316d4e81817bb9fd88"}, @@ -197,26 +244,13 @@ typing-extensions = {version = ">=4.6.0", markers = "python_version < \"3.13\""} [package.extras] test = ["pytest (>=6)"] -[[package]] -name = "glitch-python-hcl2" -version = "0.1.4" -description = "A parser for HCL2" -optional = false -python-versions = ">=3.7" -files = [ - {file = "glitch-python-hcl2-0.1.4.tar.gz", hash = "sha256:fc2961171706fc228607493c2a13f454e46aa04a28a448ab6d9d7d324494f60f"}, - {file = "glitch_python_hcl2-0.1.4-py3-none-any.whl", hash = "sha256:3baeebd261413a4870f86d824e1fd48971bfb1473db500340c3d30588c4c5e90"}, -] - -[package.dependencies] -lark = ">=1.1.5,<2.0" - [[package]] name = "idna" version = "3.10" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.6" +groups = ["main"] files = [ {file = "idna-3.10-py3-none-any.whl", hash = "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3"}, {file = "idna-3.10.tar.gz", hash = "sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9"}, @@ -231,21 +265,21 @@ version = "2.1.0" description = "brain-dead simple config-ini parsing" optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "iniconfig-2.1.0-py3-none-any.whl", hash = "sha256:9deba5723312380e77435581c6bf4935c94cbfab9b1ed33ef8d238ea168eb760"}, {file = "iniconfig-2.1.0.tar.gz", hash = "sha256:3abbd2e30b36733fee78f9c7f7308f2d0050e88f0087fd25c2645f63c773e1c7"}, ] [[package]] -name = "jinja2" -version = "3.1.2" +name = "Jinja2" +version = "3.2.0.dev0" description = "A very fast and expressive template engine." optional = false -python-versions = ">=3.7" -files = [ - {file = "Jinja2-3.1.2-py3-none-any.whl", hash = "sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61"}, - {file = "Jinja2-3.1.2.tar.gz", hash = "sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852"}, -] +python-versions = ">=3.8" +groups = ["main"] +files = [] +develop = false [package.dependencies] MarkupSafe = ">=2.0" @@ -253,26 +287,34 @@ MarkupSafe = ">=2.0" [package.extras] i18n = ["Babel (>=2.7)"] +[package.source] +type = "git" +url = "https://github.com/Nfsaavedra/jinja" +reference = "tok_loc" +resolved_reference = "ab3be80906494926ab47da0b103af6fa2c6ca544" + [[package]] name = "joblib" -version = "1.5.2" +version = "1.4.2" description = "Lightweight pipelining with Python functions" optional = false -python-versions = ">=3.9" +python-versions = ">=3.8" +groups = ["main"] files = [ - {file = "joblib-1.5.2-py3-none-any.whl", hash = "sha256:4e1f0bdbb987e6d843c70cf43714cb276623def372df3c22fe5266b2670bc241"}, - {file = "joblib-1.5.2.tar.gz", hash = "sha256:3faa5c39054b2f03ca547da9b2f52fde67c06240c31853f306aea97f13647b55"}, + {file = "joblib-1.4.2-py3-none-any.whl", hash = "sha256:06d478d5674cbc267e7496a410ee875abd68e4340feff4490bcb7afb88060ae6"}, + {file = "joblib-1.4.2.tar.gz", hash = "sha256:2382c5816b2636fbd20a09e0f4e9dad4736765fdfb7dca582943b9c1366b3f0e"}, ] [[package]] name = "jsonschema" -version = "4.25.1" +version = "4.22.0" description = "An implementation of JSON Schema validation for Python" optional = false -python-versions = ">=3.9" +python-versions = ">=3.8" +groups = ["main"] files = [ - {file = "jsonschema-4.25.1-py3-none-any.whl", hash = "sha256:3fba0169e345c7175110351d456342c364814cfcf3b964ba4587f22915230a63"}, - {file = "jsonschema-4.25.1.tar.gz", hash = "sha256:e4a9655ce0da0c0b67a085847e00a3a51449e1157f4f75e9fb5aa545e122eb85"}, + {file = "jsonschema-4.22.0-py3-none-any.whl", hash = "sha256:ff4cfd6b1367a40e7bc6411caec72effadd3db0bbe5017de188f2d6108335802"}, + {file = "jsonschema-4.22.0.tar.gz", hash = "sha256:5b22d434a45935119af990552c862e5d6d564e8f6601206b305a61fdf661a2b7"}, ] [package.dependencies] @@ -283,7 +325,7 @@ rpds-py = ">=0.7.1" [package.extras] format = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3987", "uri-template", "webcolors (>=1.11)"] -format-nongpl = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "rfc3987-syntax (>=1.1.0)", "uri-template", "webcolors (>=24.6.0)"] +format-nongpl = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "uri-template", "webcolors (>=1.11)"] [[package]] name = "jsonschema-specifications" @@ -291,6 +333,7 @@ version = "2025.9.1" description = "The JSON Schema meta-schemas and vocabularies, exposed as a Registry" optional = false python-versions = ">=3.9" +groups = ["main"] files = [ {file = "jsonschema_specifications-2025.9.1-py3-none-any.whl", hash = "sha256:98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe"}, {file = "jsonschema_specifications-2025.9.1.tar.gz", hash = "sha256:b540987f239e745613c7a9176f3edb72b832a4ac465cf02712288397832b5e8d"}, @@ -301,13 +344,14 @@ referencing = ">=0.31.0" [[package]] name = "lark" -version = "1.3.0" +version = "1.2.2" description = "a modern parsing library" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ - {file = "lark-1.3.0-py3-none-any.whl", hash = "sha256:80661f261fb2584a9828a097a2432efd575af27d20be0fd35d17f0fe37253831"}, - {file = "lark-1.3.0.tar.gz", hash = "sha256:9a3839d0ca5e1faf7cfa3460e420e859b66bcbde05b634e73c369c8244c5fa48"}, + {file = "lark-1.2.2-py3-none-any.whl", hash = "sha256:c2276486b02f0f1b90be155f2c8ba4a8e194d42775786db622faccd652d8e80c"}, + {file = "lark-1.2.2.tar.gz", hash = "sha256:ca807d0162cd16cef15a8feecb862d7319e7a09bdb13aef927968e45040fed80"}, ] [package.extras] @@ -322,6 +366,7 @@ version = "3.0.3" description = "Safely add untrusted strings to HTML/XML markup." optional = false python-versions = ">=3.9" +groups = ["main"] files = [ {file = "markupsafe-3.0.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:2f981d352f04553a7171b8e44369f2af4055f888dfb147d55e42d29e29e74559"}, {file = "markupsafe-3.0.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e1c1493fb6e50ab01d20a22826e57520f1284df32f2d8601fdd90b6304601419"}, @@ -414,12 +459,25 @@ files = [ {file = "markupsafe-3.0.3.tar.gz", hash = "sha256:722695808f4b6457b320fdc131280796bdceb04ab50fe1795cd540799ebe1698"}, ] +[[package]] +name = "mypy-extensions" +version = "1.1.0" +description = "Type system extensions for programs checked with the mypy type checker." +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505"}, + {file = "mypy_extensions-1.1.0.tar.gz", hash = "sha256:52e68efc3284861e772bbcd66823fde5ae21fd2fdb51c62a211403730b916558"}, +] + [[package]] name = "nltk" version = "3.9.2" description = "Natural Language Toolkit" optional = false python-versions = ">=3.9" +groups = ["main"] files = [ {file = "nltk-3.9.2-py3-none-any.whl", hash = "sha256:1e209d2b3009110635ed9709a67a1a3e33a10f799490fa71cf4bec218c11c88a"}, {file = "nltk-3.9.2.tar.gz", hash = "sha256:0f409e9b069ca4177c1903c3e843eef90c7e92992fa4931ae607da6de49e1419"}, @@ -439,12 +497,26 @@ plot = ["matplotlib"] tgrep = ["pyparsing"] twitter = ["twython"] +[[package]] +name = "nodeenv" +version = "1.10.0" +description = "Node.js virtual environment builder" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +groups = ["main"] +files = [ + {file = "nodeenv-1.10.0-py2.py3-none-any.whl", hash = "sha256:5bb13e3eed2923615535339b3c620e76779af4cb4c6a90deccc9e36b274d3827"}, + {file = "nodeenv-1.10.0.tar.gz", hash = "sha256:996c191ad80897d076bdfba80a41994c2b47c68e224c542b48feba42ba00f8bb"}, +] + [[package]] name = "numpy" version = "2.2.6" description = "Fundamental package for array computing in Python" optional = false python-versions = ">=3.10" +groups = ["main"] +markers = "python_version == \"3.10\"" files = [ {file = "numpy-2.2.6-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b412caa66f72040e6d268491a59f2c43bf03eb6c96dd8f0307829feb7fa2b6fb"}, {file = "numpy-2.2.6-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8e41fd67c52b86603a91c1a505ebaef50b3314de0213461c7a6e99c9a3beff90"}, @@ -509,6 +581,8 @@ version = "2.3.3" description = "Fundamental package for array computing in Python" optional = false python-versions = ">=3.11" +groups = ["main"] +markers = "python_version >= \"3.11\"" files = [ {file = "numpy-2.3.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0ffc4f5caba7dfcbe944ed674b7eef683c7e94874046454bb79ed7ee0236f59d"}, {file = "numpy-2.3.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e7e946c7170858a0295f79a60214424caac2ffdb0063d4d79cb681f9aa0aa569"}, @@ -588,13 +662,14 @@ files = [ [[package]] name = "packaging" -version = "25.0" +version = "24.1" description = "Core utilities for Python packages" optional = false python-versions = ">=3.8" +groups = ["main", "dev"] files = [ - {file = "packaging-25.0-py3-none-any.whl", hash = "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484"}, - {file = "packaging-25.0.tar.gz", hash = "sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f"}, + {file = "packaging-24.1-py3-none-any.whl", hash = "sha256:5b8f2217dbdbd2f7f384c41c628544e6d52f2d0f53c6d0c3ea61aa5d1d7ff124"}, + {file = "packaging-24.1.tar.gz", hash = "sha256:026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002"}, ] [[package]] @@ -603,6 +678,7 @@ version = "2.3.3" description = "Powerful data structures for data analysis, time series, and statistics" optional = false python-versions = ">=3.9" +groups = ["main"] files = [ {file = "pandas-2.3.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:376c6446ae31770764215a6c937f72d917f214b43560603cd60da6408f183b6c"}, {file = "pandas-2.3.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e19d192383eab2f4ceb30b412b22ea30690c9e618f78870357ae1d682912015a"}, @@ -696,12 +772,48 @@ sql-other = ["SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "adbc-d test = ["hypothesis (>=6.46.1)", "pytest (>=7.3.2)", "pytest-xdist (>=2.2.0)"] xml = ["lxml (>=4.9.2)"] +[[package]] +name = "pathspec" +version = "1.0.4" +description = "Utility library for gitignore style pattern matching of file paths." +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "pathspec-1.0.4-py3-none-any.whl", hash = "sha256:fb6ae2fd4e7c921a165808a552060e722767cfa526f99ca5156ed2ce45a5c723"}, + {file = "pathspec-1.0.4.tar.gz", hash = "sha256:0210e2ae8a21a9137c0d470578cb0e595af87edaa6ebf12ff176f14a02e0e645"}, +] + +[package.extras] +hyperscan = ["hyperscan (>=0.7)"] +optional = ["typing-extensions (>=4)"] +re2 = ["google-re2 (>=1.1)"] +tests = ["pytest (>=9)", "typing-extensions (>=4.15)"] + +[[package]] +name = "platformdirs" +version = "4.5.1" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a `user data dir`." +optional = false +python-versions = ">=3.10" +groups = ["main"] +files = [ + {file = "platformdirs-4.5.1-py3-none-any.whl", hash = "sha256:d03afa3963c806a9bed9d5125c8f4cb2fdaf74a55ab60e5d59b3fde758104d31"}, + {file = "platformdirs-4.5.1.tar.gz", hash = "sha256:61d5cdcc6065745cdd94f0f878977f8de9437be93de97c1c12f853c9c0cdcbda"}, +] + +[package.extras] +docs = ["furo (>=2025.9.25)", "proselint (>=0.14)", "sphinx (>=8.2.3)", "sphinx-autodoc-typehints (>=3.2)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=8.4.2)", "pytest-cov (>=7)", "pytest-mock (>=3.15.1)"] +type = ["mypy (>=1.18.2)"] + [[package]] name = "pluggy" version = "1.6.0" description = "plugin and hook calling mechanisms for python" optional = false python-versions = ">=3.9" +groups = ["dev"] files = [ {file = "pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746"}, {file = "pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3"}, @@ -717,6 +829,7 @@ version = "3.11" description = "Python Lex & Yacc" optional = false python-versions = "*" +groups = ["main"] files = [ {file = "ply-3.11-py2.py3-none-any.whl", hash = "sha256:096f9b8350b65ebd2fd1346b12452efe5b9607f7482813ffca50c22722a807ce"}, {file = "ply-3.11.tar.gz", hash = "sha256:00c7c1aaa88358b9c765b6d3000c6eec0ba42abca5351b095321aef446081da3"}, @@ -728,6 +841,7 @@ version = "3.6.0" description = "A simple Python library for easily displaying tabular data in a visually appealing ASCII table format" optional = false python-versions = ">=3.7" +groups = ["main"] files = [ {file = "prettytable-3.6.0-py3-none-any.whl", hash = "sha256:3b767129491767a3a5108e6f305cbaa650f8020a7db5dfe994a2df7ef7bad0fe"}, {file = "prettytable-3.6.0.tar.gz", hash = "sha256:2e0026af955b4ea67b22122f310b90eae890738c08cb0458693a49b6221530ac"}, @@ -741,24 +855,47 @@ tests = ["pytest", "pytest-cov", "pytest-lazy-fixture"] [[package]] name = "puppetparser" -version = "0.2.4" +version = "0.2.14" description = "A parser from Puppet to an object model" optional = false python-versions = "<4.0,>=3.9" +groups = ["main"] files = [ - {file = "puppetparser-0.2.4-py3-none-any.whl", hash = "sha256:3c625c7f8826f705b61c21aef5b59a759dd0ba79e72779ec9e664f900d8c713c"}, - {file = "puppetparser-0.2.4.tar.gz", hash = "sha256:2db6273653e94f018582aa87da2780a5b7e2b2320cfa1485e312e4a16029accd"}, + {file = "puppetparser-0.2.14-py3-none-any.whl", hash = "sha256:b7c09042337bbb7e94e37d19cfd88e9a5062a823bc05629141f33267a0f75734"}, + {file = "puppetparser-0.2.14.tar.gz", hash = "sha256:762f11d72b0b5face39a5e9b961f52e892c0e05a85a0b2cf2c5a437b91c759dc"}, ] [package.dependencies] ply = "3.11" +[[package]] +name = "pyright" +version = "1.1.408" +description = "Command line wrapper for pyright" +optional = false +python-versions = ">=3.7" +groups = ["main"] +files = [ + {file = "pyright-1.1.408-py3-none-any.whl", hash = "sha256:090b32865f4fdb1e0e6cd82bf5618480d48eecd2eb2e70f960982a3d9a4c17c1"}, + {file = "pyright-1.1.408.tar.gz", hash = "sha256:f28f2321f96852fa50b5829ea492f6adb0e6954568d1caa3f3af3a5f555eb684"}, +] + +[package.dependencies] +nodeenv = ">=1.6.0" +typing-extensions = ">=4.1" + +[package.extras] +all = ["nodejs-wheel-binaries", "twine (>=3.4.1)"] +dev = ["twine (>=3.4.1)"] +nodejs = ["nodejs-wheel-binaries"] + [[package]] name = "pytest" version = "7.3.1" description = "pytest: simple powerful testing with Python" optional = false python-versions = ">=3.7" +groups = ["dev"] files = [ {file = "pytest-7.3.1-py3-none-any.whl", hash = "sha256:3799fa815351fea3a5e96ac7e503a96fa51cc9942c3753cda7651b93c1cfa362"}, {file = "pytest-7.3.1.tar.gz", hash = "sha256:434afafd78b1d78ed0addf160ad2b77a30d35d4bdf8af234fe621919d9ed15e3"}, @@ -781,6 +918,7 @@ version = "2.9.0.post0" description = "Extensions to the standard Python datetime module" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +groups = ["main"] files = [ {file = "python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3"}, {file = "python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"}, @@ -789,12 +927,87 @@ files = [ [package.dependencies] six = ">=1.5" +[[package]] +name = "python-hcl2" +version = "0.1.dev296+g986d879" +description = "A parser for HCL2" +optional = false +python-versions = ">=3.7.0" +groups = ["main"] +files = [] +develop = false + +[package.dependencies] +lark = ">=1,<2" + +[package.source] +type = "git" +url = "https://github.com/Nfsaavedra/python-hcl2.git" +reference = "feature/hcl2-reverse-transformer" +resolved_reference = "986d879aa65e800322221d61169d52d5111ce7e6" + +[[package]] +name = "pytokens" +version = "0.4.1" +description = "A Fast, spec compliant Python 3.14+ tokenizer that runs on older Pythons." +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "pytokens-0.4.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2a44ed93ea23415c54f3face3b65ef2b844d96aeb3455b8a69b3df6beab6acc5"}, + {file = "pytokens-0.4.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:add8bf86b71a5d9fb5b89f023a80b791e04fba57960aa790cc6125f7f1d39dfe"}, + {file = "pytokens-0.4.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:670d286910b531c7b7e3c0b453fd8156f250adb140146d234a82219459b9640c"}, + {file = "pytokens-0.4.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:4e691d7f5186bd2842c14813f79f8884bb03f5995f0575272009982c5ac6c0f7"}, + {file = "pytokens-0.4.1-cp310-cp310-win_amd64.whl", hash = "sha256:27b83ad28825978742beef057bfe406ad6ed524b2d28c252c5de7b4a6dd48fa2"}, + {file = "pytokens-0.4.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d70e77c55ae8380c91c0c18dea05951482e263982911fc7410b1ffd1dadd3440"}, + {file = "pytokens-0.4.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4a58d057208cb9075c144950d789511220b07636dd2e4708d5645d24de666bdc"}, + {file = "pytokens-0.4.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b49750419d300e2b5a3813cf229d4e5a4c728dae470bcc89867a9ad6f25a722d"}, + {file = "pytokens-0.4.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:d9907d61f15bf7261d7e775bd5d7ee4d2930e04424bab1972591918497623a16"}, + {file = "pytokens-0.4.1-cp311-cp311-win_amd64.whl", hash = "sha256:ee44d0f85b803321710f9239f335aafe16553b39106384cef8e6de40cb4ef2f6"}, + {file = "pytokens-0.4.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:140709331e846b728475786df8aeb27d24f48cbcf7bcd449f8de75cae7a45083"}, + {file = "pytokens-0.4.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6d6c4268598f762bc8e91f5dbf2ab2f61f7b95bdc07953b602db879b3c8c18e1"}, + {file = "pytokens-0.4.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:24afde1f53d95348b5a0eb19488661147285ca4dd7ed752bbc3e1c6242a304d1"}, + {file = "pytokens-0.4.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5ad948d085ed6c16413eb5fec6b3e02fa00dc29a2534f088d3302c47eb59adf9"}, + {file = "pytokens-0.4.1-cp312-cp312-win_amd64.whl", hash = "sha256:3f901fe783e06e48e8cbdc82d631fca8f118333798193e026a50ce1b3757ea68"}, + {file = "pytokens-0.4.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:8bdb9d0ce90cbf99c525e75a2fa415144fd570a1ba987380190e8b786bc6ef9b"}, + {file = "pytokens-0.4.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5502408cab1cb18e128570f8d598981c68a50d0cbd7c61312a90507cd3a1276f"}, + {file = "pytokens-0.4.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:29d1d8fb1030af4d231789959f21821ab6325e463f0503a61d204343c9b355d1"}, + {file = "pytokens-0.4.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:970b08dd6b86058b6dc07efe9e98414f5102974716232d10f32ff39701e841c4"}, + {file = "pytokens-0.4.1-cp313-cp313-win_amd64.whl", hash = "sha256:9bd7d7f544d362576be74f9d5901a22f317efc20046efe2034dced238cbbfe78"}, + {file = "pytokens-0.4.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:4a14d5f5fc78ce85e426aa159489e2d5961acf0e47575e08f35584009178e321"}, + {file = "pytokens-0.4.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:97f50fd18543be72da51dd505e2ed20d2228c74e0464e4262e4899797803d7fa"}, + {file = "pytokens-0.4.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:dc74c035f9bfca0255c1af77ddd2d6ae8419012805453e4b0e7513e17904545d"}, + {file = "pytokens-0.4.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f66a6bbe741bd431f6d741e617e0f39ec7257ca1f89089593479347cc4d13324"}, + {file = "pytokens-0.4.1-cp314-cp314-win_amd64.whl", hash = "sha256:b35d7e5ad269804f6697727702da3c517bb8a5228afa450ab0fa787732055fc9"}, + {file = "pytokens-0.4.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:8fcb9ba3709ff77e77f1c7022ff11d13553f3c30299a9fe246a166903e9091eb"}, + {file = "pytokens-0.4.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:79fc6b8699564e1f9b521582c35435f1bd32dd06822322ec44afdeba666d8cb3"}, + {file = "pytokens-0.4.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d31b97b3de0f61571a124a00ffe9a81fb9939146c122c11060725bd5aea79975"}, + {file = "pytokens-0.4.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:967cf6e3fd4adf7de8fc73cd3043754ae79c36475c1c11d514fc72cf5490094a"}, + {file = "pytokens-0.4.1-cp314-cp314t-win_amd64.whl", hash = "sha256:584c80c24b078eec1e227079d56dc22ff755e0ba8654d8383b2c549107528918"}, + {file = "pytokens-0.4.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:da5baeaf7116dced9c6bb76dc31ba04a2dc3695f3d9f74741d7910122b456edc"}, + {file = "pytokens-0.4.1-cp38-cp38-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:11edda0942da80ff58c4408407616a310adecae1ddd22eef8c692fe266fa5009"}, + {file = "pytokens-0.4.1-cp38-cp38-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0fc71786e629cef478cbf29d7ea1923299181d0699dbe7c3c0f4a583811d9fc1"}, + {file = "pytokens-0.4.1-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:dcafc12c30dbaf1e2af0490978352e0c4041a7cde31f4f81435c2a5e8b9cabb6"}, + {file = "pytokens-0.4.1-cp38-cp38-win_amd64.whl", hash = "sha256:42f144f3aafa5d92bad964d471a581651e28b24434d184871bd02e3a0d956037"}, + {file = "pytokens-0.4.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:34bcc734bd2f2d5fe3b34e7b3c0116bfb2397f2d9666139988e7a3eb5f7400e3"}, + {file = "pytokens-0.4.1-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:941d4343bf27b605e9213b26bfa1c4bf197c9c599a9627eb7305b0defcfe40c1"}, + {file = "pytokens-0.4.1-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3ad72b851e781478366288743198101e5eb34a414f1d5627cdd585ca3b25f1db"}, + {file = "pytokens-0.4.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:682fa37ff4d8e95f7df6fe6fe6a431e8ed8e788023c6bcc0f0880a12eab80ad1"}, + {file = "pytokens-0.4.1-cp39-cp39-win_amd64.whl", hash = "sha256:30f51edd9bb7f85c748979384165601d028b84f7bd13fe14d3e065304093916a"}, + {file = "pytokens-0.4.1-py3-none-any.whl", hash = "sha256:26cef14744a8385f35d0e095dc8b3a7583f6c953c2e3d269c7f82484bf5ad2de"}, + {file = "pytokens-0.4.1.tar.gz", hash = "sha256:292052fe80923aae2260c073f822ceba21f3872ced9a68bb7953b348e561179a"}, +] + +[package.extras] +dev = ["black", "build", "mypy", "pytest", "pytest-cov", "setuptools", "tox", "twine", "wheel"] + [[package]] name = "pytz" version = "2025.2" description = "World timezone definitions, modern and historical" optional = false python-versions = "*" +groups = ["main"] files = [ {file = "pytz-2025.2-py2.py3-none-any.whl", hash = "sha256:5ddf76296dd8c44c26eb8f4b6f35488f3ccbf6fbbd7adee0b7262d43f0ec2f00"}, {file = "pytz-2025.2.tar.gz", hash = "sha256:360b9e3dbb49a209c21ad61809c7fb453643e048b38924c765813546746e81c3"}, @@ -802,158 +1015,124 @@ files = [ [[package]] name = "referencing" -version = "0.36.2" +version = "0.35.1" description = "JSON Referencing + Python" optional = false -python-versions = ">=3.9" +python-versions = ">=3.8" +groups = ["main"] files = [ - {file = "referencing-0.36.2-py3-none-any.whl", hash = "sha256:e8699adbbf8b5c7de96d8ffa0eb5c158b3beafce084968e2ea8bb08c6794dcd0"}, - {file = "referencing-0.36.2.tar.gz", hash = "sha256:df2e89862cd09deabbdba16944cc3f10feb6b3e6f18e902f7cc25609a34775aa"}, + {file = "referencing-0.35.1-py3-none-any.whl", hash = "sha256:eda6d3234d62814d1c64e305c1331c9a3a6132da475ab6382eaa997b21ee75de"}, + {file = "referencing-0.35.1.tar.gz", hash = "sha256:25b42124a6c8b632a425174f24087783efb348a6f1e0008e63cd4466fedf703c"}, ] [package.dependencies] attrs = ">=22.2.0" rpds-py = ">=0.7.0" -typing-extensions = {version = ">=4.4.0", markers = "python_version < \"3.13\""} [[package]] name = "regex" -version = "2025.9.18" +version = "2024.5.15" description = "Alternative regular expression module, to replace re." optional = false -python-versions = ">=3.9" +python-versions = ">=3.8" +groups = ["main"] files = [ - {file = "regex-2025.9.18-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:12296202480c201c98a84aecc4d210592b2f55e200a1d193235c4db92b9f6788"}, - {file = "regex-2025.9.18-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:220381f1464a581f2ea988f2220cf2a67927adcef107d47d6897ba5a2f6d51a4"}, - {file = "regex-2025.9.18-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:87f681bfca84ebd265278b5daa1dcb57f4db315da3b5d044add7c30c10442e61"}, - {file = "regex-2025.9.18-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:34d674cbba70c9398074c8a1fcc1a79739d65d1105de2a3c695e2b05ea728251"}, - {file = "regex-2025.9.18-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:385c9b769655cb65ea40b6eea6ff763cbb6d69b3ffef0b0db8208e1833d4e746"}, - {file = "regex-2025.9.18-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8900b3208e022570ae34328712bef6696de0804c122933414014bae791437ab2"}, - {file = "regex-2025.9.18-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c204e93bf32cd7a77151d44b05eb36f469d0898e3fba141c026a26b79d9914a0"}, - {file = "regex-2025.9.18-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:3acc471d1dd7e5ff82e6cacb3b286750decd949ecd4ae258696d04f019817ef8"}, - {file = "regex-2025.9.18-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:6479d5555122433728760e5f29edb4c2b79655a8deb681a141beb5c8a025baea"}, - {file = "regex-2025.9.18-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:431bd2a8726b000eb6f12429c9b438a24062a535d06783a93d2bcbad3698f8a8"}, - {file = "regex-2025.9.18-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:0cc3521060162d02bd36927e20690129200e5ac9d2c6d32b70368870b122db25"}, - {file = "regex-2025.9.18-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:a021217b01be2d51632ce056d7a837d3fa37c543ede36e39d14063176a26ae29"}, - {file = "regex-2025.9.18-cp310-cp310-win32.whl", hash = "sha256:4a12a06c268a629cb67cc1d009b7bb0be43e289d00d5111f86a2efd3b1949444"}, - {file = "regex-2025.9.18-cp310-cp310-win_amd64.whl", hash = "sha256:47acd811589301298c49db2c56bde4f9308d6396da92daf99cba781fa74aa450"}, - {file = "regex-2025.9.18-cp310-cp310-win_arm64.whl", hash = "sha256:16bd2944e77522275e5ee36f867e19995bcaa533dcb516753a26726ac7285442"}, - {file = "regex-2025.9.18-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:51076980cd08cd13c88eb7365427ae27f0d94e7cebe9ceb2bb9ffdae8fc4d82a"}, - {file = "regex-2025.9.18-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:828446870bd7dee4e0cbeed767f07961aa07f0ea3129f38b3ccecebc9742e0b8"}, - {file = "regex-2025.9.18-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c28821d5637866479ec4cc23b8c990f5bc6dd24e5e4384ba4a11d38a526e1414"}, - {file = "regex-2025.9.18-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:726177ade8e481db669e76bf99de0b278783be8acd11cef71165327abd1f170a"}, - {file = "regex-2025.9.18-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f5cca697da89b9f8ea44115ce3130f6c54c22f541943ac8e9900461edc2b8bd4"}, - {file = "regex-2025.9.18-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:dfbde38f38004703c35666a1e1c088b778e35d55348da2b7b278914491698d6a"}, - {file = "regex-2025.9.18-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f2f422214a03fab16bfa495cfec72bee4aaa5731843b771860a471282f1bf74f"}, - {file = "regex-2025.9.18-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:a295916890f4df0902e4286bc7223ee7f9e925daa6dcdec4192364255b70561a"}, - {file = "regex-2025.9.18-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:5db95ff632dbabc8c38c4e82bf545ab78d902e81160e6e455598014f0abe66b9"}, - {file = "regex-2025.9.18-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:fb967eb441b0f15ae610b7069bdb760b929f267efbf522e814bbbfffdf125ce2"}, - {file = "regex-2025.9.18-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f04d2f20da4053d96c08f7fde6e1419b7ec9dbcee89c96e3d731fca77f411b95"}, - {file = "regex-2025.9.18-cp311-cp311-win32.whl", hash = "sha256:895197241fccf18c0cea7550c80e75f185b8bd55b6924fcae269a1a92c614a07"}, - {file = "regex-2025.9.18-cp311-cp311-win_amd64.whl", hash = "sha256:7e2b414deae99166e22c005e154a5513ac31493db178d8aec92b3269c9cce8c9"}, - {file = "regex-2025.9.18-cp311-cp311-win_arm64.whl", hash = "sha256:fb137ec7c5c54f34a25ff9b31f6b7b0c2757be80176435bf367111e3f71d72df"}, - {file = "regex-2025.9.18-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:436e1b31d7efd4dcd52091d076482031c611dde58bf9c46ca6d0a26e33053a7e"}, - {file = "regex-2025.9.18-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:c190af81e5576b9c5fdc708f781a52ff20f8b96386c6e2e0557a78402b029f4a"}, - {file = "regex-2025.9.18-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:e4121f1ce2b2b5eec4b397cc1b277686e577e658d8f5870b7eb2d726bd2300ab"}, - {file = "regex-2025.9.18-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:300e25dbbf8299d87205e821a201057f2ef9aa3deb29caa01cd2cac669e508d5"}, - {file = "regex-2025.9.18-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7b47fcf9f5316c0bdaf449e879407e1b9937a23c3b369135ca94ebc8d74b1742"}, - {file = "regex-2025.9.18-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:57a161bd3acaa4b513220b49949b07e252165e6b6dc910ee7617a37ff4f5b425"}, - {file = "regex-2025.9.18-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4f130c3a7845ba42de42f380fff3c8aebe89a810747d91bcf56d40a069f15352"}, - {file = "regex-2025.9.18-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:5f96fa342b6f54dcba928dd452e8d8cb9f0d63e711d1721cd765bb9f73bb048d"}, - {file = "regex-2025.9.18-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:0f0d676522d68c207828dcd01fb6f214f63f238c283d9f01d85fc664c7c85b56"}, - {file = "regex-2025.9.18-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:40532bff8a1a0621e7903ae57fce88feb2e8a9a9116d341701302c9302aef06e"}, - {file = "regex-2025.9.18-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:039f11b618ce8d71a1c364fdee37da1012f5a3e79b1b2819a9f389cd82fd6282"}, - {file = "regex-2025.9.18-cp312-cp312-win32.whl", hash = "sha256:e1dd06f981eb226edf87c55d523131ade7285137fbde837c34dc9d1bf309f459"}, - {file = "regex-2025.9.18-cp312-cp312-win_amd64.whl", hash = "sha256:3d86b5247bf25fa3715e385aa9ff272c307e0636ce0c9595f64568b41f0a9c77"}, - {file = "regex-2025.9.18-cp312-cp312-win_arm64.whl", hash = "sha256:032720248cbeeae6444c269b78cb15664458b7bb9ed02401d3da59fe4d68c3a5"}, - {file = "regex-2025.9.18-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:2a40f929cd907c7e8ac7566ac76225a77701a6221bca937bdb70d56cb61f57b2"}, - {file = "regex-2025.9.18-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:c90471671c2cdf914e58b6af62420ea9ecd06d1554d7474d50133ff26ae88feb"}, - {file = "regex-2025.9.18-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:1a351aff9e07a2dabb5022ead6380cff17a4f10e4feb15f9100ee56c4d6d06af"}, - {file = "regex-2025.9.18-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bc4b8e9d16e20ddfe16430c23468a8707ccad3365b06d4536142e71823f3ca29"}, - {file = "regex-2025.9.18-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4b8cdbddf2db1c5e80338ba2daa3cfa3dec73a46fff2a7dda087c8efbf12d62f"}, - {file = "regex-2025.9.18-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a276937d9d75085b2c91fb48244349c6954f05ee97bba0963ce24a9d915b8b68"}, - {file = "regex-2025.9.18-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:92a8e375ccdc1256401c90e9dc02b8642894443d549ff5e25e36d7cf8a80c783"}, - {file = "regex-2025.9.18-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:0dc6893b1f502d73037cf807a321cdc9be29ef3d6219f7970f842475873712ac"}, - {file = "regex-2025.9.18-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:a61e85bfc63d232ac14b015af1261f826260c8deb19401c0597dbb87a864361e"}, - {file = "regex-2025.9.18-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:1ef86a9ebc53f379d921fb9a7e42b92059ad3ee800fcd9e0fe6181090e9f6c23"}, - {file = "regex-2025.9.18-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:d3bc882119764ba3a119fbf2bd4f1b47bc56c1da5d42df4ed54ae1e8e66fdf8f"}, - {file = "regex-2025.9.18-cp313-cp313-win32.whl", hash = "sha256:3810a65675845c3bdfa58c3c7d88624356dd6ee2fc186628295e0969005f928d"}, - {file = "regex-2025.9.18-cp313-cp313-win_amd64.whl", hash = "sha256:16eaf74b3c4180ede88f620f299e474913ab6924d5c4b89b3833bc2345d83b3d"}, - {file = "regex-2025.9.18-cp313-cp313-win_arm64.whl", hash = "sha256:4dc98ba7dd66bd1261927a9f49bd5ee2bcb3660f7962f1ec02617280fc00f5eb"}, - {file = "regex-2025.9.18-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:fe5d50572bc885a0a799410a717c42b1a6b50e2f45872e2b40f4f288f9bce8a2"}, - {file = "regex-2025.9.18-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:1b9d9a2d6cda6621551ca8cf7a06f103adf72831153f3c0d982386110870c4d3"}, - {file = "regex-2025.9.18-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:13202e4c4ac0ef9a317fff817674b293c8f7e8c68d3190377d8d8b749f566e12"}, - {file = "regex-2025.9.18-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:874ff523b0fecffb090f80ae53dc93538f8db954c8bb5505f05b7787ab3402a0"}, - {file = "regex-2025.9.18-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d13ab0490128f2bb45d596f754148cd750411afc97e813e4b3a61cf278a23bb6"}, - {file = "regex-2025.9.18-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:05440bc172bc4b4b37fb9667e796597419404dbba62e171e1f826d7d2a9ebcef"}, - {file = "regex-2025.9.18-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5514b8e4031fdfaa3d27e92c75719cbe7f379e28cacd939807289bce76d0e35a"}, - {file = "regex-2025.9.18-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:65d3c38c39efce73e0d9dc019697b39903ba25b1ad45ebbd730d2cf32741f40d"}, - {file = "regex-2025.9.18-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:ae77e447ebc144d5a26d50055c6ddba1d6ad4a865a560ec7200b8b06bc529368"}, - {file = "regex-2025.9.18-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:e3ef8cf53dc8df49d7e28a356cf824e3623764e9833348b655cfed4524ab8a90"}, - {file = "regex-2025.9.18-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:9feb29817df349c976da9a0debf775c5c33fc1c8ad7b9f025825da99374770b7"}, - {file = "regex-2025.9.18-cp313-cp313t-win32.whl", hash = "sha256:168be0d2f9b9d13076940b1ed774f98595b4e3c7fc54584bba81b3cc4181742e"}, - {file = "regex-2025.9.18-cp313-cp313t-win_amd64.whl", hash = "sha256:d59ecf3bb549e491c8104fea7313f3563c7b048e01287db0a90485734a70a730"}, - {file = "regex-2025.9.18-cp313-cp313t-win_arm64.whl", hash = "sha256:dbef80defe9fb21310948a2595420b36c6d641d9bea4c991175829b2cc4bc06a"}, - {file = "regex-2025.9.18-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:c6db75b51acf277997f3adcd0ad89045d856190d13359f15ab5dda21581d9129"}, - {file = "regex-2025.9.18-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8f9698b6f6895d6db810e0bda5364f9ceb9e5b11328700a90cae573574f61eea"}, - {file = "regex-2025.9.18-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:29cd86aa7cb13a37d0f0d7c21d8d949fe402ffa0ea697e635afedd97ab4b69f1"}, - {file = "regex-2025.9.18-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7c9f285a071ee55cd9583ba24dde006e53e17780bb309baa8e4289cd472bcc47"}, - {file = "regex-2025.9.18-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5adf266f730431e3be9021d3e5b8d5ee65e563fec2883ea8093944d21863b379"}, - {file = "regex-2025.9.18-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1137cabc0f38807de79e28d3f6e3e3f2cc8cfb26bead754d02e6d1de5f679203"}, - {file = "regex-2025.9.18-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7cc9e5525cada99699ca9223cce2d52e88c52a3d2a0e842bd53de5497c604164"}, - {file = "regex-2025.9.18-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:bbb9246568f72dce29bcd433517c2be22c7791784b223a810225af3b50d1aafb"}, - {file = "regex-2025.9.18-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:6a52219a93dd3d92c675383efff6ae18c982e2d7651c792b1e6d121055808743"}, - {file = "regex-2025.9.18-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:ae9b3840c5bd456780e3ddf2f737ab55a79b790f6409182012718a35c6d43282"}, - {file = "regex-2025.9.18-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:d488c236ac497c46a5ac2005a952c1a0e22a07be9f10c3e735bc7d1209a34773"}, - {file = "regex-2025.9.18-cp314-cp314-win32.whl", hash = "sha256:0c3506682ea19beefe627a38872d8da65cc01ffa25ed3f2e422dffa1474f0788"}, - {file = "regex-2025.9.18-cp314-cp314-win_amd64.whl", hash = "sha256:57929d0f92bebb2d1a83af372cd0ffba2263f13f376e19b1e4fa32aec4efddc3"}, - {file = "regex-2025.9.18-cp314-cp314-win_arm64.whl", hash = "sha256:6a4b44df31d34fa51aa5c995d3aa3c999cec4d69b9bd414a8be51984d859f06d"}, - {file = "regex-2025.9.18-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:b176326bcd544b5e9b17d6943f807697c0cb7351f6cfb45bf5637c95ff7e6306"}, - {file = "regex-2025.9.18-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:0ffd9e230b826b15b369391bec167baed57c7ce39efc35835448618860995946"}, - {file = "regex-2025.9.18-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:ec46332c41add73f2b57e2f5b642f991f6b15e50e9f86285e08ffe3a512ac39f"}, - {file = "regex-2025.9.18-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b80fa342ed1ea095168a3f116637bd1030d39c9ff38dc04e54ef7c521e01fc95"}, - {file = "regex-2025.9.18-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f4d97071c0ba40f0cf2a93ed76e660654c399a0a04ab7d85472239460f3da84b"}, - {file = "regex-2025.9.18-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:0ac936537ad87cef9e0e66c5144484206c1354224ee811ab1519a32373e411f3"}, - {file = "regex-2025.9.18-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:dec57f96d4def58c422d212d414efe28218d58537b5445cf0c33afb1b4768571"}, - {file = "regex-2025.9.18-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:48317233294648bf7cd068857f248e3a57222259a5304d32c7552e2284a1b2ad"}, - {file = "regex-2025.9.18-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:274687e62ea3cf54846a9b25fc48a04459de50af30a7bd0b61a9e38015983494"}, - {file = "regex-2025.9.18-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:a78722c86a3e7e6aadf9579e3b0ad78d955f2d1f1a8ca4f67d7ca258e8719d4b"}, - {file = "regex-2025.9.18-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:06104cd203cdef3ade989a1c45b6215bf42f8b9dd705ecc220c173233f7cba41"}, - {file = "regex-2025.9.18-cp314-cp314t-win32.whl", hash = "sha256:2e1eddc06eeaffd249c0adb6fafc19e2118e6308c60df9db27919e96b5656096"}, - {file = "regex-2025.9.18-cp314-cp314t-win_amd64.whl", hash = "sha256:8620d247fb8c0683ade51217b459cb4a1081c0405a3072235ba43a40d355c09a"}, - {file = "regex-2025.9.18-cp314-cp314t-win_arm64.whl", hash = "sha256:b7531a8ef61de2c647cdf68b3229b071e46ec326b3138b2180acb4275f470b01"}, - {file = "regex-2025.9.18-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:3dbcfcaa18e9480669030d07371713c10b4f1a41f791ffa5cb1a99f24e777f40"}, - {file = "regex-2025.9.18-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:1e85f73ef7095f0380208269055ae20524bfde3f27c5384126ddccf20382a638"}, - {file = "regex-2025.9.18-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:9098e29b3ea4ffffeade423f6779665e2a4f8db64e699c0ed737ef0db6ba7b12"}, - {file = "regex-2025.9.18-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:90b6b7a2d0f45b7ecaaee1aec6b362184d6596ba2092dd583ffba1b78dd0231c"}, - {file = "regex-2025.9.18-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c81b892af4a38286101502eae7aec69f7cd749a893d9987a92776954f3943408"}, - {file = "regex-2025.9.18-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3b524d010973f2e1929aeb635418d468d869a5f77b52084d9f74c272189c251d"}, - {file = "regex-2025.9.18-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6b498437c026a3d5d0be0020023ff76d70ae4d77118e92f6f26c9d0423452446"}, - {file = "regex-2025.9.18-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:0716e4d6e58853d83f6563f3cf25c281ff46cf7107e5f11879e32cb0b59797d9"}, - {file = "regex-2025.9.18-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:065b6956749379d41db2625f880b637d4acc14c0a4de0d25d609a62850e96d36"}, - {file = "regex-2025.9.18-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:d4a691494439287c08ddb9b5793da605ee80299dd31e95fa3f323fac3c33d9d4"}, - {file = "regex-2025.9.18-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:ef8d10cc0989565bcbe45fb4439f044594d5c2b8919d3d229ea2c4238f1d55b0"}, - {file = "regex-2025.9.18-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:4baeb1b16735ac969a7eeecc216f1f8b7caf60431f38a2671ae601f716a32d25"}, - {file = "regex-2025.9.18-cp39-cp39-win32.whl", hash = "sha256:8e5f41ad24a1e0b5dfcf4c4e5d9f5bd54c895feb5708dd0c1d0d35693b24d478"}, - {file = "regex-2025.9.18-cp39-cp39-win_amd64.whl", hash = "sha256:50e8290707f2fb8e314ab3831e594da71e062f1d623b05266f8cfe4db4949afd"}, - {file = "regex-2025.9.18-cp39-cp39-win_arm64.whl", hash = "sha256:039a9d7195fd88c943d7c777d4941e8ef736731947becce773c31a1009cb3c35"}, - {file = "regex-2025.9.18.tar.gz", hash = "sha256:c5ba23274c61c6fef447ba6a39333297d0c247f53059dba0bca415cac511edc4"}, + {file = "regex-2024.5.15-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a81e3cfbae20378d75185171587cbf756015ccb14840702944f014e0d93ea09f"}, + {file = "regex-2024.5.15-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7b59138b219ffa8979013be7bc85bb60c6f7b7575df3d56dc1e403a438c7a3f6"}, + {file = "regex-2024.5.15-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a0bd000c6e266927cb7a1bc39d55be95c4b4f65c5be53e659537537e019232b1"}, + {file = "regex-2024.5.15-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5eaa7ddaf517aa095fa8da0b5015c44d03da83f5bd49c87961e3c997daed0de7"}, + {file = "regex-2024.5.15-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ba68168daedb2c0bab7fd7e00ced5ba90aebf91024dea3c88ad5063c2a562cca"}, + {file = "regex-2024.5.15-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6e8d717bca3a6e2064fc3a08df5cbe366369f4b052dcd21b7416e6d71620dca1"}, + {file = "regex-2024.5.15-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1337b7dbef9b2f71121cdbf1e97e40de33ff114801263b275aafd75303bd62b5"}, + {file = "regex-2024.5.15-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f9ebd0a36102fcad2f03696e8af4ae682793a5d30b46c647eaf280d6cfb32796"}, + {file = "regex-2024.5.15-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:9efa1a32ad3a3ea112224897cdaeb6aa00381627f567179c0314f7b65d354c62"}, + {file = "regex-2024.5.15-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:1595f2d10dff3d805e054ebdc41c124753631b6a471b976963c7b28543cf13b0"}, + {file = "regex-2024.5.15-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:b802512f3e1f480f41ab5f2cfc0e2f761f08a1f41092d6718868082fc0d27143"}, + {file = "regex-2024.5.15-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:a0981022dccabca811e8171f913de05720590c915b033b7e601f35ce4ea7019f"}, + {file = "regex-2024.5.15-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:19068a6a79cf99a19ccefa44610491e9ca02c2be3305c7760d3831d38a467a6f"}, + {file = "regex-2024.5.15-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:1b5269484f6126eee5e687785e83c6b60aad7663dafe842b34691157e5083e53"}, + {file = "regex-2024.5.15-cp310-cp310-win32.whl", hash = "sha256:ada150c5adfa8fbcbf321c30c751dc67d2f12f15bd183ffe4ec7cde351d945b3"}, + {file = "regex-2024.5.15-cp310-cp310-win_amd64.whl", hash = "sha256:ac394ff680fc46b97487941f5e6ae49a9f30ea41c6c6804832063f14b2a5a145"}, + {file = "regex-2024.5.15-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:f5b1dff3ad008dccf18e652283f5e5339d70bf8ba7c98bf848ac33db10f7bc7a"}, + {file = "regex-2024.5.15-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c6a2b494a76983df8e3d3feea9b9ffdd558b247e60b92f877f93a1ff43d26656"}, + {file = "regex-2024.5.15-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a32b96f15c8ab2e7d27655969a23895eb799de3665fa94349f3b2fbfd547236f"}, + {file = "regex-2024.5.15-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:10002e86e6068d9e1c91eae8295ef690f02f913c57db120b58fdd35a6bb1af35"}, + {file = "regex-2024.5.15-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ec54d5afa89c19c6dd8541a133be51ee1017a38b412b1321ccb8d6ddbeb4cf7d"}, + {file = "regex-2024.5.15-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:10e4ce0dca9ae7a66e6089bb29355d4432caed736acae36fef0fdd7879f0b0cb"}, + {file = "regex-2024.5.15-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3e507ff1e74373c4d3038195fdd2af30d297b4f0950eeda6f515ae3d84a1770f"}, + {file = "regex-2024.5.15-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d1f059a4d795e646e1c37665b9d06062c62d0e8cc3c511fe01315973a6542e40"}, + {file = "regex-2024.5.15-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:0721931ad5fe0dda45d07f9820b90b2148ccdd8e45bb9e9b42a146cb4f695649"}, + {file = "regex-2024.5.15-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:833616ddc75ad595dee848ad984d067f2f31be645d603e4d158bba656bbf516c"}, + {file = "regex-2024.5.15-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:287eb7f54fc81546346207c533ad3c2c51a8d61075127d7f6d79aaf96cdee890"}, + {file = "regex-2024.5.15-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:19dfb1c504781a136a80ecd1fff9f16dddf5bb43cec6871778c8a907a085bb3d"}, + {file = "regex-2024.5.15-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:119af6e56dce35e8dfb5222573b50c89e5508d94d55713c75126b753f834de68"}, + {file = "regex-2024.5.15-cp311-cp311-win32.whl", hash = "sha256:1c1c174d6ec38d6c8a7504087358ce9213d4332f6293a94fbf5249992ba54efa"}, + {file = "regex-2024.5.15-cp311-cp311-win_amd64.whl", hash = "sha256:9e717956dcfd656f5055cc70996ee2cc82ac5149517fc8e1b60261b907740201"}, + {file = "regex-2024.5.15-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:632b01153e5248c134007209b5c6348a544ce96c46005d8456de1d552455b014"}, + {file = "regex-2024.5.15-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:e64198f6b856d48192bf921421fdd8ad8eb35e179086e99e99f711957ffedd6e"}, + {file = "regex-2024.5.15-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:68811ab14087b2f6e0fc0c2bae9ad689ea3584cad6917fc57be6a48bbd012c49"}, + {file = "regex-2024.5.15-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f8ec0c2fea1e886a19c3bee0cd19d862b3aa75dcdfb42ebe8ed30708df64687a"}, + {file = "regex-2024.5.15-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d0c0c0003c10f54a591d220997dd27d953cd9ccc1a7294b40a4be5312be8797b"}, + {file = "regex-2024.5.15-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2431b9e263af1953c55abbd3e2efca67ca80a3de8a0437cb58e2421f8184717a"}, + {file = "regex-2024.5.15-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4a605586358893b483976cffc1723fb0f83e526e8f14c6e6614e75919d9862cf"}, + {file = "regex-2024.5.15-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:391d7f7f1e409d192dba8bcd42d3e4cf9e598f3979cdaed6ab11288da88cb9f2"}, + {file = "regex-2024.5.15-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:9ff11639a8d98969c863d4617595eb5425fd12f7c5ef6621a4b74b71ed8726d5"}, + {file = "regex-2024.5.15-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:4eee78a04e6c67e8391edd4dad3279828dd66ac4b79570ec998e2155d2e59fd5"}, + {file = "regex-2024.5.15-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:8fe45aa3f4aa57faabbc9cb46a93363edd6197cbc43523daea044e9ff2fea83e"}, + {file = "regex-2024.5.15-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:d0a3d8d6acf0c78a1fff0e210d224b821081330b8524e3e2bc5a68ef6ab5803d"}, + {file = "regex-2024.5.15-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c486b4106066d502495b3025a0a7251bf37ea9540433940a23419461ab9f2a80"}, + {file = "regex-2024.5.15-cp312-cp312-win32.whl", hash = "sha256:c49e15eac7c149f3670b3e27f1f28a2c1ddeccd3a2812cba953e01be2ab9b5fe"}, + {file = "regex-2024.5.15-cp312-cp312-win_amd64.whl", hash = "sha256:673b5a6da4557b975c6c90198588181029c60793835ce02f497ea817ff647cb2"}, + {file = "regex-2024.5.15-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:87e2a9c29e672fc65523fb47a90d429b70ef72b901b4e4b1bd42387caf0d6835"}, + {file = "regex-2024.5.15-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c3bea0ba8b73b71b37ac833a7f3fd53825924165da6a924aec78c13032f20850"}, + {file = "regex-2024.5.15-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:bfc4f82cabe54f1e7f206fd3d30fda143f84a63fe7d64a81558d6e5f2e5aaba9"}, + {file = "regex-2024.5.15-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e5bb9425fe881d578aeca0b2b4b3d314ec88738706f66f219c194d67179337cb"}, + {file = "regex-2024.5.15-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:64c65783e96e563103d641760664125e91bd85d8e49566ee560ded4da0d3e704"}, + {file = "regex-2024.5.15-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cf2430df4148b08fb4324b848672514b1385ae3807651f3567871f130a728cc3"}, + {file = "regex-2024.5.15-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5397de3219a8b08ae9540c48f602996aa6b0b65d5a61683e233af8605c42b0f2"}, + {file = "regex-2024.5.15-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:455705d34b4154a80ead722f4f185b04c4237e8e8e33f265cd0798d0e44825fa"}, + {file = "regex-2024.5.15-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:b2b6f1b3bb6f640c1a92be3bbfbcb18657b125b99ecf141fb3310b5282c7d4ed"}, + {file = "regex-2024.5.15-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:3ad070b823ca5890cab606c940522d05d3d22395d432f4aaaf9d5b1653e47ced"}, + {file = "regex-2024.5.15-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:5b5467acbfc153847d5adb21e21e29847bcb5870e65c94c9206d20eb4e99a384"}, + {file = "regex-2024.5.15-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:e6662686aeb633ad65be2a42b4cb00178b3fbf7b91878f9446075c404ada552f"}, + {file = "regex-2024.5.15-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:2b4c884767504c0e2401babe8b5b7aea9148680d2e157fa28f01529d1f7fcf67"}, + {file = "regex-2024.5.15-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:3cd7874d57f13bf70078f1ff02b8b0aa48d5b9ed25fc48547516c6aba36f5741"}, + {file = "regex-2024.5.15-cp38-cp38-win32.whl", hash = "sha256:e4682f5ba31f475d58884045c1a97a860a007d44938c4c0895f41d64481edbc9"}, + {file = "regex-2024.5.15-cp38-cp38-win_amd64.whl", hash = "sha256:d99ceffa25ac45d150e30bd9ed14ec6039f2aad0ffa6bb87a5936f5782fc1569"}, + {file = "regex-2024.5.15-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:13cdaf31bed30a1e1c2453ef6015aa0983e1366fad2667657dbcac7b02f67133"}, + {file = "regex-2024.5.15-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:cac27dcaa821ca271855a32188aa61d12decb6fe45ffe3e722401fe61e323cd1"}, + {file = "regex-2024.5.15-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:7dbe2467273b875ea2de38ded4eba86cbcbc9a1a6d0aa11dcf7bd2e67859c435"}, + {file = "regex-2024.5.15-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:64f18a9a3513a99c4bef0e3efd4c4a5b11228b48aa80743be822b71e132ae4f5"}, + {file = "regex-2024.5.15-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d347a741ea871c2e278fde6c48f85136c96b8659b632fb57a7d1ce1872547600"}, + {file = "regex-2024.5.15-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1878b8301ed011704aea4c806a3cadbd76f84dece1ec09cc9e4dc934cfa5d4da"}, + {file = "regex-2024.5.15-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4babf07ad476aaf7830d77000874d7611704a7fcf68c9c2ad151f5d94ae4bfc4"}, + {file = "regex-2024.5.15-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:35cb514e137cb3488bce23352af3e12fb0dbedd1ee6e60da053c69fb1b29cc6c"}, + {file = "regex-2024.5.15-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:cdd09d47c0b2efee9378679f8510ee6955d329424c659ab3c5e3a6edea696294"}, + {file = "regex-2024.5.15-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:72d7a99cd6b8f958e85fc6ca5b37c4303294954eac1376535b03c2a43eb72629"}, + {file = "regex-2024.5.15-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:a094801d379ab20c2135529948cb84d417a2169b9bdceda2a36f5f10977ebc16"}, + {file = "regex-2024.5.15-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:c0c18345010870e58238790a6779a1219b4d97bd2e77e1140e8ee5d14df071aa"}, + {file = "regex-2024.5.15-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:16093f563098448ff6b1fa68170e4acbef94e6b6a4e25e10eae8598bb1694b5d"}, + {file = "regex-2024.5.15-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:e38a7d4e8f633a33b4c7350fbd8bad3b70bf81439ac67ac38916c4a86b465456"}, + {file = "regex-2024.5.15-cp39-cp39-win32.whl", hash = "sha256:71a455a3c584a88f654b64feccc1e25876066c4f5ef26cd6dd711308aa538694"}, + {file = "regex-2024.5.15-cp39-cp39-win_amd64.whl", hash = "sha256:cab12877a9bdafde5500206d1020a584355a97884dfd388af3699e9137bf7388"}, + {file = "regex-2024.5.15.tar.gz", hash = "sha256:d3ee02d9e5f482cc8309134a91eeaacbdd2261ba111b0fef3748eeb4913e6a2c"}, ] [[package]] name = "requests" -version = "2.32.5" +version = "2.32.3" description = "Python HTTP for Humans." optional = false -python-versions = ">=3.9" +python-versions = ">=3.8" +groups = ["main"] files = [ - {file = "requests-2.32.5-py3-none-any.whl", hash = "sha256:2462f94637a34fd532264295e186976db0f5d453d1cdd31473c85a6a161affb6"}, - {file = "requests-2.32.5.tar.gz", hash = "sha256:dbba0bac56e100853db0ea71b82b4dfd5fe2bf6d3754a8893c3af500cec7d7cf"}, + {file = "requests-2.32.3-py3-none-any.whl", hash = "sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6"}, + {file = "requests-2.32.3.tar.gz", hash = "sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760"}, ] [package.dependencies] certifi = ">=2017.4.17" -charset_normalizer = ">=2,<4" +charset-normalizer = ">=2,<4" idna = ">=2.5,<4" urllib3 = ">=1.21.1,<3" @@ -963,166 +1142,111 @@ use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] [[package]] name = "rpds-py" -version = "0.27.1" +version = "0.18.1" description = "Python bindings to Rust's persistent data structures (rpds)" optional = false -python-versions = ">=3.9" +python-versions = ">=3.8" +groups = ["main"] files = [ - {file = "rpds_py-0.27.1-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:68afeec26d42ab3b47e541b272166a0b4400313946871cba3ed3a4fc0cab1cef"}, - {file = "rpds_py-0.27.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:74e5b2f7bb6fa38b1b10546d27acbacf2a022a8b5543efb06cfebc72a59c85be"}, - {file = "rpds_py-0.27.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9024de74731df54546fab0bfbcdb49fae19159ecaecfc8f37c18d2c7e2c0bd61"}, - {file = "rpds_py-0.27.1-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:31d3ebadefcd73b73928ed0b2fd696f7fefda8629229f81929ac9c1854d0cffb"}, - {file = "rpds_py-0.27.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b2e7f8f169d775dd9092a1743768d771f1d1300453ddfe6325ae3ab5332b4657"}, - {file = "rpds_py-0.27.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3d905d16f77eb6ab2e324e09bfa277b4c8e5e6b8a78a3e7ff8f3cdf773b4c013"}, - {file = "rpds_py-0.27.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:50c946f048209e6362e22576baea09193809f87687a95a8db24e5fbdb307b93a"}, - {file = "rpds_py-0.27.1-cp310-cp310-manylinux_2_31_riscv64.whl", hash = "sha256:3deab27804d65cd8289eb814c2c0e807c4b9d9916c9225e363cb0cf875eb67c1"}, - {file = "rpds_py-0.27.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:8b61097f7488de4be8244c89915da8ed212832ccf1e7c7753a25a394bf9b1f10"}, - {file = "rpds_py-0.27.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:8a3f29aba6e2d7d90528d3c792555a93497fe6538aa65eb675b44505be747808"}, - {file = "rpds_py-0.27.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:dd6cd0485b7d347304067153a6dc1d73f7d4fd995a396ef32a24d24b8ac63ac8"}, - {file = "rpds_py-0.27.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:6f4461bf931108c9fa226ffb0e257c1b18dc2d44cd72b125bec50ee0ab1248a9"}, - {file = "rpds_py-0.27.1-cp310-cp310-win32.whl", hash = "sha256:ee5422d7fb21f6a00c1901bf6559c49fee13a5159d0288320737bbf6585bd3e4"}, - {file = "rpds_py-0.27.1-cp310-cp310-win_amd64.whl", hash = "sha256:3e039aabf6d5f83c745d5f9a0a381d031e9ed871967c0a5c38d201aca41f3ba1"}, - {file = "rpds_py-0.27.1-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:be898f271f851f68b318872ce6ebebbc62f303b654e43bf72683dbdc25b7c881"}, - {file = "rpds_py-0.27.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:62ac3d4e3e07b58ee0ddecd71d6ce3b1637de2d373501412df395a0ec5f9beb5"}, - {file = "rpds_py-0.27.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4708c5c0ceb2d034f9991623631d3d23cb16e65c83736ea020cdbe28d57c0a0e"}, - {file = "rpds_py-0.27.1-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:abfa1171a9952d2e0002aba2ad3780820b00cc3d9c98c6630f2e93271501f66c"}, - {file = "rpds_py-0.27.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4b507d19f817ebaca79574b16eb2ae412e5c0835542c93fe9983f1e432aca195"}, - {file = "rpds_py-0.27.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:168b025f8fd8d8d10957405f3fdcef3dc20f5982d398f90851f4abc58c566c52"}, - {file = "rpds_py-0.27.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cb56c6210ef77caa58e16e8c17d35c63fe3f5b60fd9ba9d424470c3400bcf9ed"}, - {file = "rpds_py-0.27.1-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:d252f2d8ca0195faa707f8eb9368955760880b2b42a8ee16d382bf5dd807f89a"}, - {file = "rpds_py-0.27.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:6e5e54da1e74b91dbc7996b56640f79b195d5925c2b78efaa8c5d53e1d88edde"}, - {file = "rpds_py-0.27.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:ffce0481cc6e95e5b3f0a47ee17ffbd234399e6d532f394c8dce320c3b089c21"}, - {file = "rpds_py-0.27.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:a205fdfe55c90c2cd8e540ca9ceba65cbe6629b443bc05db1f590a3db8189ff9"}, - {file = "rpds_py-0.27.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:689fb5200a749db0415b092972e8eba85847c23885c8543a8b0f5c009b1a5948"}, - {file = "rpds_py-0.27.1-cp311-cp311-win32.whl", hash = "sha256:3182af66048c00a075010bc7f4860f33913528a4b6fc09094a6e7598e462fe39"}, - {file = "rpds_py-0.27.1-cp311-cp311-win_amd64.whl", hash = "sha256:b4938466c6b257b2f5c4ff98acd8128ec36b5059e5c8f8372d79316b1c36bb15"}, - {file = "rpds_py-0.27.1-cp311-cp311-win_arm64.whl", hash = "sha256:2f57af9b4d0793e53266ee4325535a31ba48e2f875da81a9177c9926dfa60746"}, - {file = "rpds_py-0.27.1-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:ae2775c1973e3c30316892737b91f9283f9908e3cc7625b9331271eaaed7dc90"}, - {file = "rpds_py-0.27.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2643400120f55c8a96f7c9d858f7be0c88d383cd4653ae2cf0d0c88f668073e5"}, - {file = "rpds_py-0.27.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:16323f674c089b0360674a4abd28d5042947d54ba620f72514d69be4ff64845e"}, - {file = "rpds_py-0.27.1-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9a1f4814b65eacac94a00fc9a526e3fdafd78e439469644032032d0d63de4881"}, - {file = "rpds_py-0.27.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7ba32c16b064267b22f1850a34051121d423b6f7338a12b9459550eb2096e7ec"}, - {file = "rpds_py-0.27.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e5c20f33fd10485b80f65e800bbe5f6785af510b9f4056c5a3c612ebc83ba6cb"}, - {file = "rpds_py-0.27.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:466bfe65bd932da36ff279ddd92de56b042f2266d752719beb97b08526268ec5"}, - {file = "rpds_py-0.27.1-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:41e532bbdcb57c92ba3be62c42e9f096431b4cf478da9bc3bc6ce5c38ab7ba7a"}, - {file = "rpds_py-0.27.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f149826d742b406579466283769a8ea448eed82a789af0ed17b0cd5770433444"}, - {file = "rpds_py-0.27.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:80c60cfb5310677bd67cb1e85a1e8eb52e12529545441b43e6f14d90b878775a"}, - {file = "rpds_py-0.27.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:7ee6521b9baf06085f62ba9c7a3e5becffbc32480d2f1b351559c001c38ce4c1"}, - {file = "rpds_py-0.27.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a512c8263249a9d68cac08b05dd59d2b3f2061d99b322813cbcc14c3c7421998"}, - {file = "rpds_py-0.27.1-cp312-cp312-win32.whl", hash = "sha256:819064fa048ba01b6dadc5116f3ac48610435ac9a0058bbde98e569f9e785c39"}, - {file = "rpds_py-0.27.1-cp312-cp312-win_amd64.whl", hash = "sha256:d9199717881f13c32c4046a15f024971a3b78ad4ea029e8da6b86e5aa9cf4594"}, - {file = "rpds_py-0.27.1-cp312-cp312-win_arm64.whl", hash = "sha256:33aa65b97826a0e885ef6e278fbd934e98cdcfed80b63946025f01e2f5b29502"}, - {file = "rpds_py-0.27.1-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:e4b9fcfbc021633863a37e92571d6f91851fa656f0180246e84cbd8b3f6b329b"}, - {file = "rpds_py-0.27.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:1441811a96eadca93c517d08df75de45e5ffe68aa3089924f963c782c4b898cf"}, - {file = "rpds_py-0.27.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:55266dafa22e672f5a4f65019015f90336ed31c6383bd53f5e7826d21a0e0b83"}, - {file = "rpds_py-0.27.1-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d78827d7ac08627ea2c8e02c9e5b41180ea5ea1f747e9db0915e3adf36b62dcf"}, - {file = "rpds_py-0.27.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae92443798a40a92dc5f0b01d8a7c93adde0c4dc965310a29ae7c64d72b9fad2"}, - {file = "rpds_py-0.27.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c46c9dd2403b66a2a3b9720ec4b74d4ab49d4fabf9f03dfdce2d42af913fe8d0"}, - {file = "rpds_py-0.27.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2efe4eb1d01b7f5f1939f4ef30ecea6c6b3521eec451fb93191bf84b2a522418"}, - {file = "rpds_py-0.27.1-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:15d3b4d83582d10c601f481eca29c3f138d44c92187d197aff663a269197c02d"}, - {file = "rpds_py-0.27.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:4ed2e16abbc982a169d30d1a420274a709949e2cbdef119fe2ec9d870b42f274"}, - {file = "rpds_py-0.27.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a75f305c9b013289121ec0f1181931975df78738cdf650093e6b86d74aa7d8dd"}, - {file = "rpds_py-0.27.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:67ce7620704745881a3d4b0ada80ab4d99df390838839921f99e63c474f82cf2"}, - {file = "rpds_py-0.27.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9d992ac10eb86d9b6f369647b6a3f412fc0075cfd5d799530e84d335e440a002"}, - {file = "rpds_py-0.27.1-cp313-cp313-win32.whl", hash = "sha256:4f75e4bd8ab8db624e02c8e2fc4063021b58becdbe6df793a8111d9343aec1e3"}, - {file = "rpds_py-0.27.1-cp313-cp313-win_amd64.whl", hash = "sha256:f9025faafc62ed0b75a53e541895ca272815bec18abe2249ff6501c8f2e12b83"}, - {file = "rpds_py-0.27.1-cp313-cp313-win_arm64.whl", hash = "sha256:ed10dc32829e7d222b7d3b93136d25a406ba9788f6a7ebf6809092da1f4d279d"}, - {file = "rpds_py-0.27.1-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:92022bbbad0d4426e616815b16bc4127f83c9a74940e1ccf3cfe0b387aba0228"}, - {file = "rpds_py-0.27.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:47162fdab9407ec3f160805ac3e154df042e577dd53341745fc7fb3f625e6d92"}, - {file = "rpds_py-0.27.1-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb89bec23fddc489e5d78b550a7b773557c9ab58b7946154a10a6f7a214a48b2"}, - {file = "rpds_py-0.27.1-cp313-cp313t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e48af21883ded2b3e9eb48cb7880ad8598b31ab752ff3be6457001d78f416723"}, - {file = "rpds_py-0.27.1-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6f5b7bd8e219ed50299e58551a410b64daafb5017d54bbe822e003856f06a802"}, - {file = "rpds_py-0.27.1-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:08f1e20bccf73b08d12d804d6e1c22ca5530e71659e6673bce31a6bb71c1e73f"}, - {file = "rpds_py-0.27.1-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0dc5dceeaefcc96dc192e3a80bbe1d6c410c469e97bdd47494a7d930987f18b2"}, - {file = "rpds_py-0.27.1-cp313-cp313t-manylinux_2_31_riscv64.whl", hash = "sha256:d76f9cc8665acdc0c9177043746775aa7babbf479b5520b78ae4002d889f5c21"}, - {file = "rpds_py-0.27.1-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:134fae0e36022edad8290a6661edf40c023562964efea0cc0ec7f5d392d2aaef"}, - {file = "rpds_py-0.27.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:eb11a4f1b2b63337cfd3b4d110af778a59aae51c81d195768e353d8b52f88081"}, - {file = "rpds_py-0.27.1-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:13e608ac9f50a0ed4faec0e90ece76ae33b34c0e8656e3dceb9a7db994c692cd"}, - {file = "rpds_py-0.27.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:dd2135527aa40f061350c3f8f89da2644de26cd73e4de458e79606384f4f68e7"}, - {file = "rpds_py-0.27.1-cp313-cp313t-win32.whl", hash = "sha256:3020724ade63fe320a972e2ffd93b5623227e684315adce194941167fee02688"}, - {file = "rpds_py-0.27.1-cp313-cp313t-win_amd64.whl", hash = "sha256:8ee50c3e41739886606388ba3ab3ee2aae9f35fb23f833091833255a31740797"}, - {file = "rpds_py-0.27.1-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:acb9aafccaae278f449d9c713b64a9e68662e7799dbd5859e2c6b3c67b56d334"}, - {file = "rpds_py-0.27.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:b7fb801aa7f845ddf601c49630deeeccde7ce10065561d92729bfe81bd21fb33"}, - {file = "rpds_py-0.27.1-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fe0dd05afb46597b9a2e11c351e5e4283c741237e7f617ffb3252780cca9336a"}, - {file = "rpds_py-0.27.1-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b6dfb0e058adb12d8b1d1b25f686e94ffa65d9995a5157afe99743bf7369d62b"}, - {file = "rpds_py-0.27.1-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ed090ccd235f6fa8bb5861684567f0a83e04f52dfc2e5c05f2e4b1309fcf85e7"}, - {file = "rpds_py-0.27.1-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bf876e79763eecf3e7356f157540d6a093cef395b65514f17a356f62af6cc136"}, - {file = "rpds_py-0.27.1-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:12ed005216a51b1d6e2b02a7bd31885fe317e45897de81d86dcce7d74618ffff"}, - {file = "rpds_py-0.27.1-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:ee4308f409a40e50593c7e3bb8cbe0b4d4c66d1674a316324f0c2f5383b486f9"}, - {file = "rpds_py-0.27.1-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:0b08d152555acf1f455154d498ca855618c1378ec810646fcd7c76416ac6dc60"}, - {file = "rpds_py-0.27.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:dce51c828941973a5684d458214d3a36fcd28da3e1875d659388f4f9f12cc33e"}, - {file = "rpds_py-0.27.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:c1476d6f29eb81aa4151c9a31219b03f1f798dc43d8af1250a870735516a1212"}, - {file = "rpds_py-0.27.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:3ce0cac322b0d69b63c9cdb895ee1b65805ec9ffad37639f291dd79467bee675"}, - {file = "rpds_py-0.27.1-cp314-cp314-win32.whl", hash = "sha256:dfbfac137d2a3d0725758cd141f878bf4329ba25e34979797c89474a89a8a3a3"}, - {file = "rpds_py-0.27.1-cp314-cp314-win_amd64.whl", hash = "sha256:a6e57b0abfe7cc513450fcf529eb486b6e4d3f8aee83e92eb5f1ef848218d456"}, - {file = "rpds_py-0.27.1-cp314-cp314-win_arm64.whl", hash = "sha256:faf8d146f3d476abfee026c4ae3bdd9ca14236ae4e4c310cbd1cf75ba33d24a3"}, - {file = "rpds_py-0.27.1-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:ba81d2b56b6d4911ce735aad0a1d4495e808b8ee4dc58715998741a26874e7c2"}, - {file = "rpds_py-0.27.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:84f7d509870098de0e864cad0102711c1e24e9b1a50ee713b65928adb22269e4"}, - {file = "rpds_py-0.27.1-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a9e960fc78fecd1100539f14132425e1d5fe44ecb9239f8f27f079962021523e"}, - {file = "rpds_py-0.27.1-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:62f85b665cedab1a503747617393573995dac4600ff51869d69ad2f39eb5e817"}, - {file = "rpds_py-0.27.1-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fed467af29776f6556250c9ed85ea5a4dd121ab56a5f8b206e3e7a4c551e48ec"}, - {file = "rpds_py-0.27.1-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f2729615f9d430af0ae6b36cf042cb55c0936408d543fb691e1a9e36648fd35a"}, - {file = "rpds_py-0.27.1-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1b207d881a9aef7ba753d69c123a35d96ca7cb808056998f6b9e8747321f03b8"}, - {file = "rpds_py-0.27.1-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:639fd5efec029f99b79ae47e5d7e00ad8a773da899b6309f6786ecaf22948c48"}, - {file = "rpds_py-0.27.1-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:fecc80cb2a90e28af8a9b366edacf33d7a91cbfe4c2c4544ea1246e949cfebeb"}, - {file = "rpds_py-0.27.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:42a89282d711711d0a62d6f57d81aa43a1368686c45bc1c46b7f079d55692734"}, - {file = "rpds_py-0.27.1-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:cf9931f14223de59551ab9d38ed18d92f14f055a5f78c1d8ad6493f735021bbb"}, - {file = "rpds_py-0.27.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:f39f58a27cc6e59f432b568ed8429c7e1641324fbe38131de852cd77b2d534b0"}, - {file = "rpds_py-0.27.1-cp314-cp314t-win32.whl", hash = "sha256:d5fa0ee122dc09e23607a28e6d7b150da16c662e66409bbe85230e4c85bb528a"}, - {file = "rpds_py-0.27.1-cp314-cp314t-win_amd64.whl", hash = "sha256:6567d2bb951e21232c2f660c24cf3470bb96de56cdcb3f071a83feeaff8a2772"}, - {file = "rpds_py-0.27.1-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:c918c65ec2e42c2a78d19f18c553d77319119bf43aa9e2edf7fb78d624355527"}, - {file = "rpds_py-0.27.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:1fea2b1a922c47c51fd07d656324531adc787e415c8b116530a1d29c0516c62d"}, - {file = "rpds_py-0.27.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bbf94c58e8e0cd6b6f38d8de67acae41b3a515c26169366ab58bdca4a6883bb8"}, - {file = "rpds_py-0.27.1-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c2a8fed130ce946d5c585eddc7c8eeef0051f58ac80a8ee43bd17835c144c2cc"}, - {file = "rpds_py-0.27.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:037a2361db72ee98d829bc2c5b7cc55598ae0a5e0ec1823a56ea99374cfd73c1"}, - {file = "rpds_py-0.27.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5281ed1cc1d49882f9997981c88df1a22e140ab41df19071222f7e5fc4e72125"}, - {file = "rpds_py-0.27.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2fd50659a069c15eef8aa3d64bbef0d69fd27bb4a50c9ab4f17f83a16cbf8905"}, - {file = "rpds_py-0.27.1-cp39-cp39-manylinux_2_31_riscv64.whl", hash = "sha256:c4b676c4ae3921649a15d28ed10025548e9b561ded473aa413af749503c6737e"}, - {file = "rpds_py-0.27.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:079bc583a26db831a985c5257797b2b5d3affb0386e7ff886256762f82113b5e"}, - {file = "rpds_py-0.27.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:4e44099bd522cba71a2c6b97f68e19f40e7d85399de899d66cdb67b32d7cb786"}, - {file = "rpds_py-0.27.1-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:e202e6d4188e53c6661af813b46c37ca2c45e497fc558bacc1a7630ec2695aec"}, - {file = "rpds_py-0.27.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:f41f814b8eaa48768d1bb551591f6ba45f87ac76899453e8ccd41dba1289b04b"}, - {file = "rpds_py-0.27.1-cp39-cp39-win32.whl", hash = "sha256:9e71f5a087ead99563c11fdaceee83ee982fd39cf67601f4fd66cb386336ee52"}, - {file = "rpds_py-0.27.1-cp39-cp39-win_amd64.whl", hash = "sha256:71108900c9c3c8590697244b9519017a400d9ba26a36c48381b3f64743a44aab"}, - {file = "rpds_py-0.27.1-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:7ba22cb9693df986033b91ae1d7a979bc399237d45fccf875b76f62bb9e52ddf"}, - {file = "rpds_py-0.27.1-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:5b640501be9288c77738b5492b3fd3abc4ba95c50c2e41273c8a1459f08298d3"}, - {file = "rpds_py-0.27.1-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb08b65b93e0c6dd70aac7f7890a9c0938d5ec71d5cb32d45cf844fb8ae47636"}, - {file = "rpds_py-0.27.1-pp310-pypy310_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d7ff07d696a7a38152ebdb8212ca9e5baab56656749f3d6004b34ab726b550b8"}, - {file = "rpds_py-0.27.1-pp310-pypy310_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fb7c72262deae25366e3b6c0c0ba46007967aea15d1eea746e44ddba8ec58dcc"}, - {file = "rpds_py-0.27.1-pp310-pypy310_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7b002cab05d6339716b03a4a3a2ce26737f6231d7b523f339fa061d53368c9d8"}, - {file = "rpds_py-0.27.1-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:23f6b69d1c26c4704fec01311963a41d7de3ee0570a84ebde4d544e5a1859ffc"}, - {file = "rpds_py-0.27.1-pp310-pypy310_pp73-manylinux_2_31_riscv64.whl", hash = "sha256:530064db9146b247351f2a0250b8f00b289accea4596a033e94be2389977de71"}, - {file = "rpds_py-0.27.1-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:7b90b0496570bd6b0321724a330d8b545827c4df2034b6ddfc5f5275f55da2ad"}, - {file = "rpds_py-0.27.1-pp310-pypy310_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:879b0e14a2da6a1102a3fc8af580fc1ead37e6d6692a781bd8c83da37429b5ab"}, - {file = "rpds_py-0.27.1-pp310-pypy310_pp73-musllinux_1_2_i686.whl", hash = "sha256:0d807710df3b5faa66c731afa162ea29717ab3be17bdc15f90f2d9f183da4059"}, - {file = "rpds_py-0.27.1-pp310-pypy310_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:3adc388fc3afb6540aec081fa59e6e0d3908722771aa1e37ffe22b220a436f0b"}, - {file = "rpds_py-0.27.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:c796c0c1cc68cb08b0284db4229f5af76168172670c74908fdbd4b7d7f515819"}, - {file = "rpds_py-0.27.1-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:cdfe4bb2f9fe7458b7453ad3c33e726d6d1c7c0a72960bcc23800d77384e42df"}, - {file = "rpds_py-0.27.1-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:8fabb8fd848a5f75a2324e4a84501ee3a5e3c78d8603f83475441866e60b94a3"}, - {file = "rpds_py-0.27.1-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:eda8719d598f2f7f3e0f885cba8646644b55a187762bec091fa14a2b819746a9"}, - {file = "rpds_py-0.27.1-pp311-pypy311_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3c64d07e95606ec402a0a1c511fe003873fa6af630bda59bac77fac8b4318ebc"}, - {file = "rpds_py-0.27.1-pp311-pypy311_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:93a2ed40de81bcff59aabebb626562d48332f3d028ca2036f1d23cbb52750be4"}, - {file = "rpds_py-0.27.1-pp311-pypy311_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:387ce8c44ae94e0ec50532d9cb0edce17311024c9794eb196b90e1058aadeb66"}, - {file = "rpds_py-0.27.1-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aaf94f812c95b5e60ebaf8bfb1898a7d7cb9c1af5744d4a67fa47796e0465d4e"}, - {file = "rpds_py-0.27.1-pp311-pypy311_pp73-manylinux_2_31_riscv64.whl", hash = "sha256:4848ca84d6ded9b58e474dfdbad4b8bfb450344c0551ddc8d958bf4b36aa837c"}, - {file = "rpds_py-0.27.1-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2bde09cbcf2248b73c7c323be49b280180ff39fadcfe04e7b6f54a678d02a7cf"}, - {file = "rpds_py-0.27.1-pp311-pypy311_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:94c44ee01fd21c9058f124d2d4f0c9dc7634bec93cd4b38eefc385dabe71acbf"}, - {file = "rpds_py-0.27.1-pp311-pypy311_pp73-musllinux_1_2_i686.whl", hash = "sha256:df8b74962e35c9249425d90144e721eed198e6555a0e22a563d29fe4486b51f6"}, - {file = "rpds_py-0.27.1-pp311-pypy311_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:dc23e6820e3b40847e2f4a7726462ba0cf53089512abe9ee16318c366494c17a"}, - {file = "rpds_py-0.27.1-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:aa8933159edc50be265ed22b401125c9eebff3171f570258854dbce3ecd55475"}, - {file = "rpds_py-0.27.1-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:a50431bf02583e21bf273c71b89d710e7a710ad5e39c725b14e685610555926f"}, - {file = "rpds_py-0.27.1-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:78af06ddc7fe5cc0e967085a9115accee665fb912c22a3f54bad70cc65b05fe6"}, - {file = "rpds_py-0.27.1-pp39-pypy39_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:70d0738ef8fee13c003b100c2fbd667ec4f133468109b3472d249231108283a3"}, - {file = "rpds_py-0.27.1-pp39-pypy39_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e2f6fd8a1cea5bbe599b6e78a6e5ee08db434fc8ffea51ff201c8765679698b3"}, - {file = "rpds_py-0.27.1-pp39-pypy39_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8177002868d1426305bb5de1e138161c2ec9eb2d939be38291d7c431c4712df8"}, - {file = "rpds_py-0.27.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:008b839781d6c9bf3b6a8984d1d8e56f0ec46dc56df61fd669c49b58ae800400"}, - {file = "rpds_py-0.27.1-pp39-pypy39_pp73-manylinux_2_31_riscv64.whl", hash = "sha256:a55b9132bb1ade6c734ddd2759c8dc132aa63687d259e725221f106b83a0e485"}, - {file = "rpds_py-0.27.1-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a46fdec0083a26415f11d5f236b79fa1291c32aaa4a17684d82f7017a1f818b1"}, - {file = "rpds_py-0.27.1-pp39-pypy39_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:8a63b640a7845f2bdd232eb0d0a4a2dd939bcdd6c57e6bb134526487f3160ec5"}, - {file = "rpds_py-0.27.1-pp39-pypy39_pp73-musllinux_1_2_i686.whl", hash = "sha256:7e32721e5d4922deaaf963469d795d5bde6093207c52fec719bd22e5d1bedbc4"}, - {file = "rpds_py-0.27.1-pp39-pypy39_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:2c426b99a068601b5f4623573df7a7c3d72e87533a2dd2253353a03e7502566c"}, - {file = "rpds_py-0.27.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:4fc9b7fe29478824361ead6e14e4f5aed570d477e06088826537e202d25fe859"}, - {file = "rpds_py-0.27.1.tar.gz", hash = "sha256:26a1c73171d10b7acccbded82bf6a586ab8203601e565badc74bbbf8bc5a10f8"}, + {file = "rpds_py-0.18.1-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:d31dea506d718693b6b2cffc0648a8929bdc51c70a311b2770f09611caa10d53"}, + {file = "rpds_py-0.18.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:732672fbc449bab754e0b15356c077cc31566df874964d4801ab14f71951ea80"}, + {file = "rpds_py-0.18.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4a98a1f0552b5f227a3d6422dbd61bc6f30db170939bd87ed14f3c339aa6c7c9"}, + {file = "rpds_py-0.18.1-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7f1944ce16401aad1e3f7d312247b3d5de7981f634dc9dfe90da72b87d37887d"}, + {file = "rpds_py-0.18.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:38e14fb4e370885c4ecd734f093a2225ee52dc384b86fa55fe3f74638b2cfb09"}, + {file = "rpds_py-0.18.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:08d74b184f9ab6289b87b19fe6a6d1a97fbfea84b8a3e745e87a5de3029bf944"}, + {file = "rpds_py-0.18.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d70129cef4a8d979caa37e7fe957202e7eee8ea02c5e16455bc9808a59c6b2f0"}, + {file = "rpds_py-0.18.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ce0bb20e3a11bd04461324a6a798af34d503f8d6f1aa3d2aa8901ceaf039176d"}, + {file = "rpds_py-0.18.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:81c5196a790032e0fc2464c0b4ab95f8610f96f1f2fa3d4deacce6a79852da60"}, + {file = "rpds_py-0.18.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:f3027be483868c99b4985fda802a57a67fdf30c5d9a50338d9db646d590198da"}, + {file = "rpds_py-0.18.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:d44607f98caa2961bab4fa3c4309724b185b464cdc3ba6f3d7340bac3ec97cc1"}, + {file = "rpds_py-0.18.1-cp310-none-win32.whl", hash = "sha256:c273e795e7a0f1fddd46e1e3cb8be15634c29ae8ff31c196debb620e1edb9333"}, + {file = "rpds_py-0.18.1-cp310-none-win_amd64.whl", hash = "sha256:8352f48d511de5f973e4f2f9412736d7dea76c69faa6d36bcf885b50c758ab9a"}, + {file = "rpds_py-0.18.1-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:6b5ff7e1d63a8281654b5e2896d7f08799378e594f09cf3674e832ecaf396ce8"}, + {file = "rpds_py-0.18.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8927638a4d4137a289e41d0fd631551e89fa346d6dbcfc31ad627557d03ceb6d"}, + {file = "rpds_py-0.18.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:154bf5c93d79558b44e5b50cc354aa0459e518e83677791e6adb0b039b7aa6a7"}, + {file = "rpds_py-0.18.1-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:07f2139741e5deb2c5154a7b9629bc5aa48c766b643c1a6750d16f865a82c5fc"}, + {file = "rpds_py-0.18.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8c7672e9fba7425f79019db9945b16e308ed8bc89348c23d955c8c0540da0a07"}, + {file = "rpds_py-0.18.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:489bdfe1abd0406eba6b3bb4fdc87c7fa40f1031de073d0cfb744634cc8fa261"}, + {file = "rpds_py-0.18.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3c20f05e8e3d4fc76875fc9cb8cf24b90a63f5a1b4c5b9273f0e8225e169b100"}, + {file = "rpds_py-0.18.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:967342e045564cef76dfcf1edb700b1e20838d83b1aa02ab313e6a497cf923b8"}, + {file = "rpds_py-0.18.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:2cc7c1a47f3a63282ab0f422d90ddac4aa3034e39fc66a559ab93041e6505da7"}, + {file = "rpds_py-0.18.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:f7afbfee1157e0f9376c00bb232e80a60e59ed716e3211a80cb8506550671e6e"}, + {file = "rpds_py-0.18.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:9e6934d70dc50f9f8ea47081ceafdec09245fd9f6032669c3b45705dea096b88"}, + {file = "rpds_py-0.18.1-cp311-none-win32.whl", hash = "sha256:c69882964516dc143083d3795cb508e806b09fc3800fd0d4cddc1df6c36e76bb"}, + {file = "rpds_py-0.18.1-cp311-none-win_amd64.whl", hash = "sha256:70a838f7754483bcdc830444952fd89645569e7452e3226de4a613a4c1793fb2"}, + {file = "rpds_py-0.18.1-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:3dd3cd86e1db5aadd334e011eba4e29d37a104b403e8ca24dcd6703c68ca55b3"}, + {file = "rpds_py-0.18.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:05f3d615099bd9b13ecf2fc9cf2d839ad3f20239c678f461c753e93755d629ee"}, + {file = "rpds_py-0.18.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:35b2b771b13eee8729a5049c976197ff58a27a3829c018a04341bcf1ae409b2b"}, + {file = "rpds_py-0.18.1-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ee17cd26b97d537af8f33635ef38be873073d516fd425e80559f4585a7b90c43"}, + {file = "rpds_py-0.18.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b646bf655b135ccf4522ed43d6902af37d3f5dbcf0da66c769a2b3938b9d8184"}, + {file = "rpds_py-0.18.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:19ba472b9606c36716062c023afa2484d1e4220548751bda14f725a7de17b4f6"}, + {file = "rpds_py-0.18.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6e30ac5e329098903262dc5bdd7e2086e0256aa762cc8b744f9e7bf2a427d3f8"}, + {file = "rpds_py-0.18.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d58ad6317d188c43750cb76e9deacf6051d0f884d87dc6518e0280438648a9ac"}, + {file = "rpds_py-0.18.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e1735502458621921cee039c47318cb90b51d532c2766593be6207eec53e5c4c"}, + {file = "rpds_py-0.18.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:f5bab211605d91db0e2995a17b5c6ee5edec1270e46223e513eaa20da20076ac"}, + {file = "rpds_py-0.18.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:2fc24a329a717f9e2448f8cd1f960f9dac4e45b6224d60734edeb67499bab03a"}, + {file = "rpds_py-0.18.1-cp312-none-win32.whl", hash = "sha256:1805d5901779662d599d0e2e4159d8a82c0b05faa86ef9222bf974572286b2b6"}, + {file = "rpds_py-0.18.1-cp312-none-win_amd64.whl", hash = "sha256:720edcb916df872d80f80a1cc5ea9058300b97721efda8651efcd938a9c70a72"}, + {file = "rpds_py-0.18.1-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:c827576e2fa017a081346dce87d532a5310241648eb3700af9a571a6e9fc7e74"}, + {file = "rpds_py-0.18.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:aa3679e751408d75a0b4d8d26d6647b6d9326f5e35c00a7ccd82b78ef64f65f8"}, + {file = "rpds_py-0.18.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0abeee75434e2ee2d142d650d1e54ac1f8b01e6e6abdde8ffd6eeac6e9c38e20"}, + {file = "rpds_py-0.18.1-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ed402d6153c5d519a0faf1bb69898e97fb31613b49da27a84a13935ea9164dfc"}, + {file = "rpds_py-0.18.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:338dee44b0cef8b70fd2ef54b4e09bb1b97fc6c3a58fea5db6cc083fd9fc2724"}, + {file = "rpds_py-0.18.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7750569d9526199c5b97e5a9f8d96a13300950d910cf04a861d96f4273d5b104"}, + {file = "rpds_py-0.18.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:607345bd5912aacc0c5a63d45a1f73fef29e697884f7e861094e443187c02be5"}, + {file = "rpds_py-0.18.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:207c82978115baa1fd8d706d720b4a4d2b0913df1c78c85ba73fe6c5804505f0"}, + {file = "rpds_py-0.18.1-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:6d1e42d2735d437e7e80bab4d78eb2e459af48c0a46e686ea35f690b93db792d"}, + {file = "rpds_py-0.18.1-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:5463c47c08630007dc0fe99fb480ea4f34a89712410592380425a9b4e1611d8e"}, + {file = "rpds_py-0.18.1-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:06d218939e1bf2ca50e6b0ec700ffe755e5216a8230ab3e87c059ebb4ea06afc"}, + {file = "rpds_py-0.18.1-cp38-none-win32.whl", hash = "sha256:312fe69b4fe1ffbe76520a7676b1e5ac06ddf7826d764cc10265c3b53f96dbe9"}, + {file = "rpds_py-0.18.1-cp38-none-win_amd64.whl", hash = "sha256:9437ca26784120a279f3137ee080b0e717012c42921eb07861b412340f85bae2"}, + {file = "rpds_py-0.18.1-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:19e515b78c3fc1039dd7da0a33c28c3154458f947f4dc198d3c72db2b6b5dc93"}, + {file = "rpds_py-0.18.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a7b28c5b066bca9a4eb4e2f2663012debe680f097979d880657f00e1c30875a0"}, + {file = "rpds_py-0.18.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:673fdbbf668dd958eff750e500495ef3f611e2ecc209464f661bc82e9838991e"}, + {file = "rpds_py-0.18.1-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d960de62227635d2e61068f42a6cb6aae91a7fe00fca0e3aeed17667c8a34611"}, + {file = "rpds_py-0.18.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:352a88dc7892f1da66b6027af06a2e7e5d53fe05924cc2cfc56495b586a10b72"}, + {file = "rpds_py-0.18.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4e0ee01ad8260184db21468a6e1c37afa0529acc12c3a697ee498d3c2c4dcaf3"}, + {file = "rpds_py-0.18.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e4c39ad2f512b4041343ea3c7894339e4ca7839ac38ca83d68a832fc8b3748ab"}, + {file = "rpds_py-0.18.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:aaa71ee43a703c321906813bb252f69524f02aa05bf4eec85f0c41d5d62d0f4c"}, + {file = "rpds_py-0.18.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:6cd8098517c64a85e790657e7b1e509b9fe07487fd358e19431cb120f7d96338"}, + {file = "rpds_py-0.18.1-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:4adec039b8e2928983f885c53b7cc4cda8965b62b6596501a0308d2703f8af1b"}, + {file = "rpds_py-0.18.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:32b7daaa3e9389db3695964ce8e566e3413b0c43e3394c05e4b243a4cd7bef26"}, + {file = "rpds_py-0.18.1-cp39-none-win32.whl", hash = "sha256:2625f03b105328729f9450c8badda34d5243231eef6535f80064d57035738360"}, + {file = "rpds_py-0.18.1-cp39-none-win_amd64.whl", hash = "sha256:bf18932d0003c8c4d51a39f244231986ab23ee057d235a12b2684ea26a353590"}, + {file = "rpds_py-0.18.1-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:cbfbea39ba64f5e53ae2915de36f130588bba71245b418060ec3330ebf85678e"}, + {file = "rpds_py-0.18.1-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:a3d456ff2a6a4d2adcdf3c1c960a36f4fd2fec6e3b4902a42a384d17cf4e7a65"}, + {file = "rpds_py-0.18.1-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7700936ef9d006b7ef605dc53aa364da2de5a3aa65516a1f3ce73bf82ecfc7ae"}, + {file = "rpds_py-0.18.1-pp310-pypy310_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:51584acc5916212e1bf45edd17f3a6b05fe0cbb40482d25e619f824dccb679de"}, + {file = "rpds_py-0.18.1-pp310-pypy310_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:942695a206a58d2575033ff1e42b12b2aece98d6003c6bc739fbf33d1773b12f"}, + {file = "rpds_py-0.18.1-pp310-pypy310_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b906b5f58892813e5ba5c6056d6a5ad08f358ba49f046d910ad992196ea61397"}, + {file = "rpds_py-0.18.1-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f6f8e3fecca256fefc91bb6765a693d96692459d7d4c644660a9fff32e517843"}, + {file = "rpds_py-0.18.1-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:7732770412bab81c5a9f6d20aeb60ae943a9b36dcd990d876a773526468e7163"}, + {file = "rpds_py-0.18.1-pp310-pypy310_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:bd1105b50ede37461c1d51b9698c4f4be6e13e69a908ab7751e3807985fc0346"}, + {file = "rpds_py-0.18.1-pp310-pypy310_pp73-musllinux_1_2_i686.whl", hash = "sha256:618916f5535784960f3ecf8111581f4ad31d347c3de66d02e728de460a46303c"}, + {file = "rpds_py-0.18.1-pp310-pypy310_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:17c6d2155e2423f7e79e3bb18151c686d40db42d8645e7977442170c360194d4"}, + {file = "rpds_py-0.18.1-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:6c4c4c3f878df21faf5fac86eda32671c27889e13570645a9eea0a1abdd50922"}, + {file = "rpds_py-0.18.1-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:fab6ce90574645a0d6c58890e9bcaac8d94dff54fb51c69e5522a7358b80ab64"}, + {file = "rpds_py-0.18.1-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:531796fb842b53f2695e94dc338929e9f9dbf473b64710c28af5a160b2a8927d"}, + {file = "rpds_py-0.18.1-pp38-pypy38_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:740884bc62a5e2bbb31e584f5d23b32320fd75d79f916f15a788d527a5e83644"}, + {file = "rpds_py-0.18.1-pp38-pypy38_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:998125738de0158f088aef3cb264a34251908dd2e5d9966774fdab7402edfab7"}, + {file = "rpds_py-0.18.1-pp38-pypy38_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e2be6e9dd4111d5b31ba3b74d17da54a8319d8168890fbaea4b9e5c3de630ae5"}, + {file = "rpds_py-0.18.1-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d0cee71bc618cd93716f3c1bf56653740d2d13ddbd47673efa8bf41435a60daa"}, + {file = "rpds_py-0.18.1-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2c3caec4ec5cd1d18e5dd6ae5194d24ed12785212a90b37f5f7f06b8bedd7139"}, + {file = "rpds_py-0.18.1-pp38-pypy38_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:27bba383e8c5231cd559affe169ca0b96ec78d39909ffd817f28b166d7ddd4d8"}, + {file = "rpds_py-0.18.1-pp38-pypy38_pp73-musllinux_1_2_i686.whl", hash = "sha256:a888e8bdb45916234b99da2d859566f1e8a1d2275a801bb8e4a9644e3c7e7909"}, + {file = "rpds_py-0.18.1-pp38-pypy38_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:6031b25fb1b06327b43d841f33842b383beba399884f8228a6bb3df3088485ff"}, + {file = "rpds_py-0.18.1-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:48c2faaa8adfacefcbfdb5f2e2e7bdad081e5ace8d182e5f4ade971f128e6bb3"}, + {file = "rpds_py-0.18.1-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:d85164315bd68c0806768dc6bb0429c6f95c354f87485ee3593c4f6b14def2bd"}, + {file = "rpds_py-0.18.1-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6afd80f6c79893cfc0574956f78a0add8c76e3696f2d6a15bca2c66c415cf2d4"}, + {file = "rpds_py-0.18.1-pp39-pypy39_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:fa242ac1ff583e4ec7771141606aafc92b361cd90a05c30d93e343a0c2d82a89"}, + {file = "rpds_py-0.18.1-pp39-pypy39_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d21be4770ff4e08698e1e8e0bce06edb6ea0626e7c8f560bc08222880aca6a6f"}, + {file = "rpds_py-0.18.1-pp39-pypy39_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5c45a639e93a0c5d4b788b2613bd637468edd62f8f95ebc6fcc303d58ab3f0a8"}, + {file = "rpds_py-0.18.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:910e71711d1055b2768181efa0a17537b2622afeb0424116619817007f8a2b10"}, + {file = "rpds_py-0.18.1-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b9bb1f182a97880f6078283b3505a707057c42bf55d8fca604f70dedfdc0772a"}, + {file = "rpds_py-0.18.1-pp39-pypy39_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:1d54f74f40b1f7aaa595a02ff42ef38ca654b1469bef7d52867da474243cc633"}, + {file = "rpds_py-0.18.1-pp39-pypy39_pp73-musllinux_1_2_i686.whl", hash = "sha256:8d2e182c9ee01135e11e9676e9a62dfad791a7a467738f06726872374a83db49"}, + {file = "rpds_py-0.18.1-pp39-pypy39_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:636a15acc588f70fda1661234761f9ed9ad79ebed3f2125d44be0862708b666e"}, + {file = "rpds_py-0.18.1.tar.gz", hash = "sha256:dc48b479d540770c811fbd1eb9ba2bb66951863e448efec2e2c102625328e92f"}, ] [[package]] @@ -1131,6 +1255,7 @@ version = "0.17.21" description = "ruamel.yaml is a YAML parser/emitter that supports roundtrip preservation of comments, seq/map flow style, and map key order" optional = false python-versions = ">=3" +groups = ["main"] files = [ {file = "ruamel.yaml-0.17.21-py3-none-any.whl", hash = "sha256:742b35d3d665023981bd6d16b3d24248ce5df75fdb4e2924e93a05c1f8b61ca7"}, {file = "ruamel.yaml-0.17.21.tar.gz", hash = "sha256:8b7ce697a2f212752a35c1ac414471dc16c424c9573be4926b56ff3f5d23b7af"}, @@ -1149,6 +1274,8 @@ version = "0.2.14" description = "C version of reader, parser and emitter for ruamel.yaml derived from libyaml" optional = false python-versions = ">=3.9" +groups = ["main"] +markers = "platform_python_implementation == \"CPython\" and python_version == \"3.10\"" files = [ {file = "ruamel.yaml.clib-0.2.14-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:f8b2acb0ffdd2ce8208accbec2dca4a06937d556fdcaefd6473ba1b5daa7e3c4"}, {file = "ruamel.yaml.clib-0.2.14-cp310-cp310-macosx_13_0_arm64.whl", hash = "sha256:aef953f3b8bd0b50bd52a2e52fb54a6a2171a1889d8dea4a5959d46c6624c451"}, @@ -1205,6 +1332,8 @@ files = [ {file = "ruamel.yaml.clib-0.2.14-cp39-cp39-win32.whl", hash = "sha256:6d5472f63a31b042aadf5ed28dd3ef0523da49ac17f0463e10fda9c4a2773352"}, {file = "ruamel.yaml.clib-0.2.14-cp39-cp39-win_amd64.whl", hash = "sha256:8dd3c2cc49caa7a8d64b67146462aed6723a0495e44bf0aa0a2e94beaa8432f6"}, {file = "ruamel.yaml.clib-0.2.14.tar.gz", hash = "sha256:803f5044b13602d58ea378576dd75aa759f52116a0232608e8fdada4da33752e"}, + {file = "ruamel_yaml_clib-0.2.14-cp314-cp314-win32.whl", hash = "sha256:9b4104bf43ca0cd4e6f738cb86326a3b2f6eef00f417bd1e7efb7bdffe74c539"}, + {file = "ruamel_yaml_clib-0.2.14-cp314-cp314-win_amd64.whl", hash = "sha256:13997d7d354a9890ea1ec5937a219817464e5cc344805b37671562a401ca3008"}, ] [[package]] @@ -1213,6 +1342,7 @@ version = "69.5.1" description = "Easily download, build, install, upgrade, and uninstall Python packages" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "setuptools-69.5.1-py3-none-any.whl", hash = "sha256:c636ac361bc47580504644275c9ad802c50415c7522212252c033bd15f301f32"}, {file = "setuptools-69.5.1.tar.gz", hash = "sha256:6c1fccdac05a97e598fb0ae3bbed5904ccb317337a51139dcd51453611bbb987"}, @@ -1220,7 +1350,7 @@ files = [ [package.extras] docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] -testing = ["build[virtualenv]", "filelock (>=3.4.0)", "importlib-metadata", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "mypy (==1.9)", "packaging (>=23.2)", "pip (>=19.1)", "pytest (>=6,!=8.1.1)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-timeout", "pytest-xdist (>=3)", "tomli", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "importlib-metadata", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21) ; python_version >= \"3.9\" and sys_platform != \"cygwin\"", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "mypy (==1.9)", "packaging (>=23.2)", "pip (>=19.1)", "pytest (>=6,!=8.1.1)", "pytest-checkdocs (>=2.4)", "pytest-cov ; platform_python_implementation != \"PyPy\"", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy", "pytest-perf ; sys_platform != \"cygwin\"", "pytest-ruff (>=0.2.1) ; sys_platform != \"cygwin\"", "pytest-timeout", "pytest-xdist (>=3)", "tomli", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.2)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] [[package]] @@ -1229,6 +1359,7 @@ version = "1.17.0" description = "Python 2 and 3 compatibility utilities" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +groups = ["main"] files = [ {file = "six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274"}, {file = "six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81"}, @@ -1240,6 +1371,8 @@ version = "2.2.1" description = "A lil' TOML parser" optional = false python-versions = ">=3.8" +groups = ["main", "dev"] +markers = "python_version == \"3.10\"" files = [ {file = "tomli-2.2.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:678e4fa69e4575eb77d103de3df8a895e1591b48e740211bd1067378c69e8249"}, {file = "tomli-2.2.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:023aa114dd824ade0100497eb2318602af309e5a55595f76b626d6d9f3b7b0a6"}, @@ -1277,45 +1410,48 @@ files = [ [[package]] name = "tqdm" -version = "4.67.1" +version = "4.66.4" description = "Fast, Extensible Progress Meter" optional = false python-versions = ">=3.7" +groups = ["main"] files = [ - {file = "tqdm-4.67.1-py3-none-any.whl", hash = "sha256:26445eca388f82e72884e0d580d5464cd801a3ea01e63e5601bdff9ba6a48de2"}, - {file = "tqdm-4.67.1.tar.gz", hash = "sha256:f8aef9c52c08c13a65f30ea34f4e5aac3fd1a34959879d7e59e63027286627f2"}, + {file = "tqdm-4.66.4-py3-none-any.whl", hash = "sha256:b75ca56b413b030bc3f00af51fd2c1a1a5eac6a0c1cca83cbb37a5c52abce644"}, + {file = "tqdm-4.66.4.tar.gz", hash = "sha256:e4d936c9de8727928f3be6079590e97d9abfe8d39a590be678eb5919ffc186bb"}, ] [package.dependencies] colorama = {version = "*", markers = "platform_system == \"Windows\""} [package.extras] -dev = ["nbval", "pytest (>=6)", "pytest-asyncio (>=0.24)", "pytest-cov", "pytest-timeout"] -discord = ["requests"] +dev = ["pytest (>=6)", "pytest-cov", "pytest-timeout", "pytest-xdist"] notebook = ["ipywidgets (>=6)"] slack = ["slack-sdk"] telegram = ["requests"] [[package]] name = "typing-extensions" -version = "4.15.0" -description = "Backported and Experimental Type Hints for Python 3.9+" +version = "4.12.2" +description = "Backported and Experimental Type Hints for Python 3.8+" optional = false -python-versions = ">=3.9" +python-versions = ">=3.8" +groups = ["main", "dev"] files = [ - {file = "typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548"}, - {file = "typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466"}, + {file = "typing_extensions-4.12.2-py3-none-any.whl", hash = "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d"}, + {file = "typing_extensions-4.12.2.tar.gz", hash = "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8"}, ] +markers = {dev = "python_version == \"3.10\""} [[package]] name = "tzdata" -version = "2025.2" +version = "2025.3" description = "Provider of IANA time zone data" optional = false python-versions = ">=2" +groups = ["main"] files = [ - {file = "tzdata-2025.2-py2.py3-none-any.whl", hash = "sha256:1a403fada01ff9221ca8044d701868fa132215d84beb92242d9acd2147f667a8"}, - {file = "tzdata-2025.2.tar.gz", hash = "sha256:b60a638fcc0daffadf82fe0f57e53d06bdec2f36c4df66280ae79bce6bd6f2b9"}, + {file = "tzdata-2025.3-py2.py3-none-any.whl", hash = "sha256:06a47e5700f3081aab02b2e513160914ff0694bce9947d6b76ebd6bf57cfc5d1"}, + {file = "tzdata-2025.3.tar.gz", hash = "sha256:de39c2ca5dc7b0344f2eba86f49d614019d29f060fc4ebc8a417896a620b56a7"}, ] [[package]] @@ -1324,13 +1460,14 @@ version = "2.5.0" description = "HTTP library with thread-safe connection pooling, file post, and more." optional = false python-versions = ">=3.9" +groups = ["main"] files = [ {file = "urllib3-2.5.0-py3-none-any.whl", hash = "sha256:e6b01673c0fa6a13e374b50871808eb3bf7046c4b125b216f6bf1cc604cff0dc"}, {file = "urllib3-2.5.0.tar.gz", hash = "sha256:3fc47733c7e419d4bc3f6b3dc2b4f890bb743906a30d56ba4a5bfa4bbff92760"}, ] [package.extras] -brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +brotli = ["brotli (>=1.0.9) ; platform_python_implementation == \"CPython\"", "brotlicffi (>=0.8.0) ; platform_python_implementation != \"CPython\""] h2 = ["h2 (>=4,<5)"] socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] zstd = ["zstandard (>=0.18.0)"] @@ -1341,6 +1478,7 @@ version = "0.2.14" description = "Measures the displayed width of unicode strings in a terminal" optional = false python-versions = ">=3.6" +groups = ["main"] files = [ {file = "wcwidth-0.2.14-py2.py3-none-any.whl", hash = "sha256:a7bb560c8aee30f9957e5f9895805edd20602f2d7f720186dfd906e82b4982e1"}, {file = "wcwidth-0.2.14.tar.gz", hash = "sha256:4d478375d31bc5395a3c55c40ccdf3354688364cd61c4f6adacaa9215d0b3605"}, @@ -1348,21 +1486,22 @@ files = [ [[package]] name = "z3-solver" -version = "4.15.3.0" +version = "4.13.3.0" description = "an efficient SMT solver library" optional = false python-versions = "*" +groups = ["main"] files = [ - {file = "z3_solver-4.15.3.0-py3-none-macosx_13_0_arm64.whl", hash = "sha256:65335aab295ded7c0ce27c85556067087a87052389ff160777d1a1d48ef0d74f"}, - {file = "z3_solver-4.15.3.0-py3-none-macosx_13_0_x86_64.whl", hash = "sha256:3e62e93adff2def3537ff1ca67c3d58a6ca6d1944e0b5e774f88627b199d50e7"}, - {file = "z3_solver-4.15.3.0-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a9afd9ceb290482097474d43f08415bcc1874f433189d1449f6c1508e9c68384"}, - {file = "z3_solver-4.15.3.0-py3-none-manylinux_2_34_aarch64.whl", hash = "sha256:f61ef44552489077eedd7e6d9bed52ef1875decf86d66027742099a2703b1c77"}, - {file = "z3_solver-4.15.3.0-py3-none-win32.whl", hash = "sha256:0c603f6bad7423d6411adda6af55030b725e3d30f54ea91b714abcedd73b848a"}, - {file = "z3_solver-4.15.3.0-py3-none-win_amd64.whl", hash = "sha256:06abdf6c36f97c463aea827533504fd59476d015a65cf170a88bd6a53ba13ab5"}, - {file = "z3_solver-4.15.3.0.tar.gz", hash = "sha256:78f69aebda5519bfd8af146a129f36cf4721a3c2667e80d9fe35cc9bb4d214a6"}, + {file = "z3_solver-4.13.3.0-py3-none-macosx_13_0_arm64.whl", hash = "sha256:cae621cb47ebcf055f6a27285343d5c932f4c282b15c5d2840327e73e15a86a4"}, + {file = "z3_solver-4.13.3.0-py3-none-macosx_13_0_x86_64.whl", hash = "sha256:7900fbd1917164c938a20bea7845f7b95fcb431d0ade474d408f979196bccb8f"}, + {file = "z3_solver-4.13.3.0-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:794843e4946ade1561e40a75ffc1163b45d36b493fd6cc269ad1d6a65bddb8e5"}, + {file = "z3_solver-4.13.3.0-py3-none-manylinux_2_34_aarch64.whl", hash = "sha256:ab5057cb1f4680406a232d2c1d968daaf69fae10685baa0887b85ca8e938a5cf"}, + {file = "z3_solver-4.13.3.0-py3-none-win32.whl", hash = "sha256:ba465489e9ab609f1cf0f232cbc102165b89a507347a6093c2ac3224cf161aa3"}, + {file = "z3_solver-4.13.3.0-py3-none-win_amd64.whl", hash = "sha256:d55e4793fa48230af766c7f6f7f033198b2d9df150d89ec65b07fd7b87998897"}, + {file = "z3_solver-4.13.3.0.tar.gz", hash = "sha256:4c27466455bac65d3c512f4bb4841ac79e05e121d3f98ddc99ac27ab4bc27e05"}, ] [metadata] -lock-version = "2.0" +lock-version = "2.1" python-versions = "^3.10" -content-hash = "0d79bba23274cf2a3d1f114f4dac5d9482ee15696177243ac3985151f897b30e" +content-hash = "0ec15e3238ad65651f6604e284a6108f23177277d7a6a43b392d5be613e672a1" diff --git a/pyproject.toml b/pyproject.toml index e41dcd63..0579e56c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "glitch" -version = "1.0.1" +version = "2.0.0" description = "A tool to analyze IaC scripts" authors = ["Nuno Saavedra "] license = "GPL-3.0" @@ -14,10 +14,7 @@ click = "8.1.7" prettytable = "3.6.0" pandas = "2.3.3" configparser = "5.3.0" -puppetparser = "0.2.4" -Jinja2 = "3.1.2" -glitch-python-hcl2 = "0.1.4" -dockerfile-parse = "2.0.0" +Jinja2 = { git = "https://github.com/Nfsaavedra/jinja", branch = "tok_loc" } bashlex = "0.18" requests = "^2.31.0" z3-solver = "^4.12.4.0" @@ -25,6 +22,11 @@ nltk = "^3.8.1" jsonschema = "^4.21.1" setuptools = "^69.5.1" tqdm = "^4.66.2" +puppetparser = "0.2.14" +python-hcl2 = {git = "https://github.com/Nfsaavedra/python-hcl2.git", rev = "feature/hcl2-reverse-transformer"} +typing-extensions = "^4.12.2" +pyright = "1.1.408" +black = "25.9.0" [tool.poetry.group.dev.dependencies] pytest = "7.3.1" diff --git a/scripts/README.md b/scripts/README.md index 69dfbb47..8b978ee6 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -1,6 +1,5 @@ # Helper Scripts -In this folder we find scripts used for the extraction of data necessary for two code smells: - - Uso of non-official images +In this folder we find scripts used for the extraction of data necessary for code smells: - Use of obsolete/deprecated commands or functions ### Requirements @@ -10,11 +9,6 @@ To install them run: pip install -r requirements.txt ``` -### Use of non-official images -This code smell requires a list of official Docker images, this list can be generated -with `docker_images_scraper.py`, this script uses the docker hub API to fetch the official -images and generate a text file with an image per line. - ### Use of obsolete/deprecated commands or functions This code smell requires a list of deprecated/obsolete commands and functions, this list can be generated with `obsolete_commands_scraper.py`, this script scrapes the book diff --git a/scripts/docker_images_scraper.py b/scripts/docker_images_scraper.py deleted file mode 100644 index 4e006df3..00000000 --- a/scripts/docker_images_scraper.py +++ /dev/null @@ -1,15 +0,0 @@ -import requests - -next_url = "https://hub.docker.com/api/content/v1/products/search?image_filter=official&page=1&page_size=100&q=&type=image" -headers = {"Accept": "application/json", "Search-Version": "v3"} - -images_list = [] - -while next_url: - res = requests.get(next_url, headers=headers).json() - next_url = res["next"] - images = [i["name"] for i in res["summaries"]] - images_list += images - -with open("official_images", "w") as f: - f.write("\n".join(images_list)) diff --git a/stubs/ruamel/yaml/nodes.pyi b/stubs/ruamel/yaml/nodes.pyi index 30826046..0df77e51 100644 --- a/stubs/ruamel/yaml/nodes.pyi +++ b/stubs/ruamel/yaml/nodes.pyi @@ -7,6 +7,7 @@ RecursiveTokenList = List[Union[Token, "RecursiveTokenList", None]] class Node: comment: Optional[RecursiveTokenList] value: Any + tag: str start_mark: StreamMark end_mark: StreamMark diff --git a/stubs/z3.pyi b/stubs/z3.pyi index 457e6c29..d8031fd7 100644 --- a/stubs/z3.pyi +++ b/stubs/z3.pyi @@ -1,12 +1,21 @@ -from typing import List +from typing import List, Any +class Z3Exception(Exception): ... class CheckSatResult: ... sat: CheckSatResult class Z3PPObject: ... class AstRef(Z3PPObject): ... +class AstVector(Z3PPObject): ... class FuncDeclRef(AstRef): ... +class SortRef(AstRef): ... +class DatatypeSortRef(SortRef): ... + +class Datatype: + def __init__(self, name: str, ctx: Context | None = None) -> None: ... + def declare(self, name: str, *args: List[Any]) -> None: ... + def create(self) -> DatatypeSortRef: ... class ModelRef(Z3PPObject): def __getitem__(self, idx: AstRef) -> FuncDeclRef: ... @@ -20,12 +29,19 @@ class ArithRef(ExprRef): def __ge__(self, __value: object) -> BoolRef: ... class IntNumRef(ArithRef): ... -class SeqRef(ExprRef): ... -class Context: ... + +class SeqRef(ExprRef): + def as_string(self) -> str: ... + +class Context: + def interrupt(self) -> None: ... + class Tactic: ... class Probe: ... class Solver: + ctx: Context + def __init__( self, solver: Solver | None = None, @@ -37,6 +53,10 @@ class Solver: def pop(self) -> None: ... def check(self) -> CheckSatResult: ... def model(self) -> ModelRef: ... + def assert_and_track(self, a: Z3PPObject, p: str) -> None: ... + def unsat_core(self) -> List[AstVector]: ... + def set(self, *args: Any, **keys: Any) -> None: ... + def to_smt2(self) -> str: ... def If( a: Probe | Z3PPObject, b: Z3PPObject, c: Z3PPObject, ctx: Context | None = None @@ -45,8 +65,15 @@ def StringVal(s: str, ctx: Context | None = None) -> SeqRef: ... def String(name: str, ctx: Context | None = None) -> SeqRef: ... def Int(name: str, ctx: Context | None = None) -> ArithRef: ... def IntVal(val: int, ctx: Context | None = None) -> IntNumRef: ... +def BoolVal(val: bool, ctx: Context | None = None) -> BoolRef: ... def Bool(name: str, ctx: Context | None = None) -> BoolRef: ... -def And(*args: Z3PPObject | List[Z3PPObject]) -> BoolRef: ... -def Or(*args: Z3PPObject | List[Z3PPObject]) -> BoolRef: ... +def And(*args: Z3PPObject | List[Z3PPObject] | bool) -> BoolRef: ... +def Or(*args: Z3PPObject | List[Z3PPObject] | bool) -> BoolRef: ... +def Implies( + a: Z3PPObject | bool, b: Z3PPObject | bool, ctx: Context | None = None +) -> BoolRef: ... def Not(a: Z3PPObject) -> BoolRef: ... def Sum(*args: Z3PPObject | int | List[Z3PPObject | int]) -> ArithRef: ... +def Concat(*args: Z3PPObject | List[Z3PPObject]) -> SeqRef: ... +def Const(name: str, sort: SortRef) -> ExprRef: ... +def EnumSort(name: str, *args: List[str], ctx: Context | None = None) -> SortRef: ... diff --git a/glitch/tests/design/chef/__init__.py b/tests/__init__.py similarity index 100% rename from glitch/tests/design/chef/__init__.py rename to tests/__init__.py diff --git a/tests/base_test.py b/tests/base_test.py new file mode 100644 index 00000000..fdfcbfec --- /dev/null +++ b/tests/base_test.py @@ -0,0 +1,35 @@ +import unittest + +import csv +import os +from typing import List, Dict, Union + + +class BaseTest(unittest.TestCase): + TECH = None # subclass must override + + def setUp(self) -> None: + """Skip tests if this is the base class being run directly""" + if self.TECH is None: + self.skipTest("BaseTest is abstract and should not be run directly") + + def read_lint_csv(self, path: str) -> List[Dict[str, Union[str, int, None]]]: + """Read the CSV created by lint and return errors as a list of dicts.""" + errors: List[Dict[str, Union[str, int, None]]] = [] + + if not os.path.exists(path): + raise FileNotFoundError(f"CSV file not found: {path}") + + with open(path, newline="", encoding="utf-8") as f: + reader = csv.DictReader(f) + for row in reader: + errors.append( + { + "code": row.get("ERROR"), + "line": int(row.get("LINE", 0)), + "path": row.get("PATH"), + } + ) + + os.remove(path) + return errors diff --git a/glitch/tests/cli/resources/chef_project/test.rb b/tests/cli/resources/chef_project/test.rb similarity index 100% rename from glitch/tests/cli/resources/chef_project/test.rb rename to tests/cli/resources/chef_project/test.rb diff --git a/glitch/tests/cli/test_cli.py b/tests/cli/test_cli.py similarity index 81% rename from glitch/tests/cli/test_cli.py rename to tests/cli/test_cli.py index b4cc4d68..aec16094 100644 --- a/glitch/tests/cli/test_cli.py +++ b/tests/cli/test_cli.py @@ -37,11 +37,12 @@ def test_cli_get_paths(): assert paths == {"tests/cli/resources/chef_project"} -def test_cli_analyze(): +def test_cli_lint(): with NamedTemporaryFile() as f: run = subprocess.run( [ "glitch", + "lint", "--tech", "chef", "--folder-strategy", @@ -62,23 +63,23 @@ def test_cli_analyze(): assert len(rows) == 3 assert rows[0] == [ - "tests/cli/resources/chef_project/test.rb", - "8", - "sec_def_admin", - "user:'root'", - "-", + "PATH", + "LINE", + "ERROR", + "DESCRIPTION", + "CODE", ] assert rows[1] == [ "tests/cli/resources/chef_project/test.rb", "8", - "sec_hard_secr", - "user:'root'", - "-", + "sec_def_admin", + "Admin by default - Developers should always try to give the least privileges possible. Admin privileges may indicate a security problem. (CWE-250)", + "user 'root'", ] assert rows[2] == [ "tests/cli/resources/chef_project/test.rb", "8", "sec_hard_user", - "user:'root'", - "-", + "Hard-coded user - Developers should not reveal sensitive information in the source code. (CWE-798)", + "user 'root'", ] diff --git a/glitch/tests/design/docker/__init__.py b/tests/design/__init__.py similarity index 100% rename from glitch/tests/design/docker/__init__.py rename to tests/design/__init__.py diff --git a/glitch/tests/design/puppet/__init__.py b/tests/design/ansible/__init__.py similarity index 100% rename from glitch/tests/design/puppet/__init__.py rename to tests/design/ansible/__init__.py diff --git a/glitch/tests/design/ansible/design_ansible.ini b/tests/design/ansible/design_ansible.ini similarity index 100% rename from glitch/tests/design/ansible/design_ansible.ini rename to tests/design/ansible/design_ansible.ini diff --git a/glitch/tests/design/ansible/files/avoid_comments.yml b/tests/design/ansible/files/avoid_comments.yml similarity index 100% rename from glitch/tests/design/ansible/files/avoid_comments.yml rename to tests/design/ansible/files/avoid_comments.yml diff --git a/glitch/tests/design/ansible/files/duplicate_block.yml b/tests/design/ansible/files/duplicate_block.yml similarity index 100% rename from glitch/tests/design/ansible/files/duplicate_block.yml rename to tests/design/ansible/files/duplicate_block.yml diff --git a/glitch/tests/design/ansible/files/improper_alignment.yml b/tests/design/ansible/files/improper_alignment.yml similarity index 100% rename from glitch/tests/design/ansible/files/improper_alignment.yml rename to tests/design/ansible/files/improper_alignment.yml diff --git a/glitch/tests/design/ansible/files/long_resource.yml b/tests/design/ansible/files/long_resource.yml similarity index 100% rename from glitch/tests/design/ansible/files/long_resource.yml rename to tests/design/ansible/files/long_resource.yml diff --git a/glitch/tests/design/ansible/files/long_statement.yml b/tests/design/ansible/files/long_statement.yml similarity index 100% rename from glitch/tests/design/ansible/files/long_statement.yml rename to tests/design/ansible/files/long_statement.yml diff --git a/glitch/tests/design/ansible/files/multifaceted_abstraction.yml b/tests/design/ansible/files/multifaceted_abstraction.yml similarity index 100% rename from glitch/tests/design/ansible/files/multifaceted_abstraction.yml rename to tests/design/ansible/files/multifaceted_abstraction.yml diff --git a/glitch/tests/design/ansible/files/too_many_variables.yml b/tests/design/ansible/files/too_many_variables.yml similarity index 100% rename from glitch/tests/design/ansible/files/too_many_variables.yml rename to tests/design/ansible/files/too_many_variables.yml diff --git a/glitch/tests/design/ansible/test_design.py b/tests/design/ansible/test_design.py similarity index 65% rename from glitch/tests/design/ansible/test_design.py rename to tests/design/ansible/test_design.py index 73615d42..b7daf053 100644 --- a/glitch/tests/design/ansible/test_design.py +++ b/tests/design/ansible/test_design.py @@ -1,33 +1,16 @@ -import unittest - -from glitch.analysis.design.visitor import DesignVisitor -from glitch.parsers.ansible import AnsibleParser +from tests.design.design_helper import BaseDesignTest from glitch.tech import Tech +from glitch.repr.inter import UNDEFINED_POSITION -class TestDesign(unittest.TestCase): - def __help_test(self, path, type, n_errors: int, codes, lines) -> None: - parser = AnsibleParser() - inter = parser.parse(path, type, False) - analysis = DesignVisitor(Tech.ansible) - analysis.config("tests/design/ansible/design_ansible.ini") - errors = list( - filter( - lambda e: e.code.startswith("design_") - or e.code.startswith("implementation_"), - set(analysis.check(inter)), - ) - ) - errors = sorted(errors, key=lambda e: (e.path, e.line, e.code)) - self.assertEqual(len(errors), n_errors) - for i in range(n_errors): - self.assertEqual(errors[i].code, codes[i]) - self.assertEqual(errors[i].line, lines[i]) +class TestDesign(BaseDesignTest): + TECH = Tech.ansible def test_ansible_long_statement(self) -> None: - self.__help_test( + self._help_test( "tests/design/ansible/files/long_statement.yml", "tasks", + "tests/design/ansible/design_ansible.ini", 1, ["implementation_long_statement"], [16], @@ -35,9 +18,10 @@ def test_ansible_long_statement(self) -> None: # Tabs def test_ansible_improper_alignment(self) -> None: - self.__help_test( + self._help_test( "tests/design/ansible/files/improper_alignment.yml", "tasks", + "tests/design/ansible/design_ansible.ini", 4, [ "design_multifaceted_abstraction", @@ -49,9 +33,10 @@ def test_ansible_improper_alignment(self) -> None: ) def test_ansible_duplicate_block(self) -> None: - self.__help_test( + self._help_test( "tests/design/ansible/files/duplicate_block.yml", "tasks", + "tests/design/ansible/design_ansible.ini", 4, [ "design_duplicate_block", @@ -63,9 +48,10 @@ def test_ansible_duplicate_block(self) -> None: ) def test_ansible_avoid_comments(self) -> None: - self.__help_test( + self._help_test( "tests/design/ansible/files/avoid_comments.yml", "tasks", + "tests/design/ansible/design_ansible.ini", 1, [ "design_avoid_comments", @@ -74,9 +60,10 @@ def test_ansible_avoid_comments(self) -> None: ) def test_ansible_long_resource(self) -> None: - self.__help_test( + self._help_test( "tests/design/ansible/files/long_resource.yml", "tasks", + "tests/design/ansible/design_ansible.ini", 2, [ "design_long_resource", @@ -86,9 +73,10 @@ def test_ansible_long_resource(self) -> None: ) def test_ansible_multifaceted_abstraction(self) -> None: - self.__help_test( + self._help_test( "tests/design/ansible/files/multifaceted_abstraction.yml", "tasks", + "tests/design/ansible/design_ansible.ini", 1, [ "design_multifaceted_abstraction", @@ -97,12 +85,13 @@ def test_ansible_multifaceted_abstraction(self) -> None: ) def test_ansible_too_many_variables(self) -> None: - self.__help_test( + self._help_test( "tests/design/ansible/files/too_many_variables.yml", "script", + "tests/design/ansible/design_ansible.ini", 1, [ "implementation_too_many_variables", ], - [-1], + [UNDEFINED_POSITION], ) diff --git a/glitch/tests/design/terraform/__init__.py b/tests/design/chef/__init__.py similarity index 100% rename from glitch/tests/design/terraform/__init__.py rename to tests/design/chef/__init__.py diff --git a/glitch/tests/design/chef/design_chef.ini b/tests/design/chef/design_chef.ini similarity index 100% rename from glitch/tests/design/chef/design_chef.ini rename to tests/design/chef/design_chef.ini diff --git a/glitch/tests/design/chef/files/avoid_comments.rb b/tests/design/chef/files/avoid_comments.rb similarity index 100% rename from glitch/tests/design/chef/files/avoid_comments.rb rename to tests/design/chef/files/avoid_comments.rb diff --git a/glitch/tests/design/chef/files/duplicate_block.rb b/tests/design/chef/files/duplicate_block.rb similarity index 100% rename from glitch/tests/design/chef/files/duplicate_block.rb rename to tests/design/chef/files/duplicate_block.rb diff --git a/glitch/tests/design/chef/files/improper_alignment.rb b/tests/design/chef/files/improper_alignment.rb similarity index 100% rename from glitch/tests/design/chef/files/improper_alignment.rb rename to tests/design/chef/files/improper_alignment.rb diff --git a/glitch/tests/design/chef/files/long_resource.rb b/tests/design/chef/files/long_resource.rb similarity index 100% rename from glitch/tests/design/chef/files/long_resource.rb rename to tests/design/chef/files/long_resource.rb diff --git a/glitch/tests/design/chef/files/long_statement.rb b/tests/design/chef/files/long_statement.rb similarity index 100% rename from glitch/tests/design/chef/files/long_statement.rb rename to tests/design/chef/files/long_statement.rb diff --git a/glitch/tests/design/chef/files/misplaced_attribute.rb b/tests/design/chef/files/misplaced_attribute.rb similarity index 100% rename from glitch/tests/design/chef/files/misplaced_attribute.rb rename to tests/design/chef/files/misplaced_attribute.rb diff --git a/glitch/tests/design/chef/files/multifaceted_abstraction.rb b/tests/design/chef/files/multifaceted_abstraction.rb similarity index 100% rename from glitch/tests/design/chef/files/multifaceted_abstraction.rb rename to tests/design/chef/files/multifaceted_abstraction.rb diff --git a/glitch/tests/design/chef/files/too_many_variables.rb b/tests/design/chef/files/too_many_variables.rb similarity index 95% rename from glitch/tests/design/chef/files/too_many_variables.rb rename to tests/design/chef/files/too_many_variables.rb index 2267f588..a761231c 100644 --- a/glitch/tests/design/chef/files/too_many_variables.rb +++ b/tests/design/chef/files/too_many_variables.rb @@ -12,7 +12,6 @@ else node.default['sys']['network']['interfaces'][vlan_interface] = { "vlan-raw-device" => interface, - "up" => "ifup #{bridge}" } node.default['sys']['network']['interfaces'][bridge] = { "auto" => false, diff --git a/glitch/tests/design/chef/test_design.py b/tests/design/chef/test_design.py similarity index 63% rename from glitch/tests/design/chef/test_design.py rename to tests/design/chef/test_design.py index 11af4107..cf9599c0 100644 --- a/glitch/tests/design/chef/test_design.py +++ b/tests/design/chef/test_design.py @@ -1,48 +1,36 @@ -import unittest - -from glitch.analysis.design.visitor import DesignVisitor -from glitch.parsers.chef import ChefParser +from tests.design.design_helper import BaseDesignTest from glitch.tech import Tech +from glitch.repr.inter import UNDEFINED_POSITION -class TestDesign(unittest.TestCase): - def __help_test(self, path, n_errors: int, codes, lines) -> None: - parser = ChefParser() - inter = parser.parse(path, "script", False) - analysis = DesignVisitor(Tech.chef) - analysis.config("tests/design/chef/design_chef.ini") - errors = list( - filter( - lambda e: e.code.startswith("design_") - or e.code.startswith("implementation_"), - set(analysis.check(inter)), - ) - ) - errors = sorted(errors, key=lambda e: (e.path, e.line, e.code)) - self.assertEqual(len(errors), n_errors) - for i in range(n_errors): - self.assertEqual(errors[i].code, codes[i]) - self.assertEqual(errors[i].line, lines[i]) +class TestDesign(BaseDesignTest): + TECH = Tech.chef def test_chef_long_statement(self) -> None: - self.__help_test( + self._help_test( "tests/design/chef/files/long_statement.rb", + "script", + "tests/design/chef/design_chef.ini", 1, ["implementation_long_statement"], [6], ) def test_chef_improper_alignment(self) -> None: - self.__help_test( + self._help_test( "tests/design/chef/files/improper_alignment.rb", + "script", + "tests/design/chef/design_chef.ini", 1, ["implementation_improper_alignment"], [1], ) def test_chef_duplicate_block(self) -> None: - self.__help_test( + self._help_test( "tests/design/chef/files/duplicate_block.rb", + "script", + "tests/design/chef/design_chef.ini", 4, [ "design_duplicate_block", @@ -54,8 +42,10 @@ def test_chef_duplicate_block(self) -> None: ) def test_chef_avoid_comments(self) -> None: - self.__help_test( + self._help_test( "tests/design/chef/files/avoid_comments.rb", + "script", + "tests/design/chef/design_chef.ini", 1, [ "design_avoid_comments", @@ -64,8 +54,10 @@ def test_chef_avoid_comments(self) -> None: ) def test_chef_long_resource(self) -> None: - self.__help_test( + self._help_test( "tests/design/chef/files/long_resource.rb", + "script", + "tests/design/chef/design_chef.ini", 1, [ "design_long_resource", @@ -74,8 +66,10 @@ def test_chef_long_resource(self) -> None: ) def test_chef_multifaceted_abstraction(self) -> None: - self.__help_test( + self._help_test( "tests/design/chef/files/multifaceted_abstraction.rb", + "script", + "tests/design/chef/design_chef.ini", 1, [ "design_multifaceted_abstraction", @@ -84,8 +78,10 @@ def test_chef_multifaceted_abstraction(self) -> None: ) def test_chef_misplaced_attribute(self) -> None: - self.__help_test( + self._help_test( "tests/design/chef/files/misplaced_attribute.rb", + "script", + "tests/design/chef/design_chef.ini", 1, [ "design_misplaced_attribute", @@ -94,11 +90,13 @@ def test_chef_misplaced_attribute(self) -> None: ) def test_chef_too_many_variables(self) -> None: - self.__help_test( + self._help_test( "tests/design/chef/files/too_many_variables.rb", + "script", + "tests/design/chef/design_chef.ini", 1, [ "implementation_too_many_variables", ], - [-1], + [UNDEFINED_POSITION], ) diff --git a/tests/design/design_helper.py b/tests/design/design_helper.py new file mode 100644 index 00000000..f25fb191 --- /dev/null +++ b/tests/design/design_helper.py @@ -0,0 +1,55 @@ +from glitch.__main__ import lint +from click.testing import CliRunner +from glitch.tech import Tech +from tests.base_test import BaseTest + + +class BaseDesignTest(BaseTest): + TECH = None # subclass must override + + def _help_test( + self, + path: str, + type: str, + config: str, + n_errors: int, + codes: list[str], + lines: list[int], + ) -> None: + assert self.TECH is not None, "Subclasses must define TECH" + + output_path = "tests/design/dump.csv" + runner = CliRunner() + result = runner.invoke( + lint, + [ + "--tech", + self.TECH.value[0], + "--type", + type, + "--config", + config, + "--csv", + "--smell-types", + "design", + path, + output_path, + ], + ) + if result.exception: + raise result.exception + + errors = self.read_lint_csv(output_path) + + errors = [e for e in errors if e["code"].startswith("design_") or e["code"].startswith("implementation_")] # type: ignore + + errors = sorted( + errors, key=lambda e: (e["path"] or "", e["line"], e["code"] or "") + ) + + self.assertEqual(len(errors), n_errors) + for i in range(n_errors): + if self.TECH == Tech.gha: + self.assertEqual(errors[i]["path"], path) + self.assertEqual(errors[i]["code"], codes[i]) + self.assertEqual(errors[i]["line"], lines[i]) diff --git a/glitch/tests/design/gha/files/too_many_variables.yml b/tests/design/gha/files/too_many_variables.yml similarity index 83% rename from glitch/tests/design/gha/files/too_many_variables.yml rename to tests/design/gha/files/too_many_variables.yml index b425a58c..fc878b21 100644 --- a/glitch/tests/design/gha/files/too_many_variables.yml +++ b/tests/design/gha/files/too_many_variables.yml @@ -1,9 +1,6 @@ name: Package and Release on: - push: - tags: - - '**' workflow_dispatch: jobs: @@ -13,3 +10,4 @@ jobs: CF_API_KEY: ${{ secrets.CF_API_KEY }} WAGO_API_TOKEN: ${{ secrets.WAGO_API_TOKEN }} GITHUB_OAUTH: ${{ secrets.GITHUB_TOKEN }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/tests/design/gha/test_design.py b/tests/design/gha/test_design.py new file mode 100644 index 00000000..dfed554c --- /dev/null +++ b/tests/design/gha/test_design.py @@ -0,0 +1,21 @@ +from tests.design.design_helper import BaseDesignTest +from glitch.tech import Tech +from glitch.repr.inter import UNDEFINED_POSITION + + +class TestDesign(BaseDesignTest): + TECH = Tech.gha + + # NOTE: This test also verifies if the paths of errors in inner Unit Blocks + # are correctly reported. + def test_gha_too_many_variables(self) -> None: + self._help_test( + "tests/design/gha/files/too_many_variables.yml", + "script", + "configs/default.ini", + 1, + [ + "implementation_too_many_variables", + ], + [UNDEFINED_POSITION], + ) diff --git a/glitch/tests/hierarchical/__init__.py b/tests/design/puppet/__init__.py similarity index 100% rename from glitch/tests/hierarchical/__init__.py rename to tests/design/puppet/__init__.py diff --git a/glitch/tests/design/puppet/design_puppet.ini b/tests/design/puppet/design_puppet.ini similarity index 89% rename from glitch/tests/design/puppet/design_puppet.ini rename to tests/design/puppet/design_puppet.ini index ceae8ace..45a61879 100644 --- a/glitch/tests/design/puppet/design_puppet.ini +++ b/tests/design/puppet/design_puppet.ini @@ -1,5 +1,5 @@ [design] exec_atomic_units = ["exec"] -var_refer_symbol = "" +var_refer_symbol = "$" default_variables = ["$name", "$title", "$server_facts", "$trusted", "$facts", "$authenticated", "$certname", "$domain", "$extensions", "$hostname"] \ No newline at end of file diff --git a/glitch/tests/design/puppet/files/avoid_comments.pp b/tests/design/puppet/files/avoid_comments.pp similarity index 100% rename from glitch/tests/design/puppet/files/avoid_comments.pp rename to tests/design/puppet/files/avoid_comments.pp diff --git a/glitch/tests/design/puppet/files/duplicate_block.pp b/tests/design/puppet/files/duplicate_block.pp similarity index 100% rename from glitch/tests/design/puppet/files/duplicate_block.pp rename to tests/design/puppet/files/duplicate_block.pp diff --git a/glitch/tests/design/puppet/files/improper_alignment.pp b/tests/design/puppet/files/improper_alignment.pp similarity index 100% rename from glitch/tests/design/puppet/files/improper_alignment.pp rename to tests/design/puppet/files/improper_alignment.pp diff --git a/glitch/tests/design/puppet/files/long_resource.pp b/tests/design/puppet/files/long_resource.pp similarity index 100% rename from glitch/tests/design/puppet/files/long_resource.pp rename to tests/design/puppet/files/long_resource.pp diff --git a/glitch/tests/design/puppet/files/long_statement.pp b/tests/design/puppet/files/long_statement.pp similarity index 100% rename from glitch/tests/design/puppet/files/long_statement.pp rename to tests/design/puppet/files/long_statement.pp diff --git a/glitch/tests/design/puppet/files/misplaced_attribute.pp b/tests/design/puppet/files/misplaced_attribute.pp similarity index 100% rename from glitch/tests/design/puppet/files/misplaced_attribute.pp rename to tests/design/puppet/files/misplaced_attribute.pp diff --git a/glitch/tests/design/puppet/files/multifaceted_abstraction.pp b/tests/design/puppet/files/multifaceted_abstraction.pp similarity index 100% rename from glitch/tests/design/puppet/files/multifaceted_abstraction.pp rename to tests/design/puppet/files/multifaceted_abstraction.pp diff --git a/glitch/tests/design/puppet/files/too_many_variables.pp b/tests/design/puppet/files/too_many_variables.pp similarity index 100% rename from glitch/tests/design/puppet/files/too_many_variables.pp rename to tests/design/puppet/files/too_many_variables.pp diff --git a/glitch/tests/design/puppet/files/unguarded_variable.pp b/tests/design/puppet/files/unguarded_variable.pp similarity index 100% rename from glitch/tests/design/puppet/files/unguarded_variable.pp rename to tests/design/puppet/files/unguarded_variable.pp diff --git a/glitch/tests/design/puppet/test_design.py b/tests/design/puppet/test_design.py similarity index 64% rename from glitch/tests/design/puppet/test_design.py rename to tests/design/puppet/test_design.py index 26046965..63a21658 100644 --- a/glitch/tests/design/puppet/test_design.py +++ b/tests/design/puppet/test_design.py @@ -1,48 +1,36 @@ -import unittest - -from glitch.analysis.design.visitor import DesignVisitor -from glitch.parsers.puppet import PuppetParser +from tests.design.design_helper import BaseDesignTest from glitch.tech import Tech +from glitch.repr.inter import UNDEFINED_POSITION -class TestDesign(unittest.TestCase): - def __help_test(self, path, n_errors: int, codes, lines) -> None: - parser = PuppetParser() - inter = parser.parse(path, "script", False) - analysis = DesignVisitor(Tech.puppet) - analysis.config("tests/design/puppet/design_puppet.ini") - errors = list( - filter( - lambda e: e.code.startswith("design_") - or e.code.startswith("implementation_"), - set(analysis.check(inter)), - ) - ) - errors = sorted(errors, key=lambda e: (e.path, e.line, e.code)) - self.assertEqual(len(errors), n_errors) - for i in range(n_errors): - self.assertEqual(errors[i].code, codes[i]) - self.assertEqual(errors[i].line, lines[i]) +class TestDesign(BaseDesignTest): + TECH = Tech.puppet def test_puppet_long_statement(self) -> None: - self.__help_test( + self._help_test( "tests/design/puppet/files/long_statement.pp", + "script", + "tests/design/puppet/design_puppet.ini", 1, ["implementation_long_statement"], [6], ) def test_puppet_improper_alignment(self) -> None: - self.__help_test( + self._help_test( "tests/design/puppet/files/improper_alignment.pp", + "script", + "tests/design/puppet/design_puppet.ini", 1, ["implementation_improper_alignment"], [1], ) def test_puppet_duplicate_block(self) -> None: - self.__help_test( + self._help_test( "tests/design/puppet/files/duplicate_block.pp", + "script", + "tests/design/puppet/design_puppet.ini", 2, [ "design_duplicate_block", @@ -52,8 +40,10 @@ def test_puppet_duplicate_block(self) -> None: ) def test_puppet_avoid_comments(self) -> None: - self.__help_test( + self._help_test( "tests/design/puppet/files/avoid_comments.pp", + "script", + "tests/design/puppet/design_puppet.ini", 1, [ "design_avoid_comments", @@ -62,8 +52,10 @@ def test_puppet_avoid_comments(self) -> None: ) def test_puppet_long_resource(self) -> None: - self.__help_test( + self._help_test( "tests/design/puppet/files/long_resource.pp", + "script", + "tests/design/puppet/design_puppet.ini", 1, [ "design_long_resource", @@ -72,16 +64,20 @@ def test_puppet_long_resource(self) -> None: ) def test_puppet_multifaceted_abstraction(self) -> None: - self.__help_test( + self._help_test( "tests/design/puppet/files/multifaceted_abstraction.pp", + "script", + "tests/design/puppet/design_puppet.ini", 2, ["design_multifaceted_abstraction", "implementation_long_statement"], [1, 2], ) def test_puppet_unguarded_variable(self) -> None: - self.__help_test( + self._help_test( "tests/design/puppet/files/unguarded_variable.pp", + "script", + "tests/design/puppet/design_puppet.ini", 1, [ "implementation_unguarded_variable", @@ -90,8 +86,10 @@ def test_puppet_unguarded_variable(self) -> None: ) def test_puppet_misplaced_attribute(self) -> None: - self.__help_test( + self._help_test( "tests/design/puppet/files/misplaced_attribute.pp", + "script", + "tests/design/puppet/design_puppet.ini", 1, [ "design_misplaced_attribute", @@ -100,11 +98,13 @@ def test_puppet_misplaced_attribute(self) -> None: ) def test_puppet_too_many_variables(self) -> None: - self.__help_test( + self._help_test( "tests/design/puppet/files/too_many_variables.pp", + "script", + "tests/design/puppet/design_puppet.ini", 1, [ "implementation_too_many_variables", ], - [1], + [UNDEFINED_POSITION], ) diff --git a/glitch/tests/parser/__init__.py b/tests/design/terraform/__init__.py similarity index 100% rename from glitch/tests/parser/__init__.py rename to tests/design/terraform/__init__.py diff --git a/glitch/tests/design/terraform/files/avoid_comments.tf b/tests/design/terraform/files/avoid_comments.tf similarity index 100% rename from glitch/tests/design/terraform/files/avoid_comments.tf rename to tests/design/terraform/files/avoid_comments.tf diff --git a/glitch/tests/design/terraform/files/duplicate_block.tf b/tests/design/terraform/files/duplicate_block.tf similarity index 100% rename from glitch/tests/design/terraform/files/duplicate_block.tf rename to tests/design/terraform/files/duplicate_block.tf diff --git a/glitch/tests/design/terraform/files/improper_alignment.tf b/tests/design/terraform/files/improper_alignment.tf similarity index 100% rename from glitch/tests/design/terraform/files/improper_alignment.tf rename to tests/design/terraform/files/improper_alignment.tf diff --git a/glitch/tests/design/terraform/files/long_statement.tf b/tests/design/terraform/files/long_statement.tf similarity index 100% rename from glitch/tests/design/terraform/files/long_statement.tf rename to tests/design/terraform/files/long_statement.tf diff --git a/glitch/tests/design/terraform/files/too_many_variables.tf b/tests/design/terraform/files/too_many_variables.tf similarity index 87% rename from glitch/tests/design/terraform/files/too_many_variables.tf rename to tests/design/terraform/files/too_many_variables.tf index 6e9f4b75..82bde1f9 100644 --- a/glitch/tests/design/terraform/files/too_many_variables.tf +++ b/tests/design/terraform/files/too_many_variables.tf @@ -1,12 +1,9 @@ locals { instance_ids = concat(aws_instance.blue.*.id, aws_instance.green.*.id) } - locals { labels1 = ["default", {yes = "yes"}, "hello"] - common_tags = { Service = local.service_name - Owner = local.owner } } diff --git a/glitch/tests/design/terraform/test_design.py b/tests/design/terraform/test_design.py similarity index 53% rename from glitch/tests/design/terraform/test_design.py rename to tests/design/terraform/test_design.py index a58b4c49..7fa0c3cc 100644 --- a/glitch/tests/design/terraform/test_design.py +++ b/tests/design/terraform/test_design.py @@ -1,48 +1,36 @@ -import unittest - -from glitch.analysis.design.visitor import DesignVisitor -from glitch.parsers.terraform import TerraformParser +from tests.design.design_helper import BaseDesignTest from glitch.tech import Tech +from glitch.repr.inter import UNDEFINED_POSITION -class TestDesign(unittest.TestCase): - def __help_test(self, path, n_errors: int, codes, lines) -> None: - parser = TerraformParser() - inter = parser.parse(path, "script", False) - analysis = DesignVisitor(Tech.terraform) - analysis.config("configs/default.ini") - errors = list( - filter( - lambda e: e.code.startswith("design_") - or e.code.startswith("implementation_"), - set(analysis.check(inter)), - ) - ) - errors = sorted(errors, key=lambda e: (e.path, e.line, e.code)) - self.assertEqual(len(errors), n_errors) - for i in range(n_errors): - self.assertEqual(errors[i].code, codes[i]) - self.assertEqual(errors[i].line, lines[i]) +class TestDesign(BaseDesignTest): + TECH = Tech.terraform def test_terraform_long_statement(self) -> None: - self.__help_test( + self._help_test( "tests/design/terraform/files/long_statement.tf", + "script", + "configs/default.ini", 1, ["implementation_long_statement"], [6], ) def test_terraform_improper_alignment(self) -> None: - self.__help_test( + self._help_test( "tests/design/terraform/files/improper_alignment.tf", + "script", + "configs/default.ini", 1, ["implementation_improper_alignment"], [1], ) def test_terraform_duplicate_block(self) -> None: - self.__help_test( + self._help_test( "tests/design/terraform/files/duplicate_block.tf", + "script", + "configs/default.ini", 2, [ "design_duplicate_block", @@ -52,8 +40,10 @@ def test_terraform_duplicate_block(self) -> None: ) def test_terraform_avoid_comments(self) -> None: - self.__help_test( + self._help_test( "tests/design/terraform/files/avoid_comments.tf", + "script", + "configs/default.ini", 2, [ "design_avoid_comments", @@ -63,11 +53,13 @@ def test_terraform_avoid_comments(self) -> None: ) def test_terraform_too_many_variables(self) -> None: - self.__help_test( + self._help_test( "tests/design/terraform/files/too_many_variables.tf", + "script", + "configs/default.ini", 1, [ "implementation_too_many_variables", ], - [-1], + [UNDEFINED_POSITION], ) diff --git a/glitch/tests/parser/puppet/__init__.py b/tests/hierarchical/__init__.py similarity index 100% rename from glitch/tests/parser/puppet/__init__.py rename to tests/hierarchical/__init__.py diff --git a/glitch/tests/hierarchical/ansible/attributes.yml b/tests/hierarchical/ansible/attributes.yml similarity index 65% rename from glitch/tests/hierarchical/ansible/attributes.yml rename to tests/hierarchical/ansible/attributes.yml index 2df293e0..1c383fad 100644 --- a/glitch/tests/hierarchical/ansible/attributes.yml +++ b/tests/hierarchical/ansible/attributes.yml @@ -1,6 +1,5 @@ -- name: Calculate factorial of a number - hosts: localhost - debug: +--- +- debug: msg: "The factorial of 5 is {{ factorial_value }}" seq: [test: "something", "y", "z"] hash: {test1: "1", test2: "2"} diff --git a/glitch/tests/hierarchical/ansible/vars.yml b/tests/hierarchical/ansible/vars.yml similarity index 100% rename from glitch/tests/hierarchical/ansible/vars.yml rename to tests/hierarchical/ansible/vars.yml diff --git a/glitch/tests/hierarchical/chef/vars.rb b/tests/hierarchical/chef/vars.rb similarity index 100% rename from glitch/tests/hierarchical/chef/vars.rb rename to tests/hierarchical/chef/vars.rb diff --git a/glitch/tests/hierarchical/puppet/vars.pp b/tests/hierarchical/puppet/vars.pp similarity index 100% rename from glitch/tests/hierarchical/puppet/vars.pp rename to tests/hierarchical/puppet/vars.pp diff --git a/tests/hierarchical/test_parsers.py b/tests/hierarchical/test_parsers.py new file mode 100644 index 00000000..c85393c7 --- /dev/null +++ b/tests/hierarchical/test_parsers.py @@ -0,0 +1,523 @@ +import unittest +from glitch.parsers.ansible import AnsibleParser +from glitch.parsers.chef import ChefParser +from glitch.parsers.puppet import PuppetParser +from glitch.parsers.parser import Parser +from glitch.repr.inter import * +from typing import Any, Dict, Sequence, List + + +def simplify_kvs(ir: Sequence[KeyValue]) -> List[Dict[Any, Any]]: + def simplify_dict(kv_dict: Dict[Any, Any]) -> None: + for key, value in list(kv_dict.items()): + if key in ["line", "column", "end_line", "end_column"]: + del kv_dict[key] + continue + + if isinstance(value, dict): + simplify_dict(value) # type: ignore + elif isinstance(value, list): + for i, item in enumerate(value): # type: ignore + if isinstance(item, dict): + simplify_dict(item) # type: ignore + + res: List[Dict[Any, Any]] = [] + for kv in ir: + kv_dict = kv.as_dict() + res.append(kv_dict) + simplify_dict(kv_dict) + + return res + + +class TestHierarchicalParsers(unittest.TestCase): + def _test_parse_vars( + self, + path: str, + parser: Parser, + block_type: UnitBlockType, + expected: List[Dict[Any, Any]], + ) -> None: + unitblock = parser.parse_file(path, block_type) + assert unitblock is not None + self.assertEqual(simplify_kvs(unitblock.variables), expected) + + +class TestHierarchicalAnsible(TestHierarchicalParsers): + def test_hierarchical_vars_ansible(self) -> None: + self._test_parse_vars( + "tests/hierarchical/ansible/vars.yml", + AnsibleParser(), + UnitBlockType.unknown, + [ + { + "ir_type": "Variable", + "code": "test:\n", + "name": "test", + "value": { + "ir_type": "Array", + "code": " - test1:\n - [1, 2]\n - [3, 4]\n - x\n - y\n - 23\n\n", + "value": [ + { + "ir_type": "Hash", + "code": " - test1:\n - [1, 2]\n ", + "value": [ + { + "key": { + "ir_type": "String", + "code": "test1", + "value": "test1", + }, + "value": { + "ir_type": "Array", + "code": " - [1, 2]\n ", + "value": [ + { + "ir_type": "Array", + "code": "[1, 2]", + "value": [ + { + "ir_type": "Integer", + "code": "1", + "value": 1, + }, + { + "ir_type": "Integer", + "code": "2", + "value": 2, + }, + ], + } + ], + }, + } + ], + }, + { + "ir_type": "Array", + "code": "[3, 4]", + "value": [ + {"ir_type": "Integer", "code": "3", "value": 3}, + {"ir_type": "Integer", "code": "4", "value": 4}, + ], + }, + {"ir_type": "String", "code": "x", "value": "x"}, + {"ir_type": "String", "code": "y", "value": "y"}, + {"ir_type": "Integer", "code": "23", "value": 23}, + ], + }, + }, + { + "ir_type": "Variable", + "code": "test2:\n", + "name": "test2", + "value": { + "ir_type": "Array", + "code": " - [2, 5, 6]\n\n", + "value": [ + { + "ir_type": "Array", + "code": "[2, 5, 6]", + "value": [ + {"ir_type": "Integer", "code": "2", "value": 2}, + {"ir_type": "Integer", "code": "5", "value": 5}, + {"ir_type": "Integer", "code": "6", "value": 6}, + ], + } + ], + }, + }, + { + "ir_type": "Variable", + "code": "vars:\n", + "name": "vars", + "value": { + "ir_type": "Hash", + "code": " factorial_of: 5\n factorial_value: 1", + "value": [ + { + "key": { + "ir_type": "String", + "code": "factorial_of", + "value": "factorial_of", + }, + "value": { + "ir_type": "Integer", + "code": "5", + "value": 5, + }, + }, + { + "key": { + "ir_type": "String", + "code": "factorial_value", + "value": "factorial_value", + }, + "value": { + "ir_type": "Integer", + "code": "1", + "value": 1, + }, + }, + ], + }, + }, + ], + ) + + def test_hierarchical_attributes_ansible(self) -> None: + unitblock = AnsibleParser().parse_file( + "tests/hierarchical/ansible/attributes.yml", UnitBlockType.unknown + ) + assert unitblock is not None + assert len(unitblock.atomic_units) == 1 + self.assertEqual( + simplify_kvs(unitblock.atomic_units[0].attributes), + [ + { + "ir_type": "Attribute", + "code": 'msg: "The factorial of 5 is {{ factorial_value }}"', + "name": "msg", + "value": { + "ir_type": "Sum", + "code": '"The factorial of 5 is {{ factorial_value }}"', + "left": { + "ir_type": "String", + "code": '"The factorial of 5 is ', + "value": "The factorial of 5 is ", + }, + "right": { + "ir_type": "VariableReference", + "code": "factorial_value", + "value": "factorial_value", + }, + "type": "sum", + }, + }, + { + "ir_type": "Attribute", + "code": 'seq: [test: "something", "y", "z"]', + "name": "seq", + "value": { + "ir_type": "Array", + "code": '[test: "something", "y", "z"]', + "value": [ + { + "ir_type": "Hash", + "code": 'test: "something"', + "value": [ + { + "key": { + "ir_type": "String", + "code": "test", + "value": "test", + }, + "value": { + "ir_type": "String", + "code": '"something"', + "value": "something", + }, + } + ], + }, + {"ir_type": "String", "code": '"y"', "value": "y"}, + {"ir_type": "String", "code": '"z"', "value": "z"}, + ], + }, + }, + { + "ir_type": "Attribute", + "code": 'hash: {test1: "1", test2: "2"}', + "name": "hash", + "value": { + "ir_type": "Hash", + "code": '{test1: "1", test2: "2"}', + "value": [ + { + "key": { + "ir_type": "String", + "code": "test1", + "value": "test1", + }, + "value": { + "ir_type": "String", + "code": '"1"', + "value": "1", + }, + }, + { + "key": { + "ir_type": "String", + "code": "test2", + "value": "test2", + }, + "value": { + "ir_type": "String", + "code": '"2"', + "value": "2", + }, + }, + ], + }, + }, + ], + ) + + +class TestHierarchicalPuppet(TestHierarchicalParsers): + def test_hierarchical_vars_puppet(self) -> None: + self._test_parse_vars( + "tests/hierarchical/puppet/vars.pp", + PuppetParser(), + UnitBlockType.script, + [ + { + "ir_type": "Variable", + "code": "$my_hash = {\n'key1' => {\n 'test1' => '1',\n 'test2' => '2',\n },\n'key2' => 'value2',\n'key3' => 'value3',\n}", + "name": "my_hash", + "value": { + "ir_type": "Hash", + "code": "{\n'key1' => {\n 'test1' => '1',\n 'test2' => '2',\n },\n'key2' => 'value2',\n'key3' => 'value3',\n}", + "value": [ + { + "key": { + "ir_type": "String", + "code": "'key1'", + "value": "key1", + }, + "value": { + "ir_type": "Hash", + "code": "{\n 'test1' => '1',\n 'test2' => '2',\n }", + "value": [ + { + "key": { + "ir_type": "String", + "code": "'test1'", + "value": "test1", + }, + "value": { + "ir_type": "String", + "code": "'1'", + "value": "1", + }, + }, + { + "key": { + "ir_type": "String", + "code": "'test2'", + "value": "test2", + }, + "value": { + "ir_type": "String", + "code": "'2'", + "value": "2", + }, + }, + ], + }, + }, + { + "key": { + "ir_type": "String", + "code": "'key2'", + "value": "key2", + }, + "value": { + "ir_type": "String", + "code": "'value2'", + "value": "value2", + }, + }, + { + "key": { + "ir_type": "String", + "code": "'key3'", + "value": "key3", + }, + "value": { + "ir_type": "String", + "code": "'value3'", + "value": "value3", + }, + }, + ], + }, + }, + { + "ir_type": "Variable", + "code": "$my_hash['key4']['key5'] = 'value5'", + "name": "my_hash['key4']['key5']", + "value": { + "ir_type": "String", + "code": "'value5'", + "value": "value5", + }, + }, + { + "ir_type": "Variable", + "code": '$configdir = "${boxen::config::configdir}/php"', + "name": "configdir", + "value": { + "ir_type": "Sum", + "code": '"${boxen::config::configdir}/php"', + "left": { + "ir_type": "VariableReference", + "code": "boxen::config::configdir", + "value": "boxen::config::configdir", + }, + "right": { + "ir_type": "String", + "code": '"${boxen::config::configdir}/php"', + "value": "/php", + }, + "type": "sum", + }, + }, + { + "ir_type": "Variable", + "code": '$datadir = "${boxen::config::datadir}/php"', + "name": "datadir", + "value": { + "ir_type": "Sum", + "code": '"${boxen::config::datadir}/php"', + "left": { + "ir_type": "VariableReference", + "code": "boxen::config::datadir", + "value": "boxen::config::datadir", + }, + "right": { + "ir_type": "String", + "code": '"${boxen::config::datadir}/php"', + "value": "/php", + }, + "type": "sum", + }, + }, + { + "ir_type": "Variable", + "code": '$pluginsdir = "${root}/plugins"', + "name": "pluginsdir", + "value": { + "ir_type": "Sum", + "code": '"${root}/plugins"', + "left": { + "ir_type": "VariableReference", + "code": "root", + "value": "root", + }, + "right": { + "ir_type": "String", + "code": '"${root}/plugins"', + "value": "/plugins", + }, + "type": "sum", + }, + }, + { + "ir_type": "Variable", + "code": '$cachedir = "${php::config::datadir}/cache"', + "name": "cachedir", + "value": { + "ir_type": "Sum", + "code": '"${php::config::datadir}/cache"', + "left": { + "ir_type": "VariableReference", + "code": "php::config::datadir", + "value": "php::config::datadir", + }, + "right": { + "ir_type": "String", + "code": '"${php::config::datadir}/cache"', + "value": "/cache", + }, + "type": "sum", + }, + }, + { + "ir_type": "Variable", + "code": '$extensioncachedir = "${php::config::datadir}/cache/extensions"', + "name": "extensioncachedir", + "value": { + "ir_type": "Sum", + "code": '"${php::config::datadir}/cache/extensions"', + "left": { + "ir_type": "VariableReference", + "code": "php::config::datadir", + "value": "php::config::datadir", + }, + "right": { + "ir_type": "String", + "code": '"${php::config::datadir}/cache/extensions"', + "value": "/cache/extensions", + }, + "type": "sum", + }, + }, + ], + ) + + +class TestHierarchicalChef(TestHierarchicalParsers): + def test_hierarchical_vars_chef(self) -> None: + self._test_parse_vars( + "tests/hierarchical/chef/vars.rb", + ChefParser(), + UnitBlockType.script, + [ + { + "ir_type": "Variable", + "code": "grades", + "name": "grades", + "value": { + "ir_type": "Hash", + "code": '{ "Jane Doe" => 10, "Jim Doe" => 6 }', + "value": [ + { + "key": { + "ir_type": "String", + "code": "Jane Doe", + "value": "Jane Doe", + }, + "value": { + "ir_type": "Integer", + "code": "10", + "value": 10, + }, + }, + { + "key": { + "ir_type": "String", + "code": "Jim Doe", + "value": "Jim Doe", + }, + "value": { + "ir_type": "Integer", + "code": "6", + "value": 6, + }, + }, + ], + }, + }, + { + "ir_type": "Variable", + "code": "default[:zabbix][:database][:password]", + "name": "default[:zabbix][:database][:password]", + "value": { + "ir_type": "Null", + "code": "nil", + "value": None, + }, + }, + { + "ir_type": "Variable", + "code": "default[:zabbix][:test][:name]", + "name": "default[:zabbix][:test][:name]", + "value": { + "ir_type": "String", + "code": "something", + "value": "something", + }, + }, + ], + ) diff --git a/glitch/tests/parser/terraform/__init__.py b/tests/parser/__init__.py similarity index 100% rename from glitch/tests/parser/terraform/__init__.py rename to tests/parser/__init__.py diff --git a/glitch/tests/security/__init__.py b/tests/parser/ansible/__init__.py similarity index 100% rename from glitch/tests/security/__init__.py rename to tests/parser/ansible/__init__.py diff --git a/tests/parser/ansible/files/become.yml b/tests/parser/ansible/files/become.yml new file mode 100644 index 00000000..4683cf39 --- /dev/null +++ b/tests/parser/ansible/files/become.yml @@ -0,0 +1,5 @@ +--- +- become: true + name: Run a command as the apache user + command: somecommand + become_user: apache \ No newline at end of file diff --git a/tests/parser/ansible/files/node_not_supported.yml b/tests/parser/ansible/files/node_not_supported.yml new file mode 100644 index 00000000..c61a3156 --- /dev/null +++ b/tests/parser/ansible/files/node_not_supported.yml @@ -0,0 +1,15 @@ +--- +- name: Deploy certificates, if any + copy: + src: "{{ item.source }}" + dest: "{{ item.dest }}" + remote_src: "{{ item.remote }}" + owner: "{{ nginx_user }}" + group: "{{ groups['mongodb'][0] }}" + mode: 0640 + test: "{{ install.get('registry', {}).namespace|default('') }}" + with_items: "{{ nginx_certificate_files }}" + notify: + - restart nginx + tags: + - nginx \ No newline at end of file diff --git a/tests/parser/ansible/files/restore_service_set_fact.yml b/tests/parser/ansible/files/restore_service_set_fact.yml new file mode 100644 index 00000000..b3e45d77 --- /dev/null +++ b/tests/parser/ansible/files/restore_service_set_fact.yml @@ -0,0 +1,5 @@ +--- +- name: Restore Service | Set variables + set_fact: + restore_service_username: "{{ restore_service.user | hash('sha1') }}" + diff --git a/tests/parser/ansible/files/valid_playbook_hierarchical_vars.yml b/tests/parser/ansible/files/valid_playbook_hierarchical_vars.yml new file mode 100644 index 00000000..0a566612 --- /dev/null +++ b/tests/parser/ansible/files/valid_playbook_hierarchical_vars.yml @@ -0,0 +1,7 @@ +--- +- hosts: localhost + connection: local + gather_facts: no + vars: + aqua_admin: + user: "test" \ No newline at end of file diff --git a/tests/parser/ansible/files/valid_playbook_vars.yml b/tests/parser/ansible/files/valid_playbook_vars.yml new file mode 100644 index 00000000..825e0f4e --- /dev/null +++ b/tests/parser/ansible/files/valid_playbook_vars.yml @@ -0,0 +1,8 @@ +--- +- hosts: localhost + connection: local + gather_facts: no + vars: + aqua_admin_password: "{{ lookup('env', 'AQUA_ADMIN_PASSWORD') }}" + aqua_sso_client_secret: >- + "{{ lookup('env', 'AQUA_SSO_CLIENT_SECRET') }}" diff --git a/tests/parser/ansible/files/valid_playbook_vars_list.yml b/tests/parser/ansible/files/valid_playbook_vars_list.yml new file mode 100644 index 00000000..d0377d92 --- /dev/null +++ b/tests/parser/ansible/files/valid_playbook_vars_list.yml @@ -0,0 +1,8 @@ +--- +- hosts: localhost + connection: local + gather_facts: no + vars: + aqua_admin_users: + - user: test1 + - user: test2 \ No newline at end of file diff --git a/tests/parser/ansible/files/valid_tasks.yml b/tests/parser/ansible/files/valid_tasks.yml new file mode 100644 index 00000000..f379a4ce --- /dev/null +++ b/tests/parser/ansible/files/valid_tasks.yml @@ -0,0 +1,8 @@ +--- +- name: Get machine-id + shell: >- + hostnamectl --machine="{{ inventory_hostname }}" status | awk '/Machine ID/ {print $3}' + register: _container_machine_id + delegate_to: "{{ physical_host }}" + test: + executable: ["/bin/bash", "/bin/shell"] \ No newline at end of file diff --git a/tests/parser/ansible/files/valid_vars.yml b/tests/parser/ansible/files/valid_vars.yml new file mode 100644 index 00000000..03e3d146 --- /dev/null +++ b/tests/parser/ansible/files/valid_vars.yml @@ -0,0 +1,10 @@ +--- +aqua_admin_users: + - "test1" + - "test2" +aqua_admin_passwords: ["test1", "test2"] +test: ~ +test_2: null +test_3: 1.0 +test_4: 1 +test_5: true \ No newline at end of file diff --git a/tests/parser/ansible/files/valid_vars_interpolation.yml b/tests/parser/ansible/files/valid_vars_interpolation.yml new file mode 100644 index 00000000..ca5d5a7a --- /dev/null +++ b/tests/parser/ansible/files/valid_vars_interpolation.yml @@ -0,0 +1,5 @@ +--- +with_filter: "{{ var | upper }}" +with_string: "{{ 'string' }}" +with_list: "{{ [1, 2, 3] }}" +with_sum: "{{ var + 'string' + 1 }}" \ No newline at end of file diff --git a/tests/parser/ansible/files/valid_vars_list_with_variable_reference.yml b/tests/parser/ansible/files/valid_vars_list_with_variable_reference.yml new file mode 100644 index 00000000..004e0c98 --- /dev/null +++ b/tests/parser/ansible/files/valid_vars_list_with_variable_reference.yml @@ -0,0 +1,3 @@ +postgresql_users: +- user: {{ idr_omero_readonly_database.user }} + diff --git a/tests/parser/ansible/test_parser.py b/tests/parser/ansible/test_parser.py new file mode 100644 index 00000000..7676eb1a --- /dev/null +++ b/tests/parser/ansible/test_parser.py @@ -0,0 +1,585 @@ +from glitch.parsers.ansible import AnsibleParser +from glitch.repr.inter import * +from tests.parser.test_parser import TestParser + + +class TestAnsibleParser(TestParser): + def test_ansible_parser_valid_tasks(self) -> None: + """ + Value in another line + String interpolation (Variable reference) + In-line list + """ + p = AnsibleParser() + ir = p.parse_file( + "tests/parser/ansible/files/valid_tasks.yml", UnitBlockType.tasks + ) + assert ir is not None + + assert isinstance(ir, UnitBlock) + assert ir.type == UnitBlockType.tasks + assert len(ir.atomic_units) == 1 + + assert isinstance(ir.atomic_units[0], AtomicUnit) + self._check_value( + ir.atomic_units[0].name, String, "Get machine-id", 2, 9, 2, 23 + ) + assert ir.atomic_units[0].type == "shell" + assert len(ir.atomic_units[0].attributes) == 4 + + assert ir.atomic_units[0].attributes[0].name == "shell" + + assert isinstance(ir.atomic_units[0].attributes[0].value, Sum) + assert isinstance(ir.atomic_units[0].attributes[0].value.left, BinaryOperation) + self._check_value( + ir.atomic_units[0].attributes[0].value.left.left, + String, + 'hostnamectl --machine="', + 4, + 5, + 4, + 28, + ) + assert isinstance( + ir.atomic_units[0].attributes[0].value.left.right, VariableReference + ) + self._check_value( + ir.atomic_units[0].attributes[0].value.left.right, + VariableReference, + "inventory_hostname", + 4, + 31, + 4, + 49, + ) + assert isinstance(ir.atomic_units[0].attributes[0].value.right, String) + self._check_value( + ir.atomic_units[0].attributes[0].value.right, + String, + "\" status | awk '/Machine ID/ {print $3}'", + 4, + 52, + 4, + 92, + ) + + assert ir.atomic_units[0].attributes[0].line == 3 + assert ir.atomic_units[0].attributes[0].column == 3 + assert ir.atomic_units[0].attributes[0].end_line == 5 + assert ir.atomic_units[0].attributes[0].end_column == 1 + + assert ir.atomic_units[0].attributes[1].name == "register" + self._check_value( + ir.atomic_units[0].attributes[1].value, + String, + "_container_machine_id", + 5, + 13, + 5, + 34, + ) + assert ir.atomic_units[0].attributes[1].line == 5 + assert ir.atomic_units[0].attributes[1].column == 3 + assert ir.atomic_units[0].attributes[1].end_line == 5 + assert ir.atomic_units[0].attributes[1].end_column == 34 + + assert ir.atomic_units[0].attributes[2].name == "delegate_to" + self._check_value( + ir.atomic_units[0].attributes[2].value, + VariableReference, + "physical_host", + 6, + 20, + 6, + 33, + ) + assert ir.atomic_units[0].attributes[2].line == 6 + assert ir.atomic_units[0].attributes[2].column == 3 + assert ir.atomic_units[0].attributes[2].end_line == 6 + assert ir.atomic_units[0].attributes[2].end_column == 37 + + assert ir.atomic_units[0].attributes[3].name == "test" + assert isinstance(ir.atomic_units[0].attributes[3].value, Hash) + assert len(ir.atomic_units[0].attributes[3].value.value) == 1 + assert ( + String("executable", ElementInfo(8, 5, 8, 15, "executable")) + in ir.atomic_units[0].attributes[3].value.value + ) + + executable_value = ( + ir.atomic_units[0] + .attributes[3] + .value.value[String("executable", ElementInfo(8, 5, 8, 15, "executable"))] + ) + assert isinstance(executable_value, Array) + assert len(executable_value.value) == 2 + + self._check_value(executable_value.value[0], String, "/bin/bash", 8, 18, 8, 29) + self._check_value(executable_value.value[1], String, "/bin/shell", 8, 31, 8, 43) + + def test_ansible_parser_valid_playbook_vars(self) -> None: + """ + String interpolation (Variable reference) + String in another line + """ + p = AnsibleParser() + ir = p.parse_file( + "tests/parser/ansible/files/valid_playbook_vars.yml", UnitBlockType.script + ) + assert ir is not None + + assert isinstance(ir, UnitBlock) + assert ir.type == UnitBlockType.script + assert len(ir.unit_blocks) == 1 + assert isinstance(ir.unit_blocks[0], UnitBlock) + assert ir.unit_blocks[0].type == UnitBlockType.block + assert len(ir.unit_blocks[0].variables) == 2 + + assert ir.unit_blocks[0].variables[0].name == "aqua_admin_password" + assert isinstance(ir.unit_blocks[0].variables[0].value, FunctionCall) + assert ir.unit_blocks[0].variables[0].value.name == "lookup" + assert len(ir.unit_blocks[0].variables[0].value.args) == 2 + self._check_value( + ir.unit_blocks[0].variables[0].value.args[0], + String, + "env", + 6, + 37, + 6, + 42, + ) + self._check_value( + ir.unit_blocks[0].variables[0].value.args[1], + String, + "AQUA_ADMIN_PASSWORD", + 6, + 44, + 6, + 65, + ) + assert ir.unit_blocks[0].variables[0].line == 6 + assert ir.unit_blocks[0].variables[0].column == 5 + assert ir.unit_blocks[0].variables[0].end_line == 6 + assert ir.unit_blocks[0].variables[0].end_column == 70 + + assert ir.unit_blocks[0].variables[1].name == "aqua_sso_client_secret" + assert isinstance(ir.unit_blocks[0].variables[1].value, FunctionCall) + assert ir.unit_blocks[0].variables[1].value.name == "lookup" + assert len(ir.unit_blocks[0].variables[1].value.args) == 2 + self._check_value( + ir.unit_blocks[0].variables[1].value.args[0], + String, + "env", + 8, + 18, + 8, + 23, + ) + self._check_value( + ir.unit_blocks[0].variables[1].value.args[1], + String, + "AQUA_SSO_CLIENT_SECRET", + 8, + 25, + 8, + 49, + ) + assert ir.unit_blocks[0].variables[1].line == 7 + assert ir.unit_blocks[0].variables[1].column == 5 + assert ir.unit_blocks[0].variables[1].end_line == 9 + assert ir.unit_blocks[0].variables[1].end_column == 1 + + def test_ansible_parser_valid_playbook_hierarchical_vars(self) -> None: + """ + Hierarchical variables + """ + p = AnsibleParser() + ir = p.parse_file( + "tests/parser/ansible/files/valid_playbook_hierarchical_vars.yml", + UnitBlockType.script, + ) + assert ir is not None + + assert isinstance(ir, UnitBlock) + assert ir.type == UnitBlockType.script + assert len(ir.unit_blocks) == 1 + assert isinstance(ir.unit_blocks[0], UnitBlock) + assert ir.unit_blocks[0].type == UnitBlockType.block + assert len(ir.unit_blocks[0].variables) == 1 + + assert ir.unit_blocks[0].variables[0].name == "aqua_admin" + assert isinstance(ir.unit_blocks[0].variables[0].value, Hash) + assert ir.unit_blocks[0].variables[0].line == 6 + assert ir.unit_blocks[0].variables[0].column == 5 + assert ir.unit_blocks[0].variables[0].end_line == 7 + assert ir.unit_blocks[0].variables[0].end_column == 19 + + assert ( + String("user", ElementInfo(7, 7, 7, 11, "user")) + in ir.unit_blocks[0].variables[0].value.value + ) + self._check_value( + ir.unit_blocks[0] + .variables[0] + .value.value[String("user", ElementInfo(7, 7, 7, 11, "user"))], + String, + "test", + 7, + 13, + 7, + 19, + ) + + def test_ansible_parser_valid_playbook_vars_list(self) -> None: + """ + Hierarchical variables with list + """ + p = AnsibleParser() + ir = p.parse_file( + "tests/parser/ansible/files/valid_playbook_vars_list.yml", + UnitBlockType.script, + ) + assert ir is not None + + assert isinstance(ir, UnitBlock) + assert ir.type == UnitBlockType.script + assert len(ir.unit_blocks) == 1 + assert isinstance(ir.unit_blocks[0], UnitBlock) + assert ir.unit_blocks[0].type == UnitBlockType.block + assert len(ir.unit_blocks[0].variables) == 1 + + assert isinstance(ir.unit_blocks[0].variables[0].value, Array) + assert len(ir.unit_blocks[0].variables[0].value.value) == 2 + + assert isinstance(ir.unit_blocks[0].variables[0].value.value[0], Hash) + assert len(ir.unit_blocks[0].variables[0].value.value[0].value) == 1 + assert ( + String("user", ElementInfo(7, 9, 7, 13, "user")) + in ir.unit_blocks[0].variables[0].value.value[0].value + ) + self._check_value( + ir.unit_blocks[0] + .variables[0] + .value.value[0] + .value[String("user", ElementInfo(7, 9, 7, 13, "user"))], + String, + "test1", + 7, + 15, + 7, + 20, + ) + + assert isinstance(ir.unit_blocks[0].variables[0].value.value[1], Hash) + assert len(ir.unit_blocks[0].variables[0].value.value[1].value) == 1 + assert ( + String("user", ElementInfo(8, 9, 8, 13, "user")) + in ir.unit_blocks[0].variables[0].value.value[1].value + ) + self._check_value( + ir.unit_blocks[0] + .variables[0] + .value.value[1] + .value[String("user", ElementInfo(8, 9, 8, 13, "user"))], + String, + "test2", + 8, + 15, + 8, + 20, + ) + + def test_ansible_parser_valid_vars(self) -> None: + """ + In-line lists + Regular lists + Null + Integer, Float and Boolean + """ + p = AnsibleParser() + ir = p.parse_file( + "tests/parser/ansible/files/valid_vars.yml", UnitBlockType.vars + ) + assert ir is not None + + assert isinstance(ir, UnitBlock) + assert ir.type == UnitBlockType.vars + assert len(ir.variables) == 7 + + assert ir.variables[0].name == "aqua_admin_users" + self._check_value( + ir.variables[0].value, + Array, + [ + String("test1", ElementInfo(3, 5, 3, 12, "test1")), + String("test2", ElementInfo(4, 5, 4, 12, "test2")), + ], + 3, + 3, + 5, + 1, + ) + assert ir.variables[0].line == 2 + assert ir.variables[0].column == 1 + assert ir.variables[0].end_line == 5 + assert ir.variables[0].end_column == 1 + + assert ir.variables[1].name == "aqua_admin_passwords" + self._check_value( + ir.variables[1].value, + Array, + [ + String("test1", ElementInfo(5, 24, 5, 31, "test1")), + String("test2", ElementInfo(5, 33, 5, 40, "test2")), + ], + 5, + 23, + 5, + 41, + ) + assert ir.variables[1].line == 5 + assert ir.variables[1].column == 1 + assert ir.variables[1].end_line == 5 + assert ir.variables[1].end_column == 41 + + assert ir.variables[2].name == "test" + self._check_value(ir.variables[2].value, Null, None, 6, 7, 6, 8) + + assert ir.variables[3].name == "test_2" + self._check_value(ir.variables[3].value, Null, None, 7, 9, 7, 13) + + assert ir.variables[4].name == "test_3" + self._check_value(ir.variables[4].value, Float, 1.0, 8, 9, 8, 12) + + def test_ansible_parser_valid_vars_interpolation(self) -> None: + """ + String interpolation with filter + String interpolation with string/list inside + String interpolation with sum + """ + p = AnsibleParser() + ir = p.parse_file( + "tests/parser/ansible/files/valid_vars_interpolation.yml", + UnitBlockType.vars, + ) + assert ir is not None + + assert len(ir.variables) == 4 + + assert ir.variables[0].name == "with_filter" + self._check_value( + ir.variables[0].value, + FunctionCall, + "filter|upper", + 2, + 14, + 2, + 33, + ) + + assert ir.variables[1].name == "with_string" + self._check_value( + ir.variables[1].value, + String, + "string", + 3, + 18, + 3, + 26, + ) + + assert ir.variables[2].name == "with_list" + assert isinstance(ir.variables[2].value, Array) + assert len(ir.variables[2].value.value) == 3 + self._check_value( + ir.variables[2].value.value[0], + Integer, + 1, + 4, + 17, + 4, + 18, + ) + + assert ir.variables[3].name == "with_sum" + assert isinstance(ir.variables[3].value, Sum) + assert isinstance(ir.variables[3].value.left, Sum) + self._check_value( + ir.variables[3].value.left.left, + VariableReference, + "var", + 5, + 15, + 5, + 18, + ) + self._check_value( + ir.variables[3].value.left.right, + String, + "string", + 5, + 21, + 5, + 29, + ) + self._check_value( + ir.variables[3].value.right, + Integer, + 1, + 5, + 32, + 5, + 33, + ) + + def test_ansible_parser_node_not_supported(self) -> None: + """ + This file used to throw node not supported + """ + p = AnsibleParser() + ir = p.parse_file( + "tests/parser/ansible/files/node_not_supported.yml", + UnitBlockType.tasks, + ) + assert ir is not None + + def test_ansible_parser_become(self) -> None: + """ + Ensures that the type of the atomic unit is not become + """ + p = AnsibleParser() + ir = p.parse_file( + "tests/parser/ansible/files/become.yml", + UnitBlockType.tasks, + ) + assert ir is not None + assert len(ir.atomic_units) == 1 + assert ir.atomic_units[0].type == "command" + + def test_ansible_parser_restore_service_set_fact(self) -> None: + """ + Tests set_fact with variable reference and hash filter + """ + p = AnsibleParser() + ir = p.parse_file( + "tests/parser/ansible/files/restore_service_set_fact.yml", + UnitBlockType.tasks, + ) + assert ir is not None + + assert isinstance(ir, UnitBlock) + assert ir.type == UnitBlockType.tasks + assert len(ir.atomic_units) == 1 + + assert isinstance(ir.atomic_units[0], AtomicUnit) + self._check_value( + ir.atomic_units[0].name, + String, + "Restore Service | Set variables", + 2, + 9, + 2, + 40, + ) + assert ir.atomic_units[0].type == "set_fact" + assert len(ir.atomic_units[0].attributes) == 1 + + assert ir.atomic_units[0].attributes[0].name == "restore_service_username" + assert isinstance(ir.atomic_units[0].attributes[0].value, FunctionCall) + assert ir.atomic_units[0].attributes[0].value.name == "filter|hash" + assert len(ir.atomic_units[0].attributes[0].value.args) == 2 + assert ( + ir.atomic_units[0].attributes[0].value.code + == "\"{{ restore_service.user | hash('sha1') }}\"" + ) + + assert isinstance(ir.atomic_units[0].attributes[0].value.args[0], Access) + assert ( + ir.atomic_units[0].attributes[0].value.args[0].code + == "restore_service.user" + ) + self._check_value( + ir.atomic_units[0].attributes[0].value.args[0].left, + VariableReference, + "restore_service", + 4, + 35, + 4, + 50, + ) + assert ( + ir.atomic_units[0].attributes[0].value.args[0].left.code + == "restore_service" + ) + self._check_value( + ir.atomic_units[0].attributes[0].value.args[0].right, + String, + "user", + 4, + 51, + 4, + 55, + ) + assert ir.atomic_units[0].attributes[0].value.args[0].right.code == "user" + + self._check_value( + ir.atomic_units[0].attributes[0].value.args[1], + String, + "sha1", + 4, + 63, + 4, + 69, + ) + assert ir.atomic_units[0].attributes[0].value.args[1].code == "'sha1'" + + def test_ansible_parser_valid_vars_list_with_variable_reference(self) -> None: + """ + Tests list with hash containing variable reference with access + """ + p = AnsibleParser() + ir = p.parse_file( + "tests/parser/ansible/files/valid_vars_list_with_variable_reference.yml", + UnitBlockType.vars, + ) + assert ir is not None + + assert isinstance(ir, UnitBlock) + assert ir.type == UnitBlockType.vars + assert len(ir.variables) == 1 + + assert ir.variables[0].name == "postgresql_users" + assert isinstance(ir.variables[0].value, Array) + assert len(ir.variables[0].value.value) == 1 + + assert isinstance(ir.variables[0].value.value[0], Hash) + assert len(ir.variables[0].value.value[0].value) == 1 + assert ( + String("user", ElementInfo(2, 3, 2, 7, "user")) + in ir.variables[0].value.value[0].value + ) + + user_value = ( + ir.variables[0] + .value.value[0] + .value[String("user", ElementInfo(2, 3, 2, 7, "user"))] + ) + assert isinstance(user_value, Access) + self._check_value( + user_value.left, + VariableReference, + "idr_omero_readonly_database", + 2, + 12, + 2, + 39, + ) + self._check_value( + user_value.right, + String, + "user", + 2, + 40, + 2, + 44, + ) diff --git a/glitch/tests/security/ansible/__init__.py b/tests/parser/chef/__init__.py similarity index 100% rename from glitch/tests/security/ansible/__init__.py rename to tests/parser/chef/__init__.py diff --git a/tests/parser/chef/files/apache_module_case.rb b/tests/parser/chef/files/apache_module_case.rb new file mode 100644 index 00000000..ac5c3ce4 --- /dev/null +++ b/tests/parser/chef/files/apache_module_case.rb @@ -0,0 +1,8 @@ +apache_module 'php5' do + case node['platform_family'] + when 'rhel', 'fedora', 'freebsd' + conf true + filename 'libphp5.so' + end +end + diff --git a/tests/parser/chef/files/aref.rb b/tests/parser/chef/files/aref.rb new file mode 100644 index 00000000..9b6cdc69 --- /dev/null +++ b/tests/parser/chef/files/aref.rb @@ -0,0 +1,2 @@ +collection[0] = collection[1] +default[:volumes] ||= Mash.new \ No newline at end of file diff --git a/tests/parser/chef/files/arg_paren.rb b/tests/parser/chef/files/arg_paren.rb new file mode 100644 index 00000000..bb4d53ac --- /dev/null +++ b/tests/parser/chef/files/arg_paren.rb @@ -0,0 +1 @@ +c = new() \ No newline at end of file diff --git a/tests/parser/chef/files/args_add.rb b/tests/parser/chef/files/args_add.rb new file mode 100644 index 00000000..e3015917 --- /dev/null +++ b/tests/parser/chef/files/args_add.rb @@ -0,0 +1,6 @@ +def call(...) + x = test(b, *a,) + x = perform(...) + x = perform a, @@b, defined?(c) + x = a.perform c::Const, Const +end \ No newline at end of file diff --git a/tests/parser/chef/files/array.rb b/tests/parser/chef/files/array.rb new file mode 100644 index 00000000..faf02204 --- /dev/null +++ b/tests/parser/chef/files/array.rb @@ -0,0 +1,12 @@ +y = [] +y = [one, two, three] +y = [*one, two, three] +y = %w[ + one + two + three + ] +y = %i[one two three] +y = %W[one two three] +y = %I[one two three] +z = %W[three#{four}] diff --git a/tests/parser/chef/files/begin.rb b/tests/parser/chef/files/begin.rb new file mode 100644 index 00000000..5d51fa0e --- /dev/null +++ b/tests/parser/chef/files/begin.rb @@ -0,0 +1,3 @@ +@service_enable_variable_name ||= +begin +end \ No newline at end of file diff --git a/tests/parser/chef/files/binary.rb b/tests/parser/chef/files/binary.rb new file mode 100644 index 00000000..80d8e534 --- /dev/null +++ b/tests/parser/chef/files/binary.rb @@ -0,0 +1,25 @@ +a = 1 + 2 +b = 1 - 2 +c = 1 * 2 +d = 1 / 2 +e = 1 % 2 +f = 1 ** 2 +g = 1 == 2 +h = 1 != 2 +i = 1 > 2 +j = 1 < 2 +k = 1 >= 2 +l = 1 <= 2 +m = 1 <=> 2 +n = 1 === 2 +o = 1 & 2 +p = 1 | 2 +q = 1 ^ 2 +r = 1 << 2 +s = 1 >> 2 +t = (1 and 2) +u = (1 or 2) +v = 1..2 +w = 1...2 +x = (1 && 2) +y = (1 || 2) diff --git a/tests/parser/chef/files/brace_block.rb b/tests/parser/chef/files/brace_block.rb new file mode 100644 index 00000000..f55f3a14 --- /dev/null +++ b/tests/parser/chef/files/brace_block.rb @@ -0,0 +1,3 @@ +describe file(cfg) do + it { should be_file } +end \ No newline at end of file diff --git a/tests/parser/chef/files/case.rb b/tests/parser/chef/files/case.rb new file mode 100644 index 00000000..9d1cb8d7 --- /dev/null +++ b/tests/parser/chef/files/case.rb @@ -0,0 +1,17 @@ +case value +when 1 + "one" +else + "number" +end + +case value +in 2 | 3 + "two or three" +else + "number" +end + +x = case value +in { key: } +end diff --git a/tests/parser/chef/files/class.rb b/tests/parser/chef/files/class.rb new file mode 100644 index 00000000..f31004d6 --- /dev/null +++ b/tests/parser/chef/files/class.rb @@ -0,0 +1,6 @@ +class Namespace::Container + x = 1 +end + +class << self +end \ No newline at end of file diff --git a/tests/parser/chef/files/def.rb b/tests/parser/chef/files/def.rb new file mode 100644 index 00000000..dc470cd4 --- /dev/null +++ b/tests/parser/chef/files/def.rb @@ -0,0 +1,9 @@ +def method(param, *rest, **kwargs) + x = 1 +end + +def object.method(param) + x = 1 +end + +->(value) { value * 2 } diff --git a/tests/parser/chef/files/do_block.rb b/tests/parser/chef/files/do_block.rb new file mode 100644 index 00000000..52165da4 --- /dev/null +++ b/tests/parser/chef/files/do_block.rb @@ -0,0 +1,6 @@ +describe "when locking the chef-client run", :unix_only => true do + let(:random_temp_root) do + Kernel.srand(Time.now.to_i + Process.pid) + "/tmp/#{Kernel.rand(Time.now.to_i + Process.pid)}" + end +end \ No newline at end of file diff --git a/tests/parser/chef/files/field.rb b/tests/parser/chef/files/field.rb new file mode 100644 index 00000000..723e583a --- /dev/null +++ b/tests/parser/chef/files/field.rb @@ -0,0 +1 @@ +new_resource.extension ||= defaults.extension \ No newline at end of file diff --git a/tests/parser/chef/files/hash.rb b/tests/parser/chef/files/hash.rb new file mode 100644 index 00000000..5b82fb6f --- /dev/null +++ b/tests/parser/chef/files/hash.rb @@ -0,0 +1,12 @@ +x = { + a => $d, + b => :"test", + c => `echo $a`, +} +y = {} +z = { **x } + +w = method(key1: value1, key2: value2) + +begin +end diff --git a/tests/parser/chef/files/if.rb b/tests/parser/chef/files/if.rb new file mode 100644 index 00000000..56d6f07f --- /dev/null +++ b/tests/parser/chef/files/if.rb @@ -0,0 +1,17 @@ +if variable + x = 1.0 + y = $v + z = 1 == 2 ? 3 : @four +elsif other_variable + a = (1 if 2) +else + b = 1i + first, second = value + value = first, *second +end + +url = if endpoint['url'] + endpoint['url'] +else + node['datadog']['url'] +end \ No newline at end of file diff --git a/tests/parser/chef/files/list_index_out_of_range.rb b/tests/parser/chef/files/list_index_out_of_range.rb new file mode 100644 index 00000000..e825f642 --- /dev/null +++ b/tests/parser/chef/files/list_index_out_of_range.rb @@ -0,0 +1,4 @@ +z = %W{--prefix=#{python['prefix_dir']}} +x = %i(start restart reload) +y = %I{start restart reload} +w = %w(start restart reload) diff --git a/tests/parser/chef/files/method_add_block.rb b/tests/parser/chef/files/method_add_block.rb new file mode 100644 index 00000000..968c8a0a --- /dev/null +++ b/tests/parser/chef/files/method_add_block.rb @@ -0,0 +1,11 @@ +ot = template ::File.join(node['icinga2']['objects_dir'], "#{resource_name}.conf") do + source "object.#{resource_name}.conf.erb" + cookbook 'icinga2' + owner node['icinga2']['user'] + group node['icinga2']['group'] + mode 0o640 + variables(:object => new_resource.name, + :path => new_resource.path, + :severity => new_resource.severity) + notifies platform?('windows') ? :restart : :reload, 'service[icinga2]' +end \ No newline at end of file diff --git a/tests/parser/chef/files/mix.rb b/tests/parser/chef/files/mix.rb new file mode 100644 index 00000000..071bed64 --- /dev/null +++ b/tests/parser/chef/files/mix.rb @@ -0,0 +1,16 @@ +alias test test2 +x = next value +x = (z until y) +x = (z while y) + +module Namespace +x = 0o640 +end + +result = (yield(key, val, match) or next) +Rubix.connection = ::Chef::RubixConnection::CONNECTIONS[ip] = connection +@release_ext ||= $2 +fonts_dir = WIN32OLE.new('WScript.Shell').SpecialFolders('Fonts') +VER_SUITE_BACKOFFICE = 0x0d000004 +eigen_class = class << self; self; end +@additional_remotes = Hash[] \ No newline at end of file diff --git a/tests/parser/chef/files/node_object_not_subscriptable.rb b/tests/parser/chef/files/node_object_not_subscriptable.rb new file mode 100644 index 00000000..6a52f9c4 --- /dev/null +++ b/tests/parser/chef/files/node_object_not_subscriptable.rb @@ -0,0 +1,45 @@ +include_recipe 'ufw' +# enable platform default firewall +firewall "ufw" do + action :enable +end + +if node.zabbix.server.install == true + # Search for some client + zabbix_clients = search(:node ,'recipes:zabbix') + + zabbix_clients.each do |client| + + # Accept connection from zabbix_agent on server + firewall_rule "zabbix_client_#{client[:fqdn]}" do + port 10051 + protocol :udp + source client[:ipaddress] + action :allow + end + + end if zabbix_clients + +end + +# Search for some client +zabbix_servers = search(:node ,'recipes:zabbix\:\:server') +if zabbix_servers + zabbix_servers.each do |server| + + # Accept connection from zabbix_agent on server + firewall_rule "zabbix_server_#{server[:fqdn]}" do + port 10050 + protocol :udp + source server[:ipaddress] + action :allow + end + + end if zabbix_servers +end + + +# enable platform default firewall +firewall "ufw" do + action :enable +end \ No newline at end of file diff --git a/tests/parser/chef/files/opassign.rb b/tests/parser/chef/files/opassign.rb new file mode 100644 index 00000000..3be8e6ed --- /dev/null +++ b/tests/parser/chef/files/opassign.rb @@ -0,0 +1,7 @@ +x += 2 +::Constant -= 2 +Constant *= ::Test +x /= super 2 +x %= super(2) +x **= 2r +x = x =~ /cow/ diff --git a/tests/parser/chef/files/rescue_mod.rb b/tests/parser/chef/files/rescue_mod.rb new file mode 100644 index 00000000..3cc9e33b --- /dev/null +++ b/tests/parser/chef/files/rescue_mod.rb @@ -0,0 +1 @@ +cassandra_config = data_bag_item('cassandra', 'clusters') rescue nil \ No newline at end of file diff --git a/tests/parser/chef/files/ruby_block.rb b/tests/parser/chef/files/ruby_block.rb new file mode 100644 index 00000000..2ffc8d96 --- /dev/null +++ b/tests/parser/chef/files/ruby_block.rb @@ -0,0 +1,8 @@ +ruby_block "zabbix_ensure_super_admin_user_with_api_access" do + block do + username = node.zabbix.api.username + first_name = 'Zabbix' + end + notifies :restart +end + diff --git a/tests/parser/chef/files/string_index_out_of_range.rb b/tests/parser/chef/files/string_index_out_of_range.rb new file mode 100644 index 00000000..5d437c86 --- /dev/null +++ b/tests/parser/chef/files/string_index_out_of_range.rb @@ -0,0 +1,17 @@ +def sevenzip_path_from_registry + begin + basepath = ::Win32::Registry::HKEY_LOCAL_MACHINE.open('SOFTWARE\Microsoft\Windows\CurrentVersion\App Paths\7zFM.exe').read_s('Path') + + # users like pretty errors + rescue ::Win32::Registry::Error + raise 'Failed to find the path of 7zip binary by searching checking HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Windows\CurrentVersion\App Paths\7zFM.exe\Path. Make sure to install 7zip before using this resource. If 7zip is installed and you still receive this message you can also specify the 7zip binary path by setting node["ark"]["sevenzip_binary"]' + end + "#{basepath}7z.exe" +end + +https_map = " + map $scheme $https { + default off; + https on; + } +" \ No newline at end of file diff --git a/tests/parser/chef/files/strings.rb b/tests/parser/chef/files/strings.rb new file mode 100644 index 00000000..7fc9d3fc --- /dev/null +++ b/tests/parser/chef/files/strings.rb @@ -0,0 +1,10 @@ +x = "test #{test} test" +y = "#@test" +z = <<~KEYGEN.gsub(/^ +/, '') +chmod 0600 #{my_home}/.ssh/id_dsa +chmod 0644 #{my_home}/.ssh/id_dsa.pub +KEYGEN +w = /.+ #{test} .+/ +y = "first" \ + "second" +k = `echo #{test}` \ No newline at end of file diff --git a/tests/parser/chef/files/unary.rb b/tests/parser/chef/files/unary.rb new file mode 100644 index 00000000..8a6eb295 --- /dev/null +++ b/tests/parser/chef/files/unary.rb @@ -0,0 +1,3 @@ +x = -2 +x = +2 +x = !false diff --git a/tests/parser/chef/files/unless.rb b/tests/parser/chef/files/unless.rb new file mode 100644 index 00000000..2ed314df --- /dev/null +++ b/tests/parser/chef/files/unless.rb @@ -0,0 +1,5 @@ +unless test + z = (value unless 1) +else + z = super +end \ No newline at end of file diff --git a/tests/parser/chef/files/valid_manifest.rb b/tests/parser/chef/files/valid_manifest.rb new file mode 100644 index 00000000..a92787b4 --- /dev/null +++ b/tests/parser/chef/files/valid_manifest.rb @@ -0,0 +1,7 @@ +my_home = "/home/test" + +execute "create ssh keypair for #{new_resource.username}" do + user new_resource.username + command "test" + action :nothing +end \ No newline at end of file diff --git a/tests/parser/chef/test_parser.py b/tests/parser/chef/test_parser.py new file mode 100644 index 00000000..66fb8ba0 --- /dev/null +++ b/tests/parser/chef/test_parser.py @@ -0,0 +1,1020 @@ +from glitch.parsers.chef import ChefParser +from glitch.repr.inter import * +from tests.parser.test_parser import TestParser + + +class TestChefParser(TestParser): + def __parse(self, file: str) -> UnitBlock: + p = ChefParser() + ir = p.parse_file(file, UnitBlockType.script) + assert ir is not None + assert isinstance(ir, UnitBlock) + assert ir.type == UnitBlockType.script + return ir + + def __check_code_element( + self, + code_element: CodeElement, + line: int, + column: int, + end_line: int, + end_column: int, + ) -> None: + assert code_element.line == line + assert code_element.end_line == end_line + assert code_element.column == column + assert code_element.end_column == end_column + + def test_chef_parser_valid_manifest(self) -> None: + """ + string_literal | regexp_literal | vcall | symbol_literal | @ident + call | args_add_block | arg_paren | @period | method_add_arg + """ + + ir = self.__parse("tests/parser/chef/files/valid_manifest.rb") + assert len(ir.variables) == 1 + assert len(ir.atomic_units) == 1 + + assert isinstance(ir.variables[0], Variable) + assert ir.variables[0].name == "my_home" + self._check_value(ir.variables[0].value, String, "/home/test", 1, 11, 1, 23) + assert ir.variables[0].line == 1 + assert ir.variables[0].column == 1 + assert ir.variables[0].end_line == 1 + assert ir.variables[0].end_column == 23 + + assert isinstance(ir.atomic_units[0], AtomicUnit) + assert isinstance(ir.atomic_units[0].name, Sum) + assert isinstance(ir.atomic_units[0].name.left, String) + assert ir.atomic_units[0].name.left.value == "create ssh keypair for " + assert isinstance(ir.atomic_units[0].name.right, VariableReference) + assert ir.atomic_units[0].name.right.value == "new_resource.username" + assert ir.atomic_units[0].type == "execute" + assert len(ir.atomic_units[0].attributes) == 3 + + assert isinstance(ir.atomic_units[0].attributes[0], Attribute) + self.__check_code_element(ir.atomic_units[0].attributes[0], 4, 5, 4, 36) + assert ir.atomic_units[0].attributes[0].name == "user" + assert isinstance(ir.atomic_units[0].attributes[0].value, MethodCall) + self._check_value( + ir.atomic_units[0].attributes[0].value.receiver, + VariableReference, + "new_resource", + 4, + 15, + 4, + 27, + ) + assert ir.atomic_units[0].attributes[0].value.method == "username" + assert len(ir.atomic_units[0].attributes[0].value.args) == 0 + + assert isinstance(ir.atomic_units[0].attributes[1], Attribute) + self.__check_code_element(ir.atomic_units[0].attributes[1], 5, 5, 5, 21) + assert ir.atomic_units[0].attributes[1].name == "command" + self._check_value( + ir.atomic_units[0].attributes[1].value, + String, + "test", + 5, + 15, + 5, + 21, + ) + + assert isinstance(ir.atomic_units[0].attributes[2], Attribute) + self.__check_code_element(ir.atomic_units[0].attributes[2], 6, 5, 6, 23) + assert ir.atomic_units[0].attributes[2].name == "action" + self._check_value( + ir.atomic_units[0].attributes[2].value, + VariableReference, + ":nothing", + 6, + 15, + 6, + 23, + ) + + def test_chef_parser_mix(self) -> None: + """ + alias | next | module | oct integer | yield | assign | backref | @const + binary | sclass + """ + # TODO: support alias + ir = self.__parse("tests/parser/chef/files/mix.rb") + assert len(ir.variables) == 10 + assert len(ir.unit_blocks) == 1 + + assert isinstance(ir.variables[0], Variable) + assert ir.variables[0].name == "x" + assert isinstance(ir.variables[0].value, FunctionCall) + assert ir.variables[0].value.name == "next" + self.__check_code_element( + ir.variables[0].value, 2, 10, 2, 15 + ) # FIXME: should be 2, 5, 2, 14 + + assert isinstance(ir.variables[1], Variable) + assert isinstance(ir.variables[1].value, Null) # FIXME + + assert isinstance(ir.variables[2], Variable) + assert isinstance(ir.variables[2].value, Null) # FIXME + + assert isinstance(ir.variables[3], Variable) + assert isinstance(ir.variables[3].value, Or) + + assert isinstance(ir.variables[4], Variable) + assert isinstance(ir.variables[4].value, Assign) + + assert isinstance(ir.variables[5], Variable) + assert isinstance(ir.variables[5].value, ConditionalStatement) + assert ir.variables[5].value.is_top == True + + assert isinstance(ir.variables[6], Variable) + assert isinstance(ir.variables[6].value, MethodCall) + + assert isinstance(ir.variables[7], Variable) + assert isinstance(ir.variables[7].value, Integer) + assert ir.variables[7].value.value == 0x0D000004 + + assert isinstance(ir.variables[8], Variable) + assert isinstance(ir.variables[8].value, Null) # FIXME + + assert isinstance(ir.unit_blocks[0], UnitBlock) + assert ir.unit_blocks[0].type == UnitBlockType.block + assert ir.unit_blocks[0].name == "Namespace" + assert len(ir.unit_blocks[0].variables) == 1 + self._check_value( + ir.unit_blocks[0].variables[0].value, Integer, 0o640, 7, 5, 7, 10 + ) + + def test_chef_parser_aref(self) -> None: + """ + aref | aref_field | @int | ||= + """ + ir = self.__parse("tests/parser/chef/files/aref.rb") + assert len(ir.variables) == 2 + assert isinstance(ir.variables[0], Variable) + self.__check_code_element(ir.variables[0], 1, 1, 1, 30) + assert ir.variables[0].name == "collection[0]" + self._check_binary_operation( + ir.variables[0].value, + Access, + VariableReference("collection", ElementInfo(1, 17, 1, 27, "collection")), + Integer(1, ElementInfo(1, 28, 1, 29, "1")), + 1, + 17, + 1, + 30, + ) + + assert isinstance(ir.variables[1], Variable) + assert isinstance(ir.variables[1].value, ConditionalStatement) + assert ir.variables[1].value.is_top == True + + def test_chef_parser_args_add(self) -> None: + """ + args_add_star | args_forward | fcall | tstring_content | + command | command_call | const_path | const_path_ref | cvar + """ + ir = self.__parse("tests/parser/chef/files/args_add.rb") + assert len(ir.unit_blocks) == 1 + assert len(ir.unit_blocks[0].variables) == 4 + + assert isinstance(ir.unit_blocks[0].variables[0], Variable) + assert ir.unit_blocks[0].variables[0].name == "x" + self.__check_code_element(ir.unit_blocks[0].variables[0], 2, 5, 2, 21) + assert isinstance(ir.unit_blocks[0].variables[0].value, FunctionCall) + assert len(ir.unit_blocks[0].variables[0].value.args) == 2 + for i in range(2): + assert isinstance( + ir.unit_blocks[0].variables[0].value.args[i], VariableReference + ) + + assert isinstance(ir.unit_blocks[0].variables[1], Variable) + assert ir.unit_blocks[0].variables[1].name == "x" + self.__check_code_element( + ir.unit_blocks[0].variables[1], 3, 5, 3, 16 + ) # FIXME should be 20 + assert isinstance(ir.unit_blocks[0].variables[1].value, FunctionCall) + assert len(ir.unit_blocks[0].variables[1].value.args) == 1 + assert isinstance(ir.unit_blocks[0].variables[1].value.args[0], Null) + + assert isinstance(ir.unit_blocks[0].variables[2], Variable) + assert ir.unit_blocks[0].variables[2].name == "x" + self.__check_code_element(ir.unit_blocks[0].variables[2], 4, 5, 4, 36) + assert isinstance(ir.unit_blocks[0].variables[2].value, FunctionCall) + assert len(ir.unit_blocks[0].variables[2].value.args) == 3 + assert isinstance(ir.unit_blocks[0].variables[2].value.args[2], FunctionCall) + assert ir.unit_blocks[0].variables[2].value.args[2].name == "defined" + + assert isinstance(ir.unit_blocks[0].variables[3], Variable) + assert ir.unit_blocks[0].variables[3].name == "x" + self.__check_code_element(ir.unit_blocks[0].variables[3], 5, 5, 5, 34) + assert isinstance(ir.unit_blocks[0].variables[3].value, MethodCall) + assert len(ir.unit_blocks[0].variables[3].value.args) == 2 + + def test_chef_parser_array(self) -> None: + """ + array | qwords_add | qwords_new | qsymbols_add | qsymbols_new + words_add | words_new | word_add | symbols_add | symbols_new | string_embexpr + """ + ir = self.__parse("tests/parser/chef/files/array.rb") + assert len(ir.variables) == 8 + for i in range(7): + assert isinstance(ir.variables[i], Variable) + assert ir.variables[i].name == "y" + + self.__check_code_element( + ir.variables[0], 1, 1, 1, 2 + ) # FIXME should be 1, 1, 1, 7 + self.__check_code_element(ir.variables[1], 2, 1, 2, 22) + self.__check_code_element(ir.variables[2], 3, 1, 3, 23) + self.__check_code_element(ir.variables[3], 4, 1, 8, 6) + for i in range(4, 7): + self.__check_code_element(ir.variables[i], i + 5, 1, i + 5, 22) + + for i in range(7): + assert isinstance(ir.variables[i].value, Array) + + assert len(ir.variables[0].value.value) == 0 # type: ignore + for i in range(1, 7): + assert len(ir.variables[i].value.value) == 3 # type: ignore + + for i in range(3): + assert isinstance(ir.variables[1].value.value[i], VariableReference) # type: ignore + assert isinstance(ir.variables[2].value.value[i], VariableReference) # type: ignore + assert isinstance(ir.variables[3].value.value[i], String) # type: ignore + assert isinstance(ir.variables[4].value.value[i], String) # type: ignore + assert isinstance(ir.variables[5].value.value[i], String) # type: ignore + assert isinstance(ir.variables[6].value.value[i], String) # type: ignore + + assert isinstance(ir.variables[7], Variable) + assert isinstance(ir.variables[7].value, Array) + assert len(ir.variables[7].value.value) == 1 + self._check_binary_operation( + ir.variables[7].value.value[0], + Sum, + String("three", ElementInfo(12, 8, 12, 13, "three")), + VariableReference("four", ElementInfo(12, 15, 12, 19, "four")), + 12, + 8, + 12, + 20, + ) + + def test_chef_parser_hash(self) -> None: + """ + hash | assoclist_from_args | assoc_splat | assoc_new | var_ref | backref | + backtick | xstring_literal | bare_assoc_hash | begin | end + """ + ir = self.__parse("tests/parser/chef/files/hash.rb") + assert len(ir.variables) == 4 + assert isinstance(ir.variables[0], Variable) + assert ir.variables[0].name == "x" + self.__check_code_element(ir.variables[0], 1, 1, 5, 2) + assert isinstance(ir.variables[0].value, Hash) + self.__check_code_element(ir.variables[0].value, 1, 5, 5, 2) + + assert len(ir.variables[0].value.value) == 3 + for i in range(3): + assert isinstance( + list(ir.variables[0].value.value.keys())[i], VariableReference + ) + assert isinstance( + list(ir.variables[0].value.value.values())[0], VariableReference + ) + assert isinstance(list(ir.variables[0].value.value.values())[1], String) + self._check_value( + list(ir.variables[0].value.value.values())[2], + String, + "echo $a", + 4, + 10, + 4, + 19, + ) + + assert isinstance(ir.variables[1], Variable) + assert ir.variables[1].name == "y" + self.__check_code_element( + ir.variables[1], 6, 1, 6, 2 + ) # FIXME: should be 5, 1, 5, 7 + assert isinstance(ir.variables[1].value, Hash) + assert len(ir.variables[1].value.value) == 0 + + assert isinstance(ir.variables[2], Variable) + assert ir.variables[2].name == "z" + self.__check_code_element(ir.variables[2], 7, 1, 7, 12) + assert isinstance(ir.variables[2].value, Hash) + assert len(ir.variables[2].value.value) == 1 + # FIXME: splat + assert isinstance( + list(ir.variables[2].value.value.keys())[0], VariableReference + ) + assert isinstance( + list(ir.variables[2].value.value.values())[0], VariableReference + ) + + assert isinstance(ir.variables[3], Variable) + assert ir.variables[3].name == "w" + self.__check_code_element(ir.variables[3], 9, 1, 9, 39) + assert isinstance(ir.variables[3].value, FunctionCall) + assert isinstance(ir.variables[3].value.args[0], Hash) + + def test_chef_parser_binary(self) -> None: + """ + binary | dot2 | dot3 | paren | stmts_add + """ + ir = self.__parse("tests/parser/chef/files/binary.rb") + assert len(ir.variables) == 25 + for i in range(25): + assert isinstance(ir.variables[i], Variable) + assert ir.variables[i].name == chr(ord("a") + i) + + tests = [ + (Sum, 0), + (Subtract, 0), + (Multiply, 0), + (Divide, 0), + (Modulo, 0), + (Power, 1), + (Equal, 1), + (NotEqual, 1), + (GreaterThan, 0), + (LessThan, 0), + (GreaterThanOrEqual, 1), + (LessThanOrEqual, 1), + (None, 1), # FIXME + (Equal, 2), + (BitwiseAnd, 0), + (BitwiseOr, 0), + (BitwiseXor, 0), + (LeftShift, 1), + (RightShift, 1), + (And, 2), + (Or, 1), + ] + for i, test in enumerate(tests): + type, offset = test + paren_offset = 0 if type not in (And, Or) else 1 + if type is None: + continue + + self._check_binary_operation( + ir.variables[i].value, + type, + Integer( + 1, + ElementInfo(i + 1, 5 + paren_offset, i + 1, 6 + paren_offset, "1"), + ), + Integer( + 2, + ElementInfo( + i + 1, + 9 + offset + paren_offset, + i + 1, + 10 + offset + paren_offset, + "2", + ), + ), + i + 1, + 5 + paren_offset, + i + 1, + 10 + offset + paren_offset, + ) + assert isinstance(ir.variables[21].value, FunctionCall) + assert ir.variables[21].value.name == "range" + assert isinstance(ir.variables[22].value, FunctionCall) + assert ir.variables[22].value.name == "range" + + def test_chef_parser_case(self) -> None: + """ + case | when | else | in | hshptn + """ + ir = self.__parse("tests/parser/chef/files/case.rb") + assert len(ir.statements) == 2 + assert isinstance(ir.statements[0], ConditionalStatement) + assert ir.statements[0].is_top == True + assert ir.statements[0].condition.line == 1 + assert isinstance(ir.statements[1], ConditionalStatement) + assert ir.statements[1].is_top == True + assert ir.statements[1].condition.line == 8 + assert ir.statements[0].type == ConditionalStatement.ConditionType.SWITCH + assert ir.statements[1].type == ConditionalStatement.ConditionType.SWITCH + + self._check_binary_operation( + ir.statements[0].condition, + Equal, + VariableReference("value", ElementInfo(1, 6, 1, 11, "value")), + Integer(1, ElementInfo(2, 6, 2, 7, "1")), + 1, + 6, + 1, + 11, + ) + assert isinstance(ir.statements[0].else_statement, ConditionalStatement) + assert ir.statements[0].else_statement.else_statement is None + + assert len(ir.statements[0].statements) == 1 + assert isinstance(ir.statements[0].statements[0], String) + assert ir.statements[0].statements[0].value == "one" + + assert len(ir.statements[0].else_statement.statements) == 1 + assert isinstance(ir.statements[0].else_statement.statements[0], String) + assert ir.statements[0].else_statement.statements[0].value == "number" + + self._check_binary_operation( + ir.statements[1].condition, + Equal, + VariableReference("value", ElementInfo(8, 6, 8, 11, "value")), + BitwiseOr( + ElementInfo(9, 4, 9, 9, "2 | 3"), + Integer(2, ElementInfo(9, 4, 9, 5, "2")), + Integer(3, ElementInfo(9, 8, 9, 9, "3")), + ), + 8, + 6, + 8, + 11, + ) + assert isinstance(ir.statements[1].else_statement, ConditionalStatement) + assert ir.statements[1].else_statement.else_statement is None + + assert len(ir.statements[1].statements) == 1 + assert isinstance(ir.statements[1].statements[0], String) + assert ir.statements[1].statements[0].value == "two or three" + + assert len(ir.statements[1].else_statement.statements) == 1 + assert isinstance(ir.statements[1].else_statement.statements[0], String) + assert ir.statements[1].else_statement.statements[0].value == "number" + + assert len(ir.variables) == 1 + assert isinstance(ir.variables[0].value, ConditionalStatement) + assert ir.variables[0].value.is_top == True + + def test_chef_parser_if(self) -> None: + """ + if | elsif | else | float | if_mod | ifop | imaginary | ivar | massign + mlhs_add | mlhs_new | mrhs_add | mrhs_new | mrhs_new_from_args | mrhs_add_star + """ + ir = self.__parse("tests/parser/chef/files/if.rb") + assert len(ir.statements) == 1 + + assert isinstance(ir.statements[0], ConditionalStatement) + assert ir.statements[0].is_top == True + assert ir.statements[0].type == ConditionalStatement.ConditionType.IF + assert isinstance(ir.statements[0].condition, VariableReference) + assert len(ir.statements[0].statements) == 3 + + assert isinstance(ir.statements[0].statements[0], Variable) + self._check_value(ir.statements[0].statements[0].value, Float, 1.0, 2, 9, 2, 12) + + assert isinstance(ir.statements[0].statements[1], Variable) + self._check_value( + ir.statements[0].statements[1].value, VariableReference, "$v", 3, 9, 3, 11 + ) + + assert isinstance(ir.statements[0].statements[2], Variable) + assert isinstance(ir.statements[0].statements[2].value, ConditionalStatement) + assert ir.statements[0].statements[2].value.is_top == True + assert ( + ir.statements[0].statements[2].value.type + == ConditionalStatement.ConditionType.IF + ) + assert len(ir.statements[0].statements[2].value.statements) == 1 + assert isinstance(ir.statements[0].statements[2].value.statements[0], Integer) + assert isinstance( + ir.statements[0].statements[2].value.else_statement, ConditionalStatement + ) + assert ( + ir.statements[0].statements[2].value.else_statement.else_statement is None + ) + assert isinstance( + ir.statements[0].statements[2].value.else_statement.statements[0], + VariableReference, + ) + + assert isinstance(ir.statements[0].else_statement, ConditionalStatement) + assert ( + ir.statements[0].else_statement.type + == ConditionalStatement.ConditionType.IF + ) + assert isinstance(ir.statements[0].else_statement.condition, VariableReference) + assert len(ir.statements[0].else_statement.statements) == 1 + assert isinstance(ir.statements[0].else_statement.statements[0], Variable) + assert isinstance( + ir.statements[0].else_statement.statements[0].value, ConditionalStatement + ) + assert ( + ir.statements[0].else_statement.statements[0].value.else_statement is None + ) + + else_statement = ir.statements[0].else_statement.else_statement + assert isinstance(else_statement, ConditionalStatement) + assert else_statement.else_statement is None + assert len(else_statement.statements) == 3 + + assert isinstance(else_statement.statements[0], Variable) + self._check_value(else_statement.statements[0].value, Complex, 1j, 8, 9, 8, 11) + + assert isinstance(else_statement.statements[1], Variable) + assert else_statement.statements[1].name == "first, second" + + assert isinstance(else_statement.statements[2], Variable) + assert else_statement.statements[2].name == "value" + assert isinstance(else_statement.statements[2].value, AddArgs) # FIXME + assert len(else_statement.statements[2].value.value) == 2 + + assert len(ir.variables) == 1 + assert isinstance(ir.variables[0].value, ConditionalStatement) + + def test_chef_parser_opassign(self) -> None: + """ + opassign | rational | super | top_const_ref | top_const_field + """ + ir = self.__parse("tests/parser/chef/files/opassign.rb") + assert len(ir.variables) == 7 + + assert isinstance(ir.variables[0], Variable) + assert ir.variables[0].name == "x" + self._check_binary_operation( + ir.variables[0].value, + Sum, + VariableReference("x", ElementInfo(1, 1, 1, 2, "x")), + Integer(2, ElementInfo(1, 6, 1, 7, "2")), + 1, + 1, + 1, + 7, + ) + + assert isinstance(ir.variables[1], Variable) + assert ir.variables[1].name == "::Constant" + self._check_binary_operation( + ir.variables[1].value, + Subtract, + VariableReference("::Constant", ElementInfo(2, 1, 2, 11, "::Constant")), + Integer(2, ElementInfo(2, 15, 2, 16, "2")), + 2, + 1, + 2, + 16, + ) + + assert isinstance(ir.variables[2], Variable) + assert ir.variables[2].name == "Constant" + self._check_binary_operation( + ir.variables[2].value, + Multiply, + VariableReference("Constant", ElementInfo(3, 1, 3, 9, "Constant")), + VariableReference("::Test", ElementInfo(3, 13, 3, 19, "::Test")), + 3, + 1, + 3, + 19, + ) + + assert isinstance(ir.variables[3], Variable) + assert ir.variables[3].name == "x" + self._check_binary_operation( + ir.variables[3].value, + Divide, + VariableReference("x", ElementInfo(4, 1, 4, 2, "x")), + FunctionCall( + "super", + [Integer(2, ElementInfo(4, 12, 4, 13, "2"))], + ElementInfo(4, 12, 4, 13, "super 2"), + ), + 4, + 1, + 4, + 13, + ) + assert ir.variables[3].value.right.name == "super" # type: ignore + + assert isinstance(ir.variables[4], Variable) + assert ir.variables[4].name == "x" + self._check_binary_operation( + ir.variables[4].value, + Modulo, + VariableReference("x", ElementInfo(5, 1, 5, 2, "x")), + FunctionCall( + "super", + [Integer(2, ElementInfo(5, 12, 5, 13, "2"))], + ElementInfo(5, 11, 5, 14, "super(2)"), + ), + 5, + 1, + 5, + 14, + ) + assert ir.variables[4].value.right.name == "super" # type: ignore + + assert isinstance(ir.variables[5], Variable) + assert ir.variables[5].name == "x" + self._check_binary_operation( + ir.variables[5].value, + Power, + VariableReference("x", ElementInfo(6, 1, 6, 2, "x")), + Float(2, ElementInfo(6, 7, 6, 9, "2r")), + 6, + 1, + 6, + 9, + ) + + assert isinstance(ir.variables[6], Variable) + assert ir.variables[6].name == "x" + # TODO + assert isinstance(ir.variables[6].value, Null) + + def test_chef_parser_unary(self) -> None: + """ + unary + """ + ir = self.__parse("tests/parser/chef/files/unary.rb") + assert len(ir.variables) == 3 + + assert isinstance(ir.variables[0], Variable) + assert ir.variables[0].name == "x" + assert isinstance(ir.variables[0].value, Minus) + + assert isinstance(ir.variables[1], Variable) + assert ir.variables[1].name == "x" + self._check_value(ir.variables[1].value, Integer, 2, 2, 5, 2, 7) + + assert isinstance(ir.variables[2], Variable) + assert ir.variables[2].name == "x" + assert isinstance(ir.variables[2].value, Not) + + def test_chef_parser_unless(self) -> None: + """ + unless | unless_mod + """ + ir = self.__parse("tests/parser/chef/files/unless.rb") + assert len(ir.statements) == 1 + assert isinstance(ir.statements[0], ConditionalStatement) + assert ir.statements[0].is_top == True + assert ir.statements[0].type == ConditionalStatement.ConditionType.IF + assert isinstance(ir.statements[0].condition, Not) + assert isinstance(ir.statements[0].condition.expr, VariableReference) + assert len(ir.statements[0].statements) == 1 + assert isinstance(ir.statements[0].statements[0], Variable) + assert isinstance(ir.statements[0].statements[0].value, ConditionalStatement) + assert ( + ir.statements[0].statements[0].value.type + == ConditionalStatement.ConditionType.IF + ) + assert isinstance(ir.statements[0].statements[0].value.condition, Not) + + assert isinstance(ir.statements[0].else_statement, ConditionalStatement) + assert ( + ir.statements[0].else_statement.type + == ConditionalStatement.ConditionType.IF + ) + assert ir.statements[0].else_statement.else_statement is None + assert len(ir.statements[0].else_statement.statements) == 1 + assert isinstance(ir.statements[0].else_statement.statements[0], Variable) + assert isinstance( + ir.statements[0].else_statement.statements[0].value, FunctionCall + ) + assert ir.statements[0].else_statement.statements[0].value.name == "zsuper" + + def test_chef_parser_class(self) -> None: + """ + class + """ + ir = self.__parse("tests/parser/chef/files/class.rb") + assert len(ir.unit_blocks) == 2 + + assert isinstance(ir.unit_blocks[0], UnitBlock) + assert ir.unit_blocks[0].type == UnitBlockType.definition + assert ir.unit_blocks[0].name == "Namespace::Container" + assert len(ir.unit_blocks[0].variables) == 1 + assert ir.unit_blocks[0].variables[0].name == "x" + self._check_value(ir.unit_blocks[0].variables[0].value, Integer, 1, 2, 9, 2, 10) + + assert isinstance(ir.unit_blocks[1], UnitBlock) + assert ir.unit_blocks[1].type == UnitBlockType.definition + assert ir.unit_blocks[1].name == "self" + + def test_chef_parser_def(self) -> None: + """ + def | defs | lambda + """ + ir = self.__parse("tests/parser/chef/files/def.rb") + assert len(ir.unit_blocks) == 2 + + assert isinstance(ir.unit_blocks[0], UnitBlock) + assert ir.unit_blocks[0].type == UnitBlockType.definition + assert ir.unit_blocks[0].name == "method" + assert len(ir.unit_blocks[0].variables) == 1 + assert ir.unit_blocks[0].variables[0].name == "x" + self._check_value(ir.unit_blocks[0].variables[0].value, Integer, 1, 2, 9, 2, 10) + + assert isinstance(ir.unit_blocks[1], UnitBlock) + assert ir.unit_blocks[1].type == UnitBlockType.definition + assert ir.unit_blocks[1].name == "object.method" + assert len(ir.unit_blocks[1].variables) == 1 + assert ir.unit_blocks[1].variables[0].name == "x" + self._check_value(ir.unit_blocks[1].variables[0].value, Integer, 1, 6, 9, 6, 10) + + def test_chef_parser_strings(self) -> None: + """ + string_literal | string_add | string_embexpr | string_content + string_dvar | embvar | regexp_add | regexp_literal + """ + ir = self.__parse("tests/parser/chef/files/strings.rb") + assert len(ir.variables) == 6 + + assert isinstance(ir.variables[0], Variable) + assert isinstance(ir.variables[0].value, Sum) + assert isinstance(ir.variables[0].value.left, Sum) + self._check_value(ir.variables[0].value.left.left, String, "test ", 1, 6, 1, 11) + self._check_value( + ir.variables[0].value.left.right, VariableReference, "test", 1, 13, 1, 17 + ) + self._check_value(ir.variables[0].value.right, String, " test", 1, 18, 1, 23) + + assert isinstance(ir.variables[1], Variable) + self._check_value( + ir.variables[1].value, VariableReference, "@test", 2, 7, 2, 12 + ) + + assert isinstance(ir.variables[2], Variable) + assert isinstance(ir.variables[2].value, MethodCall) + assert ir.variables[2].value.method == "gsub" + assert len(ir.variables[2].value.args) == 2 + self._check_value(ir.variables[2].value.args[0], String, "^ +", 3, 21, 3, 25) + assert isinstance(ir.variables[2].value.args[1], String) + assert ir.variables[2].value.args[1].value == "" + + assert isinstance(ir.variables[2].value.receiver, Sum) + + assert isinstance(ir.variables[2].value.receiver.left, Sum) + self._check_value( + ir.variables[2].value.receiver.right, + String, + "/.ssh/id_dsa.pub\n", + 5, + 22, + 6, + 1, + ) + + assert isinstance(ir.variables[2].value.receiver.left.left, Sum) + self._check_value( + ir.variables[2].value.receiver.left.right, + VariableReference, + "my_home", + 5, + 14, + 5, + 21, + ) + + assert isinstance(ir.variables[2].value.receiver.left.left.left, Sum) + self._check_value( + ir.variables[2].value.receiver.left.left.right, + String, + "/.ssh/id_dsa\nchmod 0644 ", + 4, + 22, + 5, + 12, + ) + + self._check_value( + ir.variables[2].value.receiver.left.left.left.right, + VariableReference, + "my_home", + 4, + 14, + 4, + 21, + ) + self._check_value( + ir.variables[2].value.receiver.left.left.left.left, + String, + "chmod 0600 ", + 4, + 1, + 4, + 12, + ) + + assert isinstance(ir.variables[3], Variable) + assert isinstance(ir.variables[3].value, Sum) + self._check_value(ir.variables[3].value.right, String, " .+", 7, 16, 7, 19) + + assert isinstance(ir.variables[3].value.left, Sum) + self._check_value( + ir.variables[3].value.left.right, VariableReference, "test", 7, 11, 7, 15 + ) + self._check_value(ir.variables[3].value.left.left, String, ".+ ", 7, 6, 7, 9) + + assert isinstance(ir.variables[4], Variable) + assert isinstance(ir.variables[4].value, Sum) + self.__check_code_element(ir.variables[4].value, 8, 5, 9, 17) + assert isinstance(ir.variables[4].value.left, String) + assert isinstance(ir.variables[4].value.right, String) + + assert isinstance(ir.variables[5], Variable) + assert isinstance(ir.variables[5].value, Sum) + self._check_value(ir.variables[5].value.left, String, "echo ", 10, 6, 10, 11) + self._check_value( + ir.variables[5].value.right, VariableReference, "test", 10, 13, 10, 17 + ) + + def test_chef_parser_string_index_out_of_range(self) -> None: + """ + This file used to file with string index out of range. + """ + self.__parse("tests/parser/chef/files/string_index_out_of_range.rb") + + def test_chef_parser_list_index_out_of_range(self) -> None: + """ + This file used to file with list index out of range. + """ + self.__parse("tests/parser/chef/files/list_index_out_of_range.rb") + + def test_chef_parser_node_object_not_subscriptable(self) -> None: + """ + This file used to file with node object not subscriptable. + """ + self.__parse("tests/parser/chef/files/node_object_not_subscriptable.rb") + + def test_chef_parser_do_block(self) -> None: + """ + do_block + """ + ir = self.__parse("tests/parser/chef/files/do_block.rb") + assert len(ir.atomic_units) == 1 + assert isinstance(ir.atomic_units[0], AtomicUnit) + assert ir.atomic_units[0].type == "describe" + assert isinstance(ir.atomic_units[0].name, AddArgs) + assert len(ir.atomic_units[0].attributes) == 1 + assert isinstance(ir.atomic_units[0].attributes[0], Attribute) + assert ir.atomic_units[0].attributes[0].name == "let(:random_temp_root)" + assert isinstance(ir.atomic_units[0].attributes[0].value, BlockExpr) + + def test_chef_parser_brace_block(self) -> None: + """ + brace_block + """ + ir = self.__parse("tests/parser/chef/files/brace_block.rb") + assert len(ir.atomic_units) == 1 + assert isinstance(ir.atomic_units[0], AtomicUnit) + assert ir.atomic_units[0].type == "describe" + assert isinstance(ir.atomic_units[0].name, FunctionCall) + assert len(ir.atomic_units[0].attributes) == 1 + assert isinstance(ir.atomic_units[0].attributes[0], Attribute) + assert ir.atomic_units[0].attributes[0].name == "it" + assert isinstance(ir.atomic_units[0].attributes[0].value, BlockExpr) + + def test_chef_parser_rescue_mod(self) -> None: + """ + rescue_mod + """ + ir = self.__parse("tests/parser/chef/files/rescue_mod.rb") + assert len(ir.variables) == 1 + assert isinstance(ir.variables[0], Variable) + assert ir.variables[0].name == "cassandra_config" + assert isinstance(ir.variables[0].value, ConditionalStatement) + assert ir.variables[0].value.is_top == True + + def test_chef_parser_method_add_block(self) -> None: + """ + method_add_block + """ + ir = self.__parse("tests/parser/chef/files/method_add_block.rb") + assert len(ir.variables) == 1 + assert isinstance(ir.variables[0], Variable) + assert isinstance(ir.variables[0].value, MethodCall) + assert ir.variables[0].value.method == "" + assert len(ir.variables[0].value.args) == 1 + assert isinstance(ir.variables[0].value.args[0], BlockExpr) + assert len(ir.variables[0].value.args[0].statements) == 7 + + def test_chef_parser_ruby_block(self) -> None: + """ + ruby_block with block and notifies + """ + ir = self.__parse("tests/parser/chef/files/ruby_block.rb") + assert len(ir.atomic_units) == 1 + assert isinstance(ir.atomic_units[0], AtomicUnit) + assert ir.atomic_units[0].type == "ruby_block" + assert isinstance(ir.atomic_units[0].name, String) + assert ( + ir.atomic_units[0].name.value + == "zabbix_ensure_super_admin_user_with_api_access" + ) + assert len(ir.atomic_units[0].attributes) == 2 + assert isinstance(ir.atomic_units[0].attributes[0], Attribute) + assert ir.atomic_units[0].attributes[0].name == "block" + assert isinstance(ir.atomic_units[0].attributes[0].value, BlockExpr) + assert len(ir.atomic_units[0].attributes[0].value.statements) == 2 + assert isinstance( + ir.atomic_units[0].attributes[0].value.statements[0], Variable + ) + assert ir.atomic_units[0].attributes[0].value.statements[0].name == "username" + assert isinstance( + ir.atomic_units[0].attributes[0].value.statements[0].value, MethodCall + ) + assert ( + ir.atomic_units[0].attributes[0].value.statements[0].value.method + == "username" + ) + assert isinstance(ir.atomic_units[0].attributes[1], Attribute) + assert ir.atomic_units[0].attributes[1].name == "notifies" + self._check_value( + ir.atomic_units[0].attributes[1].value, + VariableReference, + ":restart", + 6, + 12, + 6, + 20, + ) + + def test_chef_parser_begin(self) -> None: + """ + begin + """ + # TODO: For now just checks if it does not crash + self.__parse("tests/parser/chef/files/begin.rb") + + def test_chef_parser_arg_paren(self) -> None: + """ + arg_paren + """ + ir = self.__parse("tests/parser/chef/files/arg_paren.rb") + assert len(ir.variables) == 1 + assert isinstance(ir.variables[0], Variable) + assert isinstance(ir.variables[0].value, FunctionCall) + + def test_chef_parser_field(self) -> None: + """ + field + """ + ir = self.__parse("tests/parser/chef/files/field.rb") + assert len(ir.variables) == 1 + assert isinstance(ir.variables[0], Variable) + + def test_chef_parser_apache_module_case(self) -> None: + """ + apache_module resource with case statement inside + """ + ir = self.__parse("tests/parser/chef/files/apache_module_case.rb") + assert len(ir.atomic_units) == 1 + assert isinstance(ir.atomic_units[0], AtomicUnit) + assert ir.atomic_units[0].type == "apache_module" + assert isinstance(ir.atomic_units[0].name, String) + assert ir.atomic_units[0].name.value == "php5" + assert len(ir.atomic_units[0].statements) == 1 + assert isinstance(ir.atomic_units[0].statements[0], ConditionalStatement) + assert ( + ir.atomic_units[0].statements[0].type + == ConditionalStatement.ConditionType.SWITCH + ) + assert ir.atomic_units[0].statements[0].is_top == True + + case_stmt = ir.atomic_units[0].statements[0] + assert isinstance(case_stmt.condition, Equal) + assert isinstance(case_stmt.condition.left, Access) + assert isinstance(case_stmt.condition.left.left, VariableReference) + assert case_stmt.condition.left.left.value == "node" + assert isinstance(case_stmt.condition.left.right, String) + assert case_stmt.condition.left.right.value == "platform_family" + + assert isinstance(case_stmt.condition.right, AddArgs) + assert len(case_stmt.statements) == 2 + assert isinstance(case_stmt.statements[0], Attribute) + assert isinstance(case_stmt.statements[1], Attribute) + + +# TODO: +# block_var +# blockarg +# break +# aryptn +# class +# const_ref +# const_path_field +# fndptn +# for +# mlhs_add_post +# mlhs_add_star +# mlhs_paren +# nokw_param +# operator_ambiguous +# redo +# rescue +# retry +# return +# return 0 +# undef +# until +# var_alias +# while +# lambda +# tlambda diff --git a/glitch/tests/security/chef/__init__.py b/tests/parser/gha/__init__.py similarity index 100% rename from glitch/tests/security/chef/__init__.py rename to tests/parser/gha/__init__.py diff --git a/glitch/tests/parser/gha/files/index_out_of_range.yml b/tests/parser/gha/files/index_out_of_range.yml similarity index 100% rename from glitch/tests/parser/gha/files/index_out_of_range.yml rename to tests/parser/gha/files/index_out_of_range.yml diff --git a/glitch/tests/parser/gha/files/valid_workflow.yml b/tests/parser/gha/files/valid_workflow.yml similarity index 100% rename from glitch/tests/parser/gha/files/valid_workflow.yml rename to tests/parser/gha/files/valid_workflow.yml diff --git a/glitch/tests/parser/gha/files/valid_workflow_2.yml b/tests/parser/gha/files/valid_workflow_2.yml similarity index 100% rename from glitch/tests/parser/gha/files/valid_workflow_2.yml rename to tests/parser/gha/files/valid_workflow_2.yml diff --git a/tests/parser/gha/test_parser.py b/tests/parser/gha/test_parser.py new file mode 100644 index 00000000..fc4568a3 --- /dev/null +++ b/tests/parser/gha/test_parser.py @@ -0,0 +1,296 @@ +from glitch.parsers.gha import GithubActionsParser +from glitch.repr.inter import * +from tests.parser.test_parser import TestParser + + +class TestGithubActionsParser(TestParser): + def test_gha_parser_valid_workflow(self) -> None: + """ + run commands + with + runs-on + """ + p = GithubActionsParser() + ir = p.parse_file( + "tests/parser/gha/files/valid_workflow.yml", UnitBlockType.script + ) + + assert ir is not None + assert isinstance(ir, UnitBlock) + assert ir.type == UnitBlockType.script + assert ir.name == "Run Python Tests" + + assert len(ir.attributes) == 1 + assert ir.attributes[0].name == "on" + assert isinstance(ir.attributes[0].value, Hash) + assert ir.attributes[0].line == 2 + assert ir.attributes[0].end_line == 10 + assert ir.attributes[0].column == 1 + assert ir.attributes[0].end_column == 1 + assert len(ir.attributes[0].value.value) == 2 + + assert ( + String("push", ElementInfo(3, 3, 3, 7, "push")) + in ir.attributes[0].value.value + ) + push = ir.attributes[0].value.value[ + String("push", ElementInfo(3, 3, 3, 7, "push")) + ] + assert isinstance(push, Hash) + assert push.line == 4 + assert push.column == 5 + assert push.end_line == 5 + assert push.end_column == 13 + assert len(push.value) == 1 + + assert String("branches", ElementInfo(4, 5, 4, 13, "branches")) in push.value + branches = push.value[String("branches", ElementInfo(4, 5, 4, 13, "branches"))] + assert isinstance(branches, Array) + assert len(branches.value) == 1 + self._check_value(branches.value[0], String, "main", 5, 9, 5, 13) + + assert ( + String("pull_request", ElementInfo(6, 3, 6, 15, "pull_request")) + in ir.attributes[0].value.value + ) + pull = ir.attributes[0].value.value[ + String("pull_request", ElementInfo(6, 3, 6, 15, "pull_request")) + ] + assert isinstance(pull, Hash) + assert pull.line == 7 + assert pull.column == 5 + assert pull.end_line == 8 + assert pull.end_column == 13 + assert len(pull.value) == 1 + + assert String("branches", ElementInfo(7, 5, 7, 13, "branches")) in pull.value + branches = pull.value[String("branches", ElementInfo(7, 5, 7, 13, "branches"))] + assert isinstance(branches, Array) + assert len(branches.value) == 1 + self._check_value(branches.value[0], String, "main", 8, 9, 8, 13) + + assert len(ir.unit_blocks) == 1 + assert ir.unit_blocks[0].type == UnitBlockType.block + assert ir.unit_blocks[0].name == "build" + + assert len(ir.unit_blocks[0].attributes) == 1 + assert ir.unit_blocks[0].attributes[0].name == "runs-on" + self._check_value( + ir.unit_blocks[0].attributes[0].value, + String, + "ubuntu-latest", + 12, + 14, + 12, + 27, + ) + assert ir.unit_blocks[0].attributes[0].line == 12 + assert ir.unit_blocks[0].attributes[0].end_line == 12 + assert ir.unit_blocks[0].attributes[0].column == 5 + assert ir.unit_blocks[0].attributes[0].end_column == 27 + + assert len(ir.unit_blocks[0].atomic_units) == 5 + + assert ir.unit_blocks[0].atomic_units[0].name == Null() + assert ir.unit_blocks[0].atomic_units[0].type == "actions/checkout@v3" + + assert ir.unit_blocks[0].atomic_units[1].name == Null() + assert ir.unit_blocks[0].atomic_units[1].type == "ruby/setup-ruby@v1" + assert len(ir.unit_blocks[0].atomic_units[1].attributes) == 1 + assert ir.unit_blocks[0].atomic_units[1].attributes[0].name == "ruby-version" + self._check_value( + ir.unit_blocks[0].atomic_units[1].attributes[0].value, + String, + "2.7.4", + 17, + 25, + 17, + 32, + ) + assert ir.unit_blocks[0].atomic_units[1].attributes[0].line == 17 + assert ir.unit_blocks[0].atomic_units[1].attributes[0].end_line == 17 + assert ir.unit_blocks[0].atomic_units[1].attributes[0].column == 11 + assert ir.unit_blocks[0].atomic_units[1].attributes[0].end_column == 32 + + self._check_value( + ir.unit_blocks[0].atomic_units[2].name, + String, + "Install Python 3", + 18, + 15, + 18, + 31, + ) + assert ir.unit_blocks[0].atomic_units[2].type == "actions/setup-python@v4" + assert len(ir.unit_blocks[0].atomic_units[2].attributes) == 1 + assert ir.unit_blocks[0].atomic_units[2].attributes[0].name == "python-version" + self._check_value( + ir.unit_blocks[0].atomic_units[2].attributes[0].value, + String, + "3.10.5", + 21, + 27, + 21, + 33, + ) + assert ir.unit_blocks[0].atomic_units[2].attributes[0].line == 21 + assert ir.unit_blocks[0].atomic_units[2].attributes[0].end_line == 21 + assert ir.unit_blocks[0].atomic_units[2].attributes[0].column == 11 + assert ir.unit_blocks[0].atomic_units[2].attributes[0].end_column == 33 + + self._check_value( + ir.unit_blocks[0].atomic_units[3].name, + String, + "Install dependencies", + 22, + 15, + 22, + 35, + ) + assert ir.unit_blocks[0].atomic_units[3].type == "shell" + assert len(ir.unit_blocks[0].atomic_units[3].attributes) == 1 + assert ir.unit_blocks[0].atomic_units[3].attributes[0].name == "run" + self._check_value( + ir.unit_blocks[0].atomic_units[3].attributes[0].value, + String, + "python -m pip install --upgrade pip\n python -m pip install -e .", + 24, + 11, + 25, + 37, + ) + assert ir.unit_blocks[0].atomic_units[3].attributes[0].line == 23 + assert ir.unit_blocks[0].atomic_units[3].attributes[0].end_line == 26 + assert ir.unit_blocks[0].atomic_units[3].attributes[0].column == 9 + assert ir.unit_blocks[0].atomic_units[3].attributes[0].end_column == 1 + + self._check_value( + ir.unit_blocks[0].atomic_units[4].name, + String, + "Run tests with pytest", + 26, + 15, + 26, + 36, + ) + assert ir.unit_blocks[0].atomic_units[4].type == "shell" + assert len(ir.unit_blocks[0].atomic_units[4].attributes) == 1 + assert ir.unit_blocks[0].atomic_units[4].attributes[0].name == "run" + self._check_value( + ir.unit_blocks[0].atomic_units[4].attributes[0].value, + String, + "cd glitch\n python -m unittest discover tests", + 28, + 11, + 29, + 44, + ) + assert ir.unit_blocks[0].atomic_units[4].attributes[0].line == 27 + assert ir.unit_blocks[0].atomic_units[4].attributes[0].end_line == 29 + assert ir.unit_blocks[0].atomic_units[4].attributes[0].column == 9 + assert ir.unit_blocks[0].atomic_units[4].attributes[0].end_column == 44 + + def test_gha_parser_valid_workflow_2(self) -> None: + """ + comments + env (global) + has_variable + defaults (job) + """ + p = GithubActionsParser() + ir = p.parse_file( + "tests/parser/gha/files/valid_workflow_2.yml", UnitBlockType.script + ) + assert ir is not None + assert isinstance(ir, UnitBlock) + assert ir.type == UnitBlockType.script + + assert len(ir.variables) == 1 + assert isinstance(ir.variables[0], Variable) + assert ir.variables[0].name == "build" + self._check_binary_operation( + ir.variables[0].value, + Sum, + Access( + ElementInfo(23, 15, 23, 31, "github.workspace"), + VariableReference("github", ElementInfo(23, 15, 23, 21, "github")), + String("workspace", ElementInfo(23, 22, 23, 31, "workspace")), + ), + String("/build", ElementInfo(23, 34, 23, 40, "/build")), + 23, + 10, + 23, + 41, + ) + + assert len(ir.unit_blocks) == 1 + + assert len(ir.unit_blocks[0].variables) == 1 + assert isinstance(ir.unit_blocks[0].variables[0], Variable) + assert ir.unit_blocks[0].variables[0].name == "run" + assert isinstance(ir.unit_blocks[0].variables[0].value, Hash) + assert len(ir.unit_blocks[0].variables[0].value.value) == 1 + assert ( + String("shell", ElementInfo(38, 9, 38, 14, "shell")) + in ir.unit_blocks[0].variables[0].value.value + ) + self._check_value( + ir.unit_blocks[0] + .variables[0] + .value.value[String("shell", ElementInfo(38, 9, 38, 14, "shell"))], + String, + "powershell", + 38, + 16, + 38, + 26, + ) + + assert len(ir.unit_blocks[0].atomic_units) == 4 + self._check_value( + ir.unit_blocks[0].atomic_units[1].name, + String, + "Configure CMake", + 44, + 15, + 44, + 30, + ) + assert ir.unit_blocks[0].atomic_units[1].type == "shell" + assert len(ir.unit_blocks[0].atomic_units[1].attributes) == 1 + assert ir.unit_blocks[0].atomic_units[1].attributes[0].name == "run" + self._check_binary_operation( + ir.unit_blocks[0].atomic_units[1].attributes[0].value, + Sum, + String("cmake -B ", ElementInfo(45, 14, 45, 23, "cmake -B ")), + Access( + ElementInfo(45, 27, 45, 36, "cmake -B ${{ env.build }}"), + VariableReference("env", ElementInfo(45, 27, 45, 30, "env")), + String("build", ElementInfo(45, 31, 45, 36, "build")), + ), + 45, + 14, + 45, + 39, + ) + + assert len(ir.comments) == 24 + + assert ( + ir.comments[0].content + == "# https://github.com/actions/starter-workflows/blob/main/code-scanning/msvc.yml" + ) + assert ir.comments[0].line == 1 + + assert ir.comments[9].content == "# for actions/checkout to fetch code" + assert ir.comments[9].line == 31 + + def test_gha_parser_index_out_of_range(self) -> None: + """ + This file previously gave an index out of range even though it is valid. + """ + p = GithubActionsParser() + ir = p.parse_file( + "tests/parser/gha/files/index_out_of_range.yml", UnitBlockType.script + ) + assert ir is not None diff --git a/glitch/tests/security/docker/__init__.py b/tests/parser/puppet/__init__.py similarity index 100% rename from glitch/tests/security/docker/__init__.py rename to tests/parser/puppet/__init__.py diff --git a/tests/parser/puppet/files/case.pp b/tests/parser/puppet/files/case.pp new file mode 100644 index 00000000..f7eb26c0 --- /dev/null +++ b/tests/parser/puppet/files/case.pp @@ -0,0 +1,10 @@ +case $facts['os']['name'] { + 'RedHat', 'CentOS': { + User <| title == 'luke' |> + + file { '/etc/ntp.conf': + ensure => file, + } + } + default: { File['/etc/ntp.conf'] ~> Service['ntpd'] } +} diff --git a/tests/parser/puppet/files/class.pp b/tests/parser/puppet/files/class.pp new file mode 100644 index 00000000..a5977662 --- /dev/null +++ b/tests/parser/puppet/files/class.pp @@ -0,0 +1,6 @@ +class apache (String $version = 'latest') { + package { $httpd: + ensure => $version, + before => File['/etc/httpd.conf'], + } +} diff --git a/tests/parser/puppet/files/defined_resource.pp b/tests/parser/puppet/files/defined_resource.pp new file mode 100644 index 00000000..140a110f --- /dev/null +++ b/tests/parser/puppet/files/defined_resource.pp @@ -0,0 +1,16 @@ +define apache::vhost ( + Integer $port, + String[1] $servername = $title, +) { + $vhost_dir = $apache::params::vhost_dir + + file { "${vhost_dir}/${servername}.conf": + ensure => file, + owner => 'www', + group => 'www', + mode => '0644', + content => template('apache/vhost-default.conf.erb'), + require => Package['httpd'], + notify => Service['httpd'], + } +} diff --git a/tests/parser/puppet/files/edge_case.pp b/tests/parser/puppet/files/edge_case.pp new file mode 100644 index 00000000..4fc1a3ea --- /dev/null +++ b/tests/parser/puppet/files/edge_case.pp @@ -0,0 +1,28 @@ +define ssh::client_config ( + $connections, + $owner = $title, + ) { + + $_user_home = getvar( "home_${owner}" ) + validate_absolute_path( $_user_home ) + + if ( ! defined( File["${_user_home}/.ssh"] ) ) { + file { "${_user_home}/.ssh": + ensure => directory, + owner => $_config_owner, + mode => '0700', + } + } + + create_resources( ssh::private_key, $connections, { + home => $_user_home, + owner => $owner, + }) + + file { "${_user_home}/.ssh/config": + owner => $owner, + mode => '0600', + content => template( 'ssh/ssh_config.erb' ) + } + +} diff --git a/glitch/tests/parser/puppet/files/if.pp b/tests/parser/puppet/files/if.pp similarity index 80% rename from glitch/tests/parser/puppet/files/if.pp rename to tests/parser/puppet/files/if.pp index a5e41b5b..ceb7e402 100644 --- a/glitch/tests/parser/puppet/files/if.pp +++ b/tests/parser/puppet/files/if.pp @@ -1,3 +1,7 @@ +$test = @("TEST"/L) + test 123 +| TEST + if $x == 'absent' { file {'/usr/sbin/policy-rc.d': ensure => absent, diff --git a/tests/parser/puppet/files/node.pp b/tests/parser/puppet/files/node.pp new file mode 100644 index 00000000..a34b230a --- /dev/null +++ b/tests/parser/puppet/files/node.pp @@ -0,0 +1,10 @@ +node 'www1.example.com' { + include common + require apache + contain squid +} + +fail "test" +debug "test" +realize "test" +tag "test" diff --git a/tests/parser/puppet/files/operations.pp b/tests/parser/puppet/files/operations.pp new file mode 100644 index 00000000..c7d0ab31 --- /dev/null +++ b/tests/parser/puppet/files/operations.pp @@ -0,0 +1,19 @@ +$x = 'a' == 'a' +$x = 'a' != 'b' +$x = true and false +$x = true or false +$x = !true +$x = 1 < 2 +$x = 1 <= 2 +$x = 1 > 2 +$x = 1 >= 2 +$x = 1 in [1, 2, 3] +$x = 1 - 2 +$x = 1 + 2 +$x = 1 * 2 +$x = 1 / 2 +$x = 1 % 2 +$x = 1 >> 2 +$x = 1 << 2 +$x = [1, 2, 3][0] +$x = $y =~ /localhost/ diff --git a/tests/parser/puppet/files/selector.pp b/tests/parser/puppet/files/selector.pp new file mode 100644 index 00000000..e905e549 --- /dev/null +++ b/tests/parser/puppet/files/selector.pp @@ -0,0 +1,6 @@ +function apache::bool2http(Variant[String, Boolean] $arg) >> String { + $rootgroup = $arg ? { + 'Redhat' => 'wheel', + default => 'root', + } +} diff --git a/tests/parser/puppet/files/special_resource.pp b/tests/parser/puppet/files/special_resource.pp new file mode 100644 index 00000000..187a5011 --- /dev/null +++ b/tests/parser/puppet/files/special_resource.pp @@ -0,0 +1,38 @@ +class {'apache': + version => '2.2.21', +} + +class { + 'apache-2': + version => '2.2.21', + ; + 'apache-3': + version => '2.2.22' +} + +file { + default: + ensure => file, + owner => 'root', + mode => '0600', + ; + ['ssh_host_dsa_key']: + # use all defaults + ; + ['ssh_config']: + mode => '0644', + group => 'wheel', + ; +} + +package { ['armitage', 'metasploit']: + ensure => 'installed', +} + +Exec <| title == 'modprobe nf_conntrack_proto_sctp' |> { returns => [0,1] } + +Exec { + provider => 'shell', + path => '/usr/bin:/bin:/sbin:/usr/sbin', + logoutput => true, +} diff --git a/tests/parser/puppet/files/unless.pp b/tests/parser/puppet/files/unless.pp new file mode 100644 index 00000000..052af6c8 --- /dev/null +++ b/tests/parser/puppet/files/unless.pp @@ -0,0 +1,3 @@ +unless $x > 1073741824 { + $maxclient = 500 +} diff --git a/tests/parser/puppet/files/values.pp b/tests/parser/puppet/files/values.pp new file mode 100644 index 00000000..884d93a2 --- /dev/null +++ b/tests/parser/puppet/files/values.pp @@ -0,0 +1,8 @@ +$x = 'Hello' +$y = 2 +$string_interpolation = "${$x + $y} World" +$z = 2.0 +$w = true +$h = undef +$a = [1, 2, 3] +$hash = {a => 1, b => 2} diff --git a/tests/parser/puppet/test_parser.py b/tests/parser/puppet/test_parser.py new file mode 100644 index 00000000..1d16b27a --- /dev/null +++ b/tests/parser/puppet/test_parser.py @@ -0,0 +1,548 @@ +from glitch.parsers.puppet import PuppetParser +from glitch.repr.inter import * +from tests.parser.test_parser import TestParser + + +class TestPuppetParser(TestParser): + def test_puppet_parser_if(self) -> None: + """ + If | Resource | Attribute | str | VariableReference | Assignment + """ + unit_block = PuppetParser().parse_file( + "tests/parser/puppet/files/if.pp", UnitBlockType.script + ) + assert len(unit_block.statements) == 1 + assert isinstance(unit_block.statements[0], ConditionalStatement) + assert unit_block.statements[0].is_top == True + self._check_binary_operation( + unit_block.statements[0].condition, + Equal, + VariableReference("x", ElementInfo(5, 4, 5, 6, "$x")), + String("absent", ElementInfo(5, 10, 5, 18, "absent")), + 5, + 4, + 5, + 18, + ) + + assert len(unit_block.statements[0].statements) == 1 + assert isinstance(unit_block.statements[0].statements[0], AtomicUnit) + atomic_unit = unit_block.statements[0].statements[0] + assert len(atomic_unit.attributes) == 1 + assert atomic_unit.attributes[0].name == "ensure" + self._check_value( + atomic_unit.attributes[0].value, String, "absent", 7, 20, 7, 26 + ) + assert atomic_unit.attributes[0].line == 7 + assert atomic_unit.attributes[0].end_line == 7 + assert atomic_unit.attributes[0].column == 9 + assert atomic_unit.attributes[0].end_column == 26 + + assert unit_block.statements[0].else_statement is not None + assert isinstance(unit_block.statements[0].else_statement, ConditionalStatement) + + assert len(unit_block.statements[0].else_statement.statements) == 1 + assert isinstance( + unit_block.statements[0].else_statement.statements[0], AtomicUnit + ) + atomic_unit = unit_block.statements[0].else_statement.statements[0] + assert len(atomic_unit.attributes) == 1 + assert atomic_unit.attributes[0].name == "ensure" + self._check_value( + atomic_unit.attributes[0].value, + String, + "present", + 11, + 20, + 11, + 27, + ) + assert atomic_unit.attributes[0].line == 11 + assert atomic_unit.attributes[0].end_line == 11 + assert atomic_unit.attributes[0].column == 9 + assert atomic_unit.attributes[0].end_column == 27 + + assert len(unit_block.variables) == 1 + assert unit_block.variables[0].name == "test" + self._check_value( + unit_block.variables[0].value, String, "\n test 123\n", 1, 9, 3, 7 + ) + assert unit_block.variables[0].line == 1 + assert unit_block.variables[0].end_line == 3 + assert unit_block.variables[0].column == 1 + assert unit_block.variables[0].end_column == 7 + + def test_puppet_parser_defined_resource(self) -> None: + """ + User-defined Resource | Function Call | Resource Reference + """ + unit_block = PuppetParser().parse_file( + "tests/parser/puppet/files/defined_resource.pp", UnitBlockType.script + ) + assert unit_block is not None + assert isinstance(unit_block, UnitBlock) + assert len(unit_block.unit_blocks) == 1 + + assert unit_block.unit_blocks[0].type == UnitBlockType.definition + assert unit_block.unit_blocks[0].name == "apache::vhost" + assert len(unit_block.unit_blocks[0].attributes) == 2 + + assert unit_block.unit_blocks[0].attributes[0].name == "port" + assert unit_block.unit_blocks[0].attributes[0].value == Null() + + assert unit_block.unit_blocks[0].attributes[1].name == "servername" + self._check_value( + unit_block.unit_blocks[0].attributes[1].value, + VariableReference, + "title", + 3, + 27, + 3, + 33, + ) + + assert len(unit_block.unit_blocks[0].atomic_units) == 1 + assert isinstance(unit_block.unit_blocks[0].atomic_units[0].name, Sum) + + assert unit_block.unit_blocks[0].atomic_units[0].type == "file" + + attribute = unit_block.unit_blocks[0].atomic_units[0].attributes[4] + assert attribute.name == "content" + self._check_value(attribute.value, FunctionCall, "template", 12, 16, 12, 57) + assert isinstance(attribute.value, FunctionCall) + assert len(attribute.value.args) == 1 + self._check_value( + attribute.value.args[0], + String, + "apache/vhost-default.conf.erb", + 12, + 25, + 12, + 56, + ) + + # TODO: Support resource reference + attribute = unit_block.unit_blocks[0].atomic_units[0].attributes[5] + assert attribute.name == "require" + self._check_value( + attribute.value, Null, None, 4294967296, 4294967296, 4294967296, 4294967296 + ) + + def test_puppet_parser_class(self) -> None: + """ + Class | str + """ + unit_block = PuppetParser().parse_file( + "tests/parser/puppet/files/class.pp", UnitBlockType.script + ) + assert unit_block is not None + assert isinstance(unit_block, UnitBlock) + assert len(unit_block.unit_blocks) == 1 + + assert unit_block.unit_blocks[0].type == UnitBlockType.definition + assert unit_block.unit_blocks[0].name == "apache" + assert len(unit_block.unit_blocks[0].attributes) == 1 + + assert isinstance(unit_block.unit_blocks[0].attributes[0], Attribute) + assert unit_block.unit_blocks[0].attributes[0].name == "version" + self._check_value( + unit_block.unit_blocks[0].attributes[0].value, + String, + "latest", + 1, + 33, + 1, + 41, + ) + + assert len(unit_block.unit_blocks[0].atomic_units) == 1 + self._check_value( + unit_block.unit_blocks[0].atomic_units[0].name, + VariableReference, + "httpd", + 2, + 13, + 2, + 19, + ) + assert unit_block.unit_blocks[0].atomic_units[0].type == "package" + assert isinstance( + unit_block.unit_blocks[0].atomic_units[0].attributes[0].value, + VariableReference, + ) + + def test_puppet_parser_values(self) -> None: + """ + String interpolation | Integer | Float | Bool | Null | Array | Hash + """ + unit_block = PuppetParser().parse_file( + "tests/parser/puppet/files/values.pp", UnitBlockType.script + ) + assert unit_block is not None + assert isinstance(unit_block, UnitBlock) + assert len(unit_block.variables) == 8 + + assert unit_block.variables[0].name == "x" + + assert unit_block.variables[1].name == "y" + self._check_value(unit_block.variables[1].value, Integer, 2, 2, 6, 2, 7) + + assert unit_block.variables[2].name == "string_interpolation" + self._check_binary_operation( + unit_block.variables[2].value, + Sum, + Sum( + ElementInfo(3, 28, 3, 35, ""), + VariableReference("x", ElementInfo(3, 28, 3, 30, "x")), + VariableReference("y", ElementInfo(3, 33, 3, 35, "y")), + ), + String(" World", ElementInfo(3, 36, 3, 42, " World")), + 3, + 25, + 3, + 43, + ) + assert isinstance(unit_block.variables[2].value, Sum) + self._check_binary_operation( + unit_block.variables[2].value.left, + Sum, + VariableReference("x", ElementInfo(3, 28, 3, 30, "$x")), + VariableReference("y", ElementInfo(3, 33, 3, 35, "$y")), + 3, + 28, + 3, + 35, + ) + + assert unit_block.variables[3].name == "z" + self._check_value(unit_block.variables[3].value, Float, 2.0, 4, 6, 4, 9) + + assert unit_block.variables[4].name == "w" + self._check_value(unit_block.variables[4].value, Boolean, True, 5, 6, 5, 10) + + assert unit_block.variables[5].name == "h" + self._check_value(unit_block.variables[5].value, Undef, None, 6, 6, 6, 11) + + assert unit_block.variables[6].name == "a" + assert isinstance(unit_block.variables[6].value, Array) + assert len(unit_block.variables[6].value.value) == 3 + self._check_value( + unit_block.variables[6].value.value[0], Integer, 1, 7, 7, 7, 8 + ) + self._check_value( + unit_block.variables[6].value.value[1], Integer, 2, 7, 10, 7, 11 + ) + self._check_value( + unit_block.variables[6].value.value[2], Integer, 3, 7, 13, 7, 14 + ) + + assert unit_block.variables[7].name == "hash" + assert isinstance(unit_block.variables[7].value, Hash) + assert len(unit_block.variables[7].value.value) == 2 + + def test_puppet_parser_node(self) -> None: + """ + Node | Include | Require | Contain | Debug/Fail/Realize/Tag + """ + unit_block = PuppetParser().parse_file( + "tests/parser/puppet/files/node.pp", UnitBlockType.script + ) + assert unit_block is not None + assert isinstance(unit_block, UnitBlock) + assert len(unit_block.unit_blocks) == 1 + + assert unit_block.unit_blocks[0].type == UnitBlockType.block + assert unit_block.unit_blocks[0].name == "node" + + assert len(unit_block.unit_blocks[0].dependencies) == 3 + assert unit_block.unit_blocks[0].dependencies[0].names == ["common"] + assert unit_block.unit_blocks[0].dependencies[1].names == ["apache"] + assert unit_block.unit_blocks[0].dependencies[2].names == ["squid"] + + assert len(unit_block.unit_blocks[0].statements) == 0 + + def test_puppet_parser_unless(self) -> None: + """ + Unless + """ + unit_block = PuppetParser().parse_file( + "tests/parser/puppet/files/unless.pp", UnitBlockType.script + ) + assert unit_block is not None + assert len(unit_block.statements) == 1 + + assert isinstance(unit_block.statements[0], ConditionalStatement) + assert unit_block.statements[0].is_top == True + assert isinstance(unit_block.statements[0].condition, Not) + self._check_binary_operation( + unit_block.statements[0].condition.expr, + GreaterThan, + VariableReference("x", ElementInfo(1, 8, 1, 10, "$x")), + Integer(1073741824, ElementInfo(1, 13, 1, 23, "1073741824")), + 1, + 8, + 1, + 23, + ) + assert len(unit_block.statements[0].statements) == 1 + assert isinstance(unit_block.statements[0].statements[0], Variable) + + def test_puppet_parser_case(self) -> None: + """ + Case | Resource Collector | Chaining + """ + unit_block = PuppetParser().parse_file( + "tests/parser/puppet/files/case.pp", UnitBlockType.script + ) + assert unit_block is not None + assert len(unit_block.statements) == 1 + + assert isinstance(unit_block.statements[0], ConditionalStatement) + assert unit_block.statements[0].is_top == True + assert isinstance(unit_block.statements[0].condition, Or) + assert unit_block.statements[0].condition.line == 1 + + assert isinstance(unit_block.statements[0].condition.left, Equal) + assert isinstance(unit_block.statements[0].condition.left.left, Access) + assert isinstance(unit_block.statements[0].condition.left.right, String) + + assert isinstance(unit_block.statements[0].condition.right, Equal) + assert isinstance(unit_block.statements[0].condition.right.left, Access) + assert isinstance(unit_block.statements[0].condition.right.right, String) + + assert len(unit_block.statements[0].statements) == 2 + assert isinstance(unit_block.statements[0].statements[0], Null) + assert isinstance(unit_block.statements[0].statements[1], AtomicUnit) + + assert isinstance(unit_block.statements[0].else_statement, ConditionalStatement) + assert isinstance(unit_block.statements[0].else_statement.condition, Null) + assert len(unit_block.statements[0].else_statement.statements) == 1 + # FIXME: This is an HACK to temporarily support chaining + assert isinstance( + unit_block.statements[0].else_statement.statements[0], UnitBlock + ) + + def test_puppet_parser_selector(self) -> None: + """ + Selector | Function + """ + unit_block = PuppetParser().parse_file( + "tests/parser/puppet/files/selector.pp", UnitBlockType.script + ) + assert unit_block is not None + assert len(unit_block.unit_blocks) == 1 + + assert unit_block.unit_blocks[0].type == UnitBlockType.function + + unit_block = unit_block.unit_blocks[0] + assert len(unit_block.variables) == 1 + + assert isinstance(unit_block.variables[0].value, ConditionalStatement) + assert unit_block.variables[0].value.is_top == True + assert isinstance(unit_block.variables[0].value.condition, Equal) + + assert isinstance( + unit_block.variables[0].value.condition.left, VariableReference + ) + assert isinstance(unit_block.variables[0].value.condition.right, String) + + assert len(unit_block.variables[0].value.statements) == 1 + assert isinstance(unit_block.variables[0].value.statements[0], String) + + assert unit_block.variables[0].value.else_statement is not None + assert isinstance( + unit_block.variables[0].value.else_statement, ConditionalStatement + ) + assert isinstance(unit_block.variables[0].value.else_statement.condition, Null) + assert len(unit_block.variables[0].value.else_statement.statements) == 1 + assert isinstance( + unit_block.variables[0].value.else_statement.statements[0], String + ) + + def test_puppet_parser_special_resources(self) -> None: + """ + Class as resource | Resource expressions + """ + unit_block = PuppetParser().parse_file( + "tests/parser/puppet/files/special_resource.pp", UnitBlockType.script + ) + assert unit_block is not None + assert len(unit_block.atomic_units) == 3 + assert len(unit_block.unit_blocks) == 3 + + self._check_value( + unit_block.atomic_units[0].name, String, "apache", 1, 8, 1, 16 + ) + assert unit_block.atomic_units[0].type == "class" + assert len(unit_block.atomic_units[0].attributes) == 1 + assert unit_block.atomic_units[0].attributes[0].name == "version" + self._check_value( + unit_block.atomic_units[0].attributes[0].value, + String, + "2.2.21", + 2, + 14, + 2, + 22, + ) + + assert ( + unit_block.atomic_units[1].type + == "Exec <| title == 'modprobe nf_conntrack_proto_sctp' |>" + ) + assert unit_block.atomic_units[2].type == "Exec" + + assert unit_block.unit_blocks[0].type == UnitBlockType.block + assert unit_block.unit_blocks[0].name == "resource_expression" + assert len(unit_block.unit_blocks[0].atomic_units) == 2 + self._check_value( + unit_block.unit_blocks[0].atomic_units[0].name, + String, + "apache-2", + 6, + 3, + 6, + 13, + ) + assert unit_block.unit_blocks[0].atomic_units[0].type == "class" + self._check_value( + unit_block.unit_blocks[0].atomic_units[1].name, + String, + "apache-3", + 9, + 3, + 9, + 13, + ) + assert unit_block.unit_blocks[0].atomic_units[1].type == "class" + + assert unit_block.unit_blocks[1].type == UnitBlockType.block + assert unit_block.unit_blocks[1].name == "resource_expression" + assert len(unit_block.unit_blocks[1].atomic_units) == 2 + + self._check_value( + unit_block.unit_blocks[1].atomic_units[0].name, + String, + "ssh_host_dsa_key", + 19, + 4, + 19, + 22, + ) + assert len(unit_block.unit_blocks[1].atomic_units[0].attributes) == 3 + assert unit_block.unit_blocks[1].atomic_units[0].attributes[0].name == "ensure" + self._check_value( + unit_block.unit_blocks[1].atomic_units[0].attributes[0].value, + String, + "file", + 15, + 15, + 15, + 19, + ) + assert unit_block.unit_blocks[1].atomic_units[0].attributes[1].name == "owner" + self._check_value( + unit_block.unit_blocks[1].atomic_units[0].attributes[1].value, + String, + "root", + 16, + 15, + 16, + 21, + ) + assert unit_block.unit_blocks[1].atomic_units[0].attributes[2].name == "mode" + self._check_value( + unit_block.unit_blocks[1].atomic_units[0].attributes[2].value, + String, + "0600", + 17, + 15, + 17, + 21, + ) + + self._check_value( + unit_block.unit_blocks[1].atomic_units[1].name, + String, + "ssh_config", + 22, + 4, + 22, + 16, + ) + assert len(unit_block.unit_blocks[1].atomic_units[1].attributes) == 4 + assert unit_block.unit_blocks[1].atomic_units[1].attributes[0].name == "mode" + self._check_value( + unit_block.unit_blocks[1].atomic_units[1].attributes[0].value, + String, + "0644", + 23, + 14, + 23, + 20, + ) + assert unit_block.unit_blocks[1].atomic_units[1].attributes[1].name == "group" + + assert isinstance(unit_block.unit_blocks[2], UnitBlock) + assert unit_block.unit_blocks[2].type == UnitBlockType.block + assert unit_block.unit_blocks[2].name == "resource_expression" + assert len(unit_block.unit_blocks[2].atomic_units) == 2 + + self._check_value( + unit_block.unit_blocks[2].atomic_units[0].name, + String, + "armitage", + 28, + 12, + 28, + 22, + ) + self._check_value( + unit_block.unit_blocks[2].atomic_units[1].name, + String, + "metasploit", + 28, + 24, + 28, + 36, + ) + + def test_puppet_parser_operations(self) -> None: + """ + All operations + """ + unit_block = PuppetParser().parse_file( + "tests/parser/puppet/files/operations.pp", UnitBlockType.script + ) + assert unit_block is not None + assert len(unit_block.variables) == 19 + for i in range(18): + assert unit_block.variables[i].name == f"x" + assert isinstance(unit_block.variables[0].value, Equal) + assert isinstance(unit_block.variables[1].value, NotEqual) + assert isinstance(unit_block.variables[2].value, And) + assert isinstance(unit_block.variables[3].value, Or) + assert isinstance(unit_block.variables[4].value, Not) + assert isinstance(unit_block.variables[5].value, LessThan) + assert isinstance(unit_block.variables[6].value, LessThanOrEqual) + assert isinstance(unit_block.variables[7].value, GreaterThan) + assert isinstance(unit_block.variables[8].value, GreaterThanOrEqual) + assert isinstance(unit_block.variables[9].value, In) + assert isinstance(unit_block.variables[10].value, Subtract) + assert isinstance(unit_block.variables[11].value, Sum) + assert isinstance(unit_block.variables[12].value, Multiply) + assert isinstance(unit_block.variables[13].value, Divide) + assert isinstance(unit_block.variables[14].value, Modulo) + assert isinstance(unit_block.variables[15].value, RightShift) + assert isinstance(unit_block.variables[16].value, LeftShift) + assert isinstance(unit_block.variables[17].value, Access) + + def test_puppet_parser_edge_case(self) -> None: + """ + The name _user_owner used to crash the parser + """ + unit_block = PuppetParser().parse_file( + "tests/parser/puppet/files/edge_case.pp", UnitBlockType.script + ) + assert unit_block is not None + assert len(unit_block.unit_blocks) == 1 diff --git a/glitch/tests/security/puppet/__init__.py b/tests/parser/terraform/__init__.py similarity index 100% rename from glitch/tests/security/puppet/__init__.py rename to tests/parser/terraform/__init__.py diff --git a/tests/parser/terraform/files/block_with_attribute.tf b/tests/parser/terraform/files/block_with_attribute.tf new file mode 100644 index 00000000..32924ec5 --- /dev/null +++ b/tests/parser/terraform/files/block_with_attribute.tf @@ -0,0 +1,9 @@ +resource "aws_s3_bucket_server_side_encryption_configuration" "good_example" { + bucket = aws_s3_bucket.good_example.id + rule { + apply_server_side_encryption_by_default { + kms_master_key_id = "something" + sse_algorithm = "aes256" + } + } +} \ No newline at end of file diff --git a/glitch/tests/parser/terraform/files/boolean_value_assign.tf b/tests/parser/terraform/files/boolean_value_assign.tf similarity index 100% rename from glitch/tests/parser/terraform/files/boolean_value_assign.tf rename to tests/parser/terraform/files/boolean_value_assign.tf diff --git a/glitch/tests/parser/terraform/files/comments.tf b/tests/parser/terraform/files/comments.tf similarity index 100% rename from glitch/tests/parser/terraform/files/comments.tf rename to tests/parser/terraform/files/comments.tf diff --git a/tests/parser/terraform/files/conditional.tf b/tests/parser/terraform/files/conditional.tf new file mode 100644 index 00000000..4dc614de --- /dev/null +++ b/tests/parser/terraform/files/conditional.tf @@ -0,0 +1,3 @@ +resource "google_service_account" "bqowner" { + account_id = var.create_service_account ? "bqowner" : null +} \ No newline at end of file diff --git a/glitch/tests/parser/terraform/files/dict_value_assign.tf b/tests/parser/terraform/files/dict_value_assign.tf similarity index 100% rename from glitch/tests/parser/terraform/files/dict_value_assign.tf rename to tests/parser/terraform/files/dict_value_assign.tf diff --git a/glitch/tests/parser/terraform/files/dynamic_block.tf b/tests/parser/terraform/files/dynamic_block.tf similarity index 100% rename from glitch/tests/parser/terraform/files/dynamic_block.tf rename to tests/parser/terraform/files/dynamic_block.tf diff --git a/glitch/tests/parser/terraform/files/empty_string_assign.tf b/tests/parser/terraform/files/empty_string_assign.tf similarity index 100% rename from glitch/tests/parser/terraform/files/empty_string_assign.tf rename to tests/parser/terraform/files/empty_string_assign.tf diff --git a/tests/parser/terraform/files/function_call.tf b/tests/parser/terraform/files/function_call.tf new file mode 100644 index 00000000..af66b562 --- /dev/null +++ b/tests/parser/terraform/files/function_call.tf @@ -0,0 +1,4 @@ +resource "google_service_account" "bqowner" { + account_id = min(55, 3453, 2) + account_id_2 = gen() +} \ No newline at end of file diff --git a/glitch/tests/parser/terraform/files/list_value_assign.tf b/tests/parser/terraform/files/list_value_assign.tf similarity index 100% rename from glitch/tests/parser/terraform/files/list_value_assign.tf rename to tests/parser/terraform/files/list_value_assign.tf diff --git a/glitch/tests/parser/terraform/files/multiline_string_assign.tf b/tests/parser/terraform/files/multiline_string_assign.tf similarity index 100% rename from glitch/tests/parser/terraform/files/multiline_string_assign.tf rename to tests/parser/terraform/files/multiline_string_assign.tf diff --git a/glitch/tests/parser/terraform/files/null_value_assign.tf b/tests/parser/terraform/files/null_value_assign.tf similarity index 100% rename from glitch/tests/parser/terraform/files/null_value_assign.tf rename to tests/parser/terraform/files/null_value_assign.tf diff --git a/tests/parser/terraform/files/operations.tf b/tests/parser/terraform/files/operations.tf new file mode 100644 index 00000000..943f7e57 --- /dev/null +++ b/tests/parser/terraform/files/operations.tf @@ -0,0 +1,17 @@ +resource "test" "test" { + sum = 1.3 + 1.4 + sub = 1.3 - 1.4 + mul = 1.3 * 1.4 + div = 1.3 / 1.4 + mod = 1.3 % 1.4 + and = 1.3 && 1.4 + or = 1.3 || 1.4 + eq = 1.3 == 1.4 + ne = 1.3 != 1.4 + gt = 1.3 > 1.4 + lt = 1.3 < 1.4 + ge = 1.3 >= 1.4 + le = 1.3 <= 1.4 + not = !true + minus = -1.3 +} diff --git a/tests/parser/terraform/files/recursive_blocks.tf b/tests/parser/terraform/files/recursive_blocks.tf new file mode 100644 index 00000000..eb3f396c --- /dev/null +++ b/tests/parser/terraform/files/recursive_blocks.tf @@ -0,0 +1,5 @@ +locals { + tags = { + application = "Modernisation Platform: ${terraform.workspace}" + } +} \ No newline at end of file diff --git a/tests/parser/terraform/files/value_has_variable.tf b/tests/parser/terraform/files/value_has_variable.tf new file mode 100644 index 00000000..f43bd56d --- /dev/null +++ b/tests/parser/terraform/files/value_has_variable.tf @@ -0,0 +1,5 @@ +resource "google_bigquery_dataset" "dataset" { + test = "test ${var.value1}" + test2 = "test ${"${var.value2}"}" + test3 = "filter('env', '${var.environment}') and filter('sfx_monitored', 'true')" +} \ No newline at end of file diff --git a/tests/parser/terraform/test_parser.py b/tests/parser/terraform/test_parser.py new file mode 100644 index 00000000..361eb613 --- /dev/null +++ b/tests/parser/terraform/test_parser.py @@ -0,0 +1,448 @@ +from glitch.parsers.terraform import TerraformParser +from glitch.repr.inter import * +from tests.parser.test_parser import TestParser + + +class TestTerraform(TestParser): + def __parse(self, path: str) -> UnitBlock: + p = TerraformParser() + ir = p.parse_file(path, UnitBlockType.script) + assert ir is not None + assert isinstance(ir, UnitBlock) + return ir + + def test_terraform_parser_null_value(self) -> None: + ir = self.__parse("tests/parser/terraform/files/null_value_assign.tf") + assert len(ir.atomic_units) == 1 + + assert isinstance(ir.atomic_units[0], AtomicUnit) + assert len(ir.atomic_units[0].attributes) == 1 + self._check_value(ir.atomic_units[0].name, String, "bqowner", 1, 35, 1, 44) + assert ir.atomic_units[0].type == "google_service_account" + + assert ir.atomic_units[0].attributes[0].name == "account_id" + assert ir.atomic_units[0].attributes[0].line == 2 + assert ir.atomic_units[0].attributes[0].column == 3 + assert ir.atomic_units[0].attributes[0].end_line == 2 + assert ir.atomic_units[0].attributes[0].end_column == 20 + assert isinstance(ir.atomic_units[0].attributes[0].value, Null) + assert ir.atomic_units[0].attributes[0].value.line == 2 + assert ir.atomic_units[0].attributes[0].value.column == 16 + assert ir.atomic_units[0].attributes[0].value.end_line == 2 + assert ir.atomic_units[0].attributes[0].value.end_column == 20 + + def test_terraform_parser_empty_string(self) -> None: + ir = self.__parse("tests/parser/terraform/files/empty_string_assign.tf") + assert len(ir.atomic_units) == 1 + + assert isinstance(ir.atomic_units[0], AtomicUnit) + assert len(ir.atomic_units[0].attributes) == 1 + self._check_value(ir.atomic_units[0].name, String, "bqowner", 1, 35, 1, 44) + assert ir.atomic_units[0].type == "google_service_account" + + assert ir.atomic_units[0].attributes[0].name == "account_id" + assert ir.atomic_units[0].attributes[0].line == 2 + assert ir.atomic_units[0].attributes[0].column == 3 + assert ir.atomic_units[0].attributes[0].end_line == 2 + assert ir.atomic_units[0].attributes[0].end_column == 18 + self._check_value( + ir.atomic_units[0].attributes[0].value, String, "", 2, 16, 2, 18 + ) + + def test_terraform_parser_boolean_value(self) -> None: + ir = self.__parse("tests/parser/terraform/files/boolean_value_assign.tf") + assert len(ir.atomic_units) == 1 + + assert isinstance(ir.atomic_units[0], AtomicUnit) + assert len(ir.atomic_units[0].attributes) == 1 + self._check_value(ir.atomic_units[0].name, String, "bqowner", 1, 35, 1, 44) + assert ir.atomic_units[0].type == "google_service_account" + + assert ir.atomic_units[0].attributes[0].name == "account_id" + + assert ir.atomic_units[0].attributes[0].line == 2 + assert ir.atomic_units[0].attributes[0].column == 3 + assert ir.atomic_units[0].attributes[0].end_line == 2 + assert ir.atomic_units[0].attributes[0].end_column == 20 + self._check_value( + ir.atomic_units[0].attributes[0].value, Boolean, True, 2, 16, 2, 20 + ) + + def test_terraform_parser_multiline_string(self) -> None: + ir = self.__parse("tests/parser/terraform/files/multiline_string_assign.tf") + assert len(ir.atomic_units) == 1 + + assert isinstance(ir.atomic_units[0], AtomicUnit) + assert len(ir.atomic_units[0].attributes) == 1 + self._check_value(ir.atomic_units[0].name, String, "example", 1, 25, 1, 34) + assert ir.atomic_units[0].type == "aws_instance" + + assert ir.atomic_units[0].attributes[0].name == "user_data" + assert ir.atomic_units[0].attributes[0].line == 2 + assert ir.atomic_units[0].attributes[0].column == 3 + assert ir.atomic_units[0].attributes[0].end_line == 7 + assert ir.atomic_units[0].attributes[0].end_column == 6 + self._check_value( + ir.atomic_units[0].attributes[0].value, + String, + " #!/bin/bash\n sudo apt-get update\n sudo apt-get install -y apache2\n sudo systemctl start apache2", + 2, + 19, + 7, + 6, + ) + + def test_terraform_parser_value_has_variable(self) -> None: + ir = self.__parse("tests/parser/terraform/files/value_has_variable.tf") + assert len(ir.atomic_units) == 1 + + assert isinstance(ir.atomic_units[0], AtomicUnit) + assert len(ir.atomic_units[0].attributes) == 3 + self._check_value(ir.atomic_units[0].name, String, "dataset", 1, 36, 1, 45) + assert ir.atomic_units[0].type == "google_bigquery_dataset" + + assert ir.atomic_units[0].attributes[0].name == "test" + assert ir.atomic_units[0].attributes[0].line == 2 + assert ir.atomic_units[0].attributes[0].column == 3 + assert ir.atomic_units[0].attributes[0].end_line == 2 + assert ir.atomic_units[0].attributes[0].end_column == 31 + self._check_binary_operation( + ir.atomic_units[0].attributes[0].value, + Sum, + String("test ", ElementInfo(2, 12, 2, 17, "test ")), + Access( + ElementInfo(2, 19, 2, 29, "var.value1"), + VariableReference("var", ElementInfo(2, 19, 2, 22, "var")), + VariableReference("value1", ElementInfo(2, 23, 2, 29, "value1")), + ), + 2, + 11, + 2, + 31, + ) + + assert ir.atomic_units[0].attributes[1].name == "test2" + assert ir.atomic_units[0].attributes[1].line == 3 + assert ir.atomic_units[0].attributes[1].column == 3 + assert ir.atomic_units[0].attributes[1].end_line == 3 + assert ir.atomic_units[0].attributes[1].end_column == 36 + self._check_binary_operation( + ir.atomic_units[0].attributes[1].value, + Sum, + String("test ", ElementInfo(3, 12, 3, 17, "test ")), + Access( + ElementInfo(3, 22, 3, 32, "var.value2"), + VariableReference("var", ElementInfo(3, 22, 3, 25, "var")), + VariableReference("value2", ElementInfo(2, 26, 3, 32, "value2")), + ), + 3, + 11, + 3, + 36, + ) + + assert isinstance(ir.atomic_units[0].attributes[2].value, Sum) + + def test_terraform_parser_dict_value(self) -> None: + ir = self.__parse("tests/parser/terraform/files/dict_value_assign.tf") + assert len(ir.atomic_units) == 1 + + assert isinstance(ir.atomic_units[0], AtomicUnit) + assert len(ir.atomic_units[0].attributes) == 1 + self._check_value(ir.atomic_units[0].name, String, "dataset", 1, 36, 1, 45) + assert ir.atomic_units[0].type == "google_bigquery_dataset" + + assert ir.atomic_units[0].attributes[0].name == "labels" + assert ir.atomic_units[0].attributes[0].line == 2 + assert ir.atomic_units[0].attributes[0].column == 3 + assert ir.atomic_units[0].attributes[0].end_line == 4 + assert ir.atomic_units[0].attributes[0].end_column == 4 + assert isinstance(ir.atomic_units[0].attributes[0].value, Hash) + assert ir.atomic_units[0].attributes[0].value.line == 2 + assert ir.atomic_units[0].attributes[0].value.column == 12 + assert ir.atomic_units[0].attributes[0].value.end_line == 4 + assert ir.atomic_units[0].attributes[0].value.end_column == 4 + assert len(ir.atomic_units[0].attributes[0].value.value) == 1 + self._check_value( + list(ir.atomic_units[0].attributes[0].value.value.items())[0][0], + VariableReference, + "env", + 3, + 5, + 3, + 8, + ) + self._check_value( + list(ir.atomic_units[0].attributes[0].value.value.items())[0][1], + String, + "default", + 3, + 11, + 3, + 20, + ) + + def test_terraform_parser_list_value(self) -> None: + ir = self.__parse("tests/parser/terraform/files/list_value_assign.tf") + assert len(ir.atomic_units) == 1 + + assert isinstance(ir.atomic_units[0], AtomicUnit) + assert len(ir.atomic_units[0].attributes) == 1 + self._check_value(ir.atomic_units[0].name, String, "bqowner", 1, 35, 1, 44) + assert ir.atomic_units[0].type == "google_service_account" + + assert ir.atomic_units[0].attributes[0].name == "keys" + assert isinstance(ir.atomic_units[0].attributes[0].value, Array) + assert len(ir.atomic_units[0].attributes[0].value.value) == 3 + + self._check_value( + ir.atomic_units[0].attributes[0].value.value[0], + String, + "value1", + 2, + 11, + 2, + 19, + ) + assert isinstance(ir.atomic_units[0].attributes[0].value.value[1], Array) + assert isinstance(ir.atomic_units[0].attributes[0].value.value[2], Hash) + + def test_terraform_parser_dynamic_block(self) -> None: + ir = self.__parse("tests/parser/terraform/files/dynamic_block.tf") + assert len(ir.atomic_units) == 1 + + assert isinstance(ir.atomic_units[0], AtomicUnit) + self._check_value(ir.atomic_units[0].name, String, "tfenvtest", 1, 46, 1, 57) + assert ir.atomic_units[0].type == "aws_elastic_beanstalk_environment" + assert len(ir.atomic_units[0].statements) == 1 + + assert isinstance(ir.atomic_units[0].statements[0], UnitBlock) + assert ir.atomic_units[0].statements[0].type == "block" + assert ir.atomic_units[0].statements[0].name == "dynamic" + assert len(ir.atomic_units[0].statements[0].unit_blocks) == 1 + + assert isinstance(ir.atomic_units[0].statements[0].unit_blocks[0], UnitBlock) + assert ir.atomic_units[0].statements[0].unit_blocks[0].type == "block" + assert ir.atomic_units[0].statements[0].unit_blocks[0].name == "content" + assert len(ir.atomic_units[0].statements[0].unit_blocks[0].attributes) == 1 + + assert ( + ir.atomic_units[0].statements[0].unit_blocks[0].attributes[0].name + == "namespace" + ) + + def test_terraform_parser_comments(self) -> None: + ir = self.__parse("tests/parser/terraform/files/comments.tf") + assert len(ir.comments) == 7 + + assert ir.comments[0].content == "#comment1\n" + assert ir.comments[0].line == 1 + + assert ir.comments[1].content == "//comment2\n" + assert ir.comments[1].line == 2 + + assert ( + ir.comments[2].content + == "/*comment3\n default_table_expiration_ms = 3600000\n \n finish comment3 */" + ) + assert ir.comments[2].line == 4 + + assert ir.comments[3].content == "#comment4\n" + assert ir.comments[3].line == 7 + + assert ir.comments[4].content == "#comment5\n" + assert ir.comments[4].line == 9 + + assert ir.comments[5].content == "#comment inside dict\n" + assert ir.comments[5].line == 12 + + assert ir.comments[6].content == "//comment2 inside dict\n" + assert ir.comments[6].line == 13 + + def test_terraform_parser_operations(self) -> None: + ir = self.__parse("tests/parser/terraform/files/operations.tf") + assert len(ir.atomic_units) == 1 + assert len(ir.atomic_units[0].attributes) == 15 + + self._check_binary_operation( + ir.atomic_units[0].attributes[0].value, + Sum, + Float(1.3, ElementInfo(2, 9, 2, 12, "1.3")), + Float(1.4, ElementInfo(2, 15, 2, 18, "1.4")), + 2, + 9, + 2, + 18, + ) + self._check_binary_operation( + ir.atomic_units[0].attributes[1].value, + Subtract, + Float(1.3, ElementInfo(3, 9, 3, 12, "1.3")), + Float(1.4, ElementInfo(3, 15, 3, 18, "1.4")), + 3, + 9, + 3, + 18, + ) + self._check_binary_operation( + ir.atomic_units[0].attributes[2].value, + Multiply, + Float(1.3, ElementInfo(4, 9, 4, 12, "1.3")), + Float(1.4, ElementInfo(4, 15, 4, 18, "1.4")), + 4, + 9, + 4, + 18, + ) + self._check_binary_operation( + ir.atomic_units[0].attributes[3].value, + Divide, + Float(1.3, ElementInfo(5, 9, 5, 12, "1.3")), + Float(1.4, ElementInfo(5, 15, 5, 18, "1.4")), + 5, + 9, + 5, + 18, + ) + self._check_binary_operation( + ir.atomic_units[0].attributes[4].value, + Modulo, + Float(1.3, ElementInfo(6, 9, 6, 12, "1.3")), + Float(1.4, ElementInfo(6, 15, 6, 18, "1.4")), + 6, + 9, + 6, + 18, + ) + self._check_binary_operation( + ir.atomic_units[0].attributes[5].value, + And, + Float(1.3, ElementInfo(7, 9, 7, 12, "1.3")), + Float(1.4, ElementInfo(7, 16, 7, 19, "1.4")), + 7, + 9, + 7, + 19, + ) + self._check_binary_operation( + ir.atomic_units[0].attributes[6].value, + Or, + Float(1.3, ElementInfo(8, 8, 8, 11, "1.3")), + Float(1.4, ElementInfo(8, 15, 8, 18, "1.4")), + 8, + 8, + 8, + 18, + ) + self._check_binary_operation( + ir.atomic_units[0].attributes[7].value, + Equal, + Float(1.3, ElementInfo(9, 8, 9, 11, "1.3")), + Float(1.4, ElementInfo(9, 15, 9, 18, "1.4")), + 9, + 8, + 9, + 18, + ) + self._check_binary_operation( + ir.atomic_units[0].attributes[8].value, + NotEqual, + Float(1.3, ElementInfo(10, 8, 10, 11, "1.3")), + Float(1.4, ElementInfo(10, 15, 10, 18, "1.4")), + 10, + 8, + 10, + 18, + ) + self._check_binary_operation( + ir.atomic_units[0].attributes[9].value, + GreaterThan, + Float(1.3, ElementInfo(11, 8, 11, 11, "1.3")), + Float(1.4, ElementInfo(11, 14, 11, 17, "1.4")), + 11, + 8, + 11, + 17, + ) + self._check_binary_operation( + ir.atomic_units[0].attributes[10].value, + LessThan, + Float(1.3, ElementInfo(12, 8, 12, 11, "1.3")), + Float(1.4, ElementInfo(12, 14, 12, 17, "1.4")), + 12, + 8, + 12, + 17, + ) + self._check_binary_operation( + ir.atomic_units[0].attributes[11].value, + GreaterThanOrEqual, + Float(1.3, ElementInfo(13, 8, 13, 11, "1.3")), + Float(1.4, ElementInfo(13, 15, 13, 18, "1.4")), + 13, + 8, + 13, + 18, + ) + self._check_binary_operation( + ir.atomic_units[0].attributes[12].value, + LessThanOrEqual, + Float(1.3, ElementInfo(14, 8, 14, 11, "1.3")), + Float(1.4, ElementInfo(14, 15, 14, 18, "1.4")), + 14, + 8, + 14, + 18, + ) + assert isinstance(ir.atomic_units[0].attributes[13].value, Not) + self._check_value( + ir.atomic_units[0].attributes[13].value.expr, Boolean, True, 15, 10, 15, 14 + ) + assert isinstance(ir.atomic_units[0].attributes[14].value, Minus) + self._check_value( + ir.atomic_units[0].attributes[14].value.expr, Float, 1.3, 16, 12, 16, 15 + ) + + def test_terraform_parser_conditional(self) -> None: + ir = self.__parse("tests/parser/terraform/files/conditional.tf") + assert len(ir.atomic_units) == 1 + assert len(ir.atomic_units[0].attributes) == 1 + + assert isinstance(ir.atomic_units[0].attributes[0].value, ConditionalStatement) + assert ir.atomic_units[0].attributes[0].value.line == 2 + assert ir.atomic_units[0].attributes[0].value.column == 16 + assert ir.atomic_units[0].attributes[0].value.end_line == 2 + assert ir.atomic_units[0].attributes[0].value.end_column == 61 + + assert isinstance(ir.atomic_units[0].attributes[0].value.condition, Access) + assert len(ir.atomic_units[0].attributes[0].value.statements) == 1 + assert isinstance(ir.atomic_units[0].attributes[0].value.statements[0], String) + + assert isinstance( + ir.atomic_units[0].attributes[0].value.else_statement, ConditionalStatement + ) + assert ( + len(ir.atomic_units[0].attributes[0].value.else_statement.statements) == 1 + ) + assert isinstance( + ir.atomic_units[0].attributes[0].value.else_statement.statements[0], Null + ) + + def test_terraform_parser_function_call(self) -> None: + ir = self.__parse("tests/parser/terraform/files/function_call.tf") + assert len(ir.atomic_units) == 1 + assert len(ir.atomic_units[0].attributes) == 2 + assert isinstance(ir.atomic_units[0].attributes[0].value, FunctionCall) + assert isinstance(ir.atomic_units[0].attributes[1].value, FunctionCall) + + def test_terraform_parser_recursive_blocks(self) -> None: + ir = self.__parse("tests/parser/terraform/files/recursive_blocks.tf") + assert len(ir.unit_blocks) == 1 + assert len(ir.unit_blocks[0].variables) == 1 + assert ir.unit_blocks[0].variables[0].name == "tags" + + def test_terraform_parser_block_with_attribute(self) -> None: + ir = self.__parse("tests/parser/terraform/files/block_with_attribute.tf") + assert len(ir.atomic_units) == 1 + assert len(ir.atomic_units[0].statements) == 1 + assert len(ir.atomic_units[0].attributes) == 1 diff --git a/tests/parser/test_parser.py b/tests/parser/test_parser.py new file mode 100644 index 00000000..9ddc7390 --- /dev/null +++ b/tests/parser/test_parser.py @@ -0,0 +1,43 @@ +import unittest +from typing import Any, Type +from glitch.repr.inter import Expr, Value, BinaryOperation + + +class TestParser(unittest.TestCase): + def _check_value( + self, + obtained: Expr, + type: Type[Expr], + value: Any, + line: int, + column: int, + end_line: int, + end_column: int, + ): + assert isinstance(obtained, type) + if isinstance(obtained, Value): + assert obtained.value == value + assert obtained.line == line + assert obtained.column == column + assert obtained.end_line == end_line + assert obtained.end_column == end_column + + def _check_binary_operation( + self, + obtained: Expr, + type: Type[BinaryOperation], + left: Expr, + right: Expr, + line: int, + column: int, + end_line: int, + end_column: int, + ): + assert isinstance(obtained, BinaryOperation) + assert isinstance(obtained, type) + assert obtained.left == left + assert obtained.right == right + assert obtained.line == line + assert obtained.column == column + assert obtained.end_line == end_line + assert obtained.end_column == end_column diff --git a/tests/repair/interactive/delta_p/delta_p_puppet_scripts.py b/tests/repair/interactive/delta_p/delta_p_puppet_scripts.py new file mode 100644 index 00000000..9daaf365 --- /dev/null +++ b/tests/repair/interactive/delta_p/delta_p_puppet_scripts.py @@ -0,0 +1,438 @@ +from glitch.repair.interactive.delta_p import * + +delta_p_puppet = PSeq( + lhs=PSeq( + lhs=PSkip(), + rhs=PSeq( + lhs=PSeq( + lhs=PSeq( + lhs=PLet( + id="state_9750", + expr=PRLet( + id="literal-7", + expr=PEConst(const=PStr(value="present")), + label=7, + ), + label=2, + body=PAttr( + path=PRLet( + id="literal-5", + expr=PEConst( + const=PStr( + value="/var/www/customers/public_html/index.php" + ) + ), + label=5, + ), + attr="state", + value=PEVar(id="state_9750"), + ), + ), + rhs=PLet( + id="content_14976", + expr=PRLet( + id="literal-6", + expr=PEConst( + const=PStr( + value="

Hello World

" + ) + ), + label=6, + ), + label=1, + body=PAttr( + path=PRLet( + id="literal-5", + expr=PEConst( + const=PStr( + value="/var/www/customers/public_html/index.php" + ) + ), + label=5, + ), + attr="content", + value=PEVar(id="content_14976"), + ), + ), + ), + rhs=PLet( + id="owner_21021", + expr=PRLet( + id="literal-9", + expr=PEConst(const=PStr(value="web_admin")), + label=9, + ), + label=4, + body=PAttr( + path=PRLet( + id="literal-5", + expr=PEConst( + const=PStr( + value="/var/www/customers/public_html/index.php" + ) + ), + label=5, + ), + attr="owner", + value=PEVar(id="owner_21021"), + ), + ), + ), + rhs=PLet( + id="mode_12636", + expr=PRLet( + id="literal-8", expr=PEConst(const=PStr(value="0755")), label=8 + ), + label=3, + body=PAttr( + path=PRLet( + id="literal-5", + expr=PEConst( + const=PStr(value="/var/www/customers/public_html/index.php") + ), + label=5, + ), + attr="mode", + value=PEVar(id="mode_12636"), + ), + ), + ), + ), + rhs=PSkip(), +) + + +delta_p_puppet_2 = PSeq( + lhs=PSeq( + lhs=PSkip(), + rhs=PSeq( + lhs=PSeq( + lhs=PSeq( + lhs=PLet( + id="state_3510", + expr=PRLet( + id="literal-1", + expr=PEConst(const=PStr(value="absent")), + label=1, + ), + label=0, + body=PAttr( + path=PRLet( + id="literal-2", + expr=PEConst(const=PStr(value="/usr/sbin/policy-rc.d")), + label=2, + ), + attr="state", + value=PEVar(id="state_3510"), + ), + ), + rhs=PLet( + id="content_16", + expr=PRLet(id="literal-3", expr=PEUndef(), label=3), + label=-1, + body=PAttr( + path=PRLet( + id="literal-2", + expr=PEConst(const=PStr(value="/usr/sbin/policy-rc.d")), + label=2, + ), + attr="content", + value=PEVar(id="content_16"), + ), + ), + ), + rhs=PLet( + id="owner_256", + expr=PRLet(id="literal-4", expr=PEUndef(), label=4), + label=-3, + body=PAttr( + path=PRLet( + id="literal-2", + expr=PEConst(const=PStr(value="/usr/sbin/policy-rc.d")), + label=2, + ), + attr="owner", + value=PEVar(id="owner_256"), + ), + ), + ), + rhs=PLet( + id="mode_1296", + expr=PRLet(id="literal-5", expr=PEUndef(), label=5), + label=-5, + body=PAttr( + path=PRLet( + id="literal-2", + expr=PEConst(const=PStr(value="/usr/sbin/policy-rc.d")), + label=2, + ), + attr="mode", + value=PEVar(id="mode_1296"), + ), + ), + ), + ), + rhs=PSkip(), +) + + +delta_p_puppet_if = PSeq( + lhs=PIf( + pred=PEVar(id="dejavu-condition-2"), + cons=PSeq( + lhs=PSkip(), + rhs=PSeq( + lhs=PSkip(), + rhs=PSeq( + lhs=PSeq( + lhs=PSeq( + lhs=PLet( + id="condition5668670:dejavu:state_6240", + expr=PRLet( + id="literal-2", + expr=PEConst(const=PStr(value="absent")), + label=2, + ), + label=0, + body=PAttr( + path=PRLet( + id="literal-3", + expr=PEConst( + const=PStr(value="/usr/sbin/policy-rc.d") + ), + label=3, + ), + attr="state", + value=PEVar( + id="condition5668670:dejavu:state_6240" + ), + ), + ), + rhs=PLet( + id="condition5668670:dejavu:content_16", + expr=PRLet(id="literal-4", expr=PEUndef(), label=4), + label=-1, + body=PAttr( + path=PRLet( + id="literal-3", + expr=PEConst( + const=PStr(value="/usr/sbin/policy-rc.d") + ), + label=3, + ), + attr="content", + value=PEVar( + id="condition5668670:dejavu:content_16" + ), + ), + ), + ), + rhs=PLet( + id="condition5668670:dejavu:owner_256", + expr=PRLet(id="literal-5", expr=PEUndef(), label=5), + label=-3, + body=PAttr( + path=PRLet( + id="literal-3", + expr=PEConst( + const=PStr(value="/usr/sbin/policy-rc.d") + ), + label=3, + ), + attr="owner", + value=PEVar(id="condition5668670:dejavu:owner_256"), + ), + ), + ), + rhs=PLet( + id="condition5668670:dejavu:mode_1296", + expr=PRLet(id="literal-6", expr=PEUndef(), label=6), + label=-5, + body=PAttr( + path=PRLet( + id="literal-3", + expr=PEConst(const=PStr(value="/usr/sbin/policy-rc.d")), + label=3, + ), + attr="mode", + value=PEVar(id="condition5668670:dejavu:mode_1296"), + ), + ), + ), + ), + ), + alt=PIf( + pred=PEVar(id="dejavu-condition-1"), + cons=PSeq( + lhs=PSkip(), + rhs=PSeq( + lhs=PSkip(), + rhs=PSeq( + lhs=PSeq( + lhs=PSeq( + lhs=PLet( + id="condition18961807:dejavu:state_25792", + expr=PRLet( + id="literal-7", + expr=PEConst(const=PStr(value="present")), + label=7, + ), + label=1, + body=PAttr( + path=PRLet( + id="literal-8", + expr=PEConst( + const=PStr( + value="/usr/sbin/policy-rc.d" + ) + ), + label=8, + ), + attr="state", + value=PEVar( + id="condition18961807:dejavu:state_25792" + ), + ), + ), + rhs=PLet( + id="condition18961807:dejavu:content_4096", + expr=PRLet(id="literal-9", expr=PEUndef(), label=9), + label=-7, + body=PAttr( + path=PRLet( + id="literal-8", + expr=PEConst( + const=PStr( + value="/usr/sbin/policy-rc.d" + ) + ), + label=8, + ), + attr="content", + value=PEVar( + id="condition18961807:dejavu:content_4096" + ), + ), + ), + ), + rhs=PLet( + id="condition18961807:dejavu:owner_10000", + expr=PRLet(id="literal-10", expr=PEUndef(), label=10), + label=-9, + body=PAttr( + path=PRLet( + id="literal-8", + expr=PEConst( + const=PStr(value="/usr/sbin/policy-rc.d") + ), + label=8, + ), + attr="owner", + value=PEVar( + id="condition18961807:dejavu:owner_10000" + ), + ), + ), + ), + rhs=PLet( + id="condition18961807:dejavu:mode_20736", + expr=PRLet(id="literal-11", expr=PEUndef(), label=11), + label=-11, + body=PAttr( + path=PRLet( + id="literal-8", + expr=PEConst( + const=PStr(value="/usr/sbin/policy-rc.d") + ), + label=8, + ), + attr="mode", + value=PEVar(id="condition18961807:dejavu:mode_20736"), + ), + ), + ), + ), + ), + alt=PSkip(), + ), + ), + rhs=PSkip(), +) + + +delta_p_puppet_default_state = PSeq( + lhs=PSeq( + lhs=PSkip(), + rhs=PSeq( + lhs=PSeq( + lhs=PSeq( + lhs=PLet( + id="state_16", + expr=PRLet( + id="literal-8", + expr=PEConst(const=PStr(value="glitch-undef")), + label=8, + ), + label=-1, + body=PAttr( + path=PRLet( + id="literal-7", + expr=PEConst(const=PStr(value="/root/.ssh/config")), + label=7, + ), + attr="state", + value=PEVar(id="state_16"), + ), + ), + rhs=PLet( + id="content_432", + expr=PRLet(id="literal-9", expr=PEUndef(), label=9), + label=0, + body=PAttr( + path=PRLet( + id="literal-7", + expr=PEConst(const=PStr(value="/root/.ssh/config")), + label=7, + ), + attr="content", + value=PEVar(id="content_432"), + ), + ), + ), + rhs=PLet( + id="owner_288", + expr=PRLet( + id="literal-4", expr=PEConst(const=PStr(value="root")), label=4 + ), + label=1, + body=PAttr( + path=PRLet( + id="literal-7", + expr=PEConst(const=PStr(value="/root/.ssh/config")), + label=7, + ), + attr="owner", + value=PEVar(id="owner_288"), + ), + ), + ), + rhs=PLet( + id="mode_648", + expr=PRLet( + id="literal-6", expr=PEConst(const=PStr(value="0600")), label=6 + ), + label=3, + body=PAttr( + path=PRLet( + id="literal-7", + expr=PEConst(const=PStr(value="/root/.ssh/config")), + label=7, + ), + attr="mode", + value=PEVar(id="mode_648"), + ), + ), + ), + ), + rhs=PSkip(), +) diff --git a/tests/repair/interactive/test_delta_p.py b/tests/repair/interactive/test_delta_p.py new file mode 100644 index 00000000..bedd0a37 --- /dev/null +++ b/tests/repair/interactive/test_delta_p.py @@ -0,0 +1,55 @@ +from tests.repair.interactive.delta_p.delta_p_puppet_scripts import * + + +def test_delta_p_to_filesystems() -> None: + statement = delta_p_puppet + fss = statement.to_filesystems() + assert len(fss) == 1 + assert len(fss[0].state) == 1 + assert "/var/www/customers/public_html/index.php" in fss[0].state + assert ( + fss[0].state["/var/www/customers/public_html/index.php"].attrs["state"] + == "present" + ) + assert ( + fss[0].state["/var/www/customers/public_html/index.php"].attrs["mode"] == "0755" + ) + assert ( + fss[0].state["/var/www/customers/public_html/index.php"].attrs["owner"] + == "web_admin" + ) + assert ( + fss[0].state["/var/www/customers/public_html/index.php"].attrs["content"] + == "

Hello World

" + ) + + +def test_delta_p_to_filesystems_2() -> None: + statement = delta_p_puppet_2 + fss = statement.to_filesystems() + assert len(fss) == 1 + assert len(fss[0].state) == 1 + assert "/usr/sbin/policy-rc.d" in fss[0].state + assert fss[0].state["/usr/sbin/policy-rc.d"].attrs["state"] == "absent" + assert fss[0].state["/usr/sbin/policy-rc.d"].attrs["mode"] == UNDEF + assert fss[0].state["/usr/sbin/policy-rc.d"].attrs["owner"] == UNDEF + assert fss[0].state["/usr/sbin/policy-rc.d"].attrs["content"] == UNDEF + + +def test_delta_p_to_filesystems_if() -> None: + statement = delta_p_puppet_if + fss = statement.to_filesystems() + assert len(fss) == 2 + assert len(fss[0].state) == 1 + assert "/usr/sbin/policy-rc.d" in fss[0].state + assert fss[0].state["/usr/sbin/policy-rc.d"].attrs["state"] == "absent" + assert fss[0].state["/usr/sbin/policy-rc.d"].attrs["mode"] == UNDEF + assert fss[0].state["/usr/sbin/policy-rc.d"].attrs["owner"] == UNDEF + assert fss[0].state["/usr/sbin/policy-rc.d"].attrs["content"] == UNDEF + + assert len(fss[1].state) == 1 + assert "/usr/sbin/policy-rc.d" in fss[1].state + assert fss[1].state["/usr/sbin/policy-rc.d"].attrs["state"] == "present" + assert fss[1].state["/usr/sbin/policy-rc.d"].attrs["mode"] == UNDEF + assert fss[1].state["/usr/sbin/policy-rc.d"].attrs["owner"] == UNDEF + assert fss[1].state["/usr/sbin/policy-rc.d"].attrs["content"] == UNDEF diff --git a/tests/repair/interactive/test_delta_p_minimize.py b/tests/repair/interactive/test_delta_p_minimize.py new file mode 100644 index 00000000..88559d31 --- /dev/null +++ b/tests/repair/interactive/test_delta_p_minimize.py @@ -0,0 +1,102 @@ +from glitch.repair.interactive.delta_p import * + + +def test_delta_p_minimize_let() -> None: + statement = PLet( + "x", + PEConst(const=PStr(value="test1")), + 1, + PAttr( + PEConst(const=PStr(value="test23456")), + "state", + PEConst(const=PStr(value="present")), + ), + ) + + minimized = PStatement.minimize(statement, ["test1"]) + assert isinstance(minimized, PSkip) + + +def test_delta_p_minimize_seq() -> None: + statement = PSeq( + PAttr( + PEConst(const=PStr(value="test1")), + "state", + PEConst(const=PStr(value="present")), + ), + PAttr( + PEConst(const=PStr(value="test2")), + "state", + PEConst(const=PStr(value="present")), + ), + ) + + minimized = PStatement.minimize(statement, ["test1"]) + assert isinstance(minimized, PAttr) + assert minimized.path == PEConst(const=PStr(value="test1")) + + minimized = PStatement.minimize(statement, ["test2"]) + assert isinstance(minimized, PAttr) + assert minimized.path == PEConst(const=PStr(value="test2")) + + minimized = PStatement.minimize(statement, ["test3"]) + assert isinstance(minimized, PSkip) + + +def test_delta_p_minimize_if() -> None: + statement = PIf( + PEConst(const=PBool(True)), + PAttr( + PEConst(const=PStr(value="test2")), + "state", + PEConst(const=PStr(value="present")), + ), + PAttr( + PEConst(const=PStr(value="test3")), + "state", + PEConst(const=PStr(value="present")), + ), + ) + + minimized = PStatement.minimize(statement, ["test2"]) + assert isinstance(minimized, PIf) + assert minimized == PIf( + PEConst(const=PBool(True)), + PAttr( + PEConst(const=PStr(value="test2")), + "state", + PEConst(const=PStr(value="present")), + ), + PSkip(), + ) + + minimized = PStatement.minimize(statement, ["test3"]) + assert isinstance(minimized, PIf) + assert minimized == PIf( + PEConst(const=PBool(True)), + PSkip(), + PAttr( + PEConst(const=PStr(value="test3")), + "state", + PEConst(const=PStr(value="present")), + ), + ) + + minimized = PStatement.minimize(statement, ["test1"]) + assert isinstance(minimized, PSkip) + + +def test_delta_p_minimize_with_add() -> None: + statement = PAttr( + PEBinOP( + PAdd(), PEConst(const=PStr(value="ola")), PEConst(const=PStr(value="2")) + ), + "state", + PEConst(const=PStr(value="present")), + ) + + minimized = PStatement.minimize(statement, ["ola2"]) + assert minimized == statement + + minimized = PStatement.minimize(statement, ["ola3"]) + assert isinstance(minimized, PSkip) diff --git a/tests/repair/interactive/test_patch_solver.py b/tests/repair/interactive/test_patch_solver.py new file mode 100644 index 00000000..e9a90003 --- /dev/null +++ b/tests/repair/interactive/test_patch_solver.py @@ -0,0 +1,2623 @@ +import os +import pytest +import unittest +from tempfile import NamedTemporaryFile + +from glitch.repair.interactive.delta_p import * +from glitch.repair.interactive.solver import PatchSolver, PatchApplier +from glitch.repair.interactive.values import UNDEF +from glitch.parsers.puppet import PuppetParser +from glitch.parsers.ansible import AnsibleParser +from glitch.parsers.chef import ChefParser +from glitch.parsers.terraform import TerraformParser +from glitch.parsers.parser import Parser +from glitch.repair.interactive.compiler.labeler import GLITCHLabeler +from glitch.repair.interactive.compiler.compiler import DeltaPCompiler +from glitch.repr.inter import UnitBlockType, ElementInfo +from glitch.repair.interactive.compiler.names_database import NormalizationVisitor +from glitch.tech import Tech + + +def get_default_file_state(): + state = State() + state.attrs["mode"] = UNDEF + state.attrs["owner"] = UNDEF + state.attrs["state"] = "present" + state.attrs["content"] = UNDEF + return state + + +def get_nil_file_state(): + state = State() + state.attrs["mode"] = UNDEF + state.attrs["owner"] = UNDEF + state.attrs["state"] = "absent" + state.attrs["content"] = UNDEF + return state + + +class TestPatchSolver(unittest.TestCase): + def setUp(self): + ElementInfo.sketched = -1 + self.f = NamedTemporaryFile(mode="w+") + self.labeled_script = None + self.statement = None + + def tearDown(self) -> None: + self.f.close() + assert not os.path.exists(self.f.name) + + def __get_parser(self, tech: Tech) -> Parser: + if tech == Tech.puppet: + return PuppetParser() + elif tech == Tech.ansible: + return AnsibleParser() + elif tech == Tech.chef: + return ChefParser() + elif tech == Tech.terraform: + return TerraformParser() + else: + raise ValueError("Invalid tech") + + def _setup_patch_solver( + self, + script: str, + script_type: UnitBlockType, + tech: Tech, + ) -> None: + self.f.write(script) + self.f.flush() + parser = self.__get_parser(tech) + parsed_file = parser.parse_file(self.f.name, script_type) + assert parsed_file is not None + NormalizationVisitor(tech).visit(parsed_file) + self.labeled_script = GLITCHLabeler.label(parsed_file, tech) + self.statement = DeltaPCompiler(self.labeled_script).compile() + + def _patch_solver_apply( + self, + solver: PatchSolver, + model: Dict[str, Any], + filesystem: SystemState, + tech: Tech, + final_file_content: str, + n_filesystems: int = 1, + ) -> None: + assert self.labeled_script is not None + PatchApplier(solver).apply_patch(model, self.labeled_script) + NormalizationVisitor(tech).visit(self.labeled_script.script) + statement = DeltaPCompiler(self.labeled_script).compile() + filesystems = statement.to_filesystems() + assert len(filesystems) == n_filesystems + assert any(fs.state == filesystem.state for fs in filesystems) + with open(self.f.name) as f: + assert final_file_content == f.read() + + +class TestPatchSolverPuppetScript1(TestPatchSolver): + def setUp(self) -> None: + super().setUp() + puppet_script_1 = """ + file { '/var/www/customers/public_html/index.php': + path => '/var/www/customers/public_html/index.php', + content => '

Hello World

', + ensure => present, + mode => '0755', + owner => 'web_admin' + } + """ + self._setup_patch_solver(puppet_script_1, UnitBlockType.script, Tech.puppet) + + def test_patch_solver_puppet_link(self) -> None: + filesystem = SystemState() + filesystem.state["/var/www/customers/public_html/index.php"] = State() + filesystem.state["/var/www/customers/public_html/index.php"].attrs[ + "state" + ] = "link" + filesystem.state["/var/www/customers/public_html/index.php"].attrs[ + "mode" + ] = "0755" + filesystem.state["/var/www/customers/public_html/index.php"].attrs[ + "owner" + ] = "web_admin" + filesystem.state["/var/www/customers/public_html/index.php"].attrs[ + "content" + ] = UNDEF + + assert self.statement is not None + solver = PatchSolver(self.statement, filesystem) + models = solver.solve() + assert models is not None + assert len(models) == 1 + + result = """ + file { '/var/www/customers/public_html/index.php': + path => '/var/www/customers/public_html/index.php', + ensure => link, + mode => '0755', + owner => 'web_admin' + } + """ + self._patch_solver_apply(solver, models[0], filesystem, Tech.puppet, result) + + def test_patch_solver_puppet_remove_content(self) -> None: + filesystem = SystemState() + filesystem.state["/var/www/customers/public_html/index.php"] = ( + get_default_file_state() + ) + filesystem.state["/var/www/customers/public_html/index.php"].attrs[ + "state" + ] = "present" + filesystem.state["/var/www/customers/public_html/index.php"].attrs[ + "mode" + ] = "0755" + filesystem.state["/var/www/customers/public_html/index.php"].attrs[ + "owner" + ] = "web_admin" + filesystem.state["/var/www/customers/public_html/index.php"].attrs[ + "content" + ] = UNDEF + + assert self.statement is not None + solver = PatchSolver(self.statement, filesystem) + models = solver.solve() + assert models is not None + assert len(models) == 1 + model = models[0] + assert model[str(solver.sum_var)] == 4 + assert model[str(solver.unchanged[5])] == 1 + assert model[str(solver.unchanged[6])] == 0 + assert model[str(solver.unchanged[7])] == 1 + assert model[str(solver.unchanged[8])] == 1 + assert model["content_20672"] == UNDEF + assert model["state_14450"] == "present" + assert model["mode_18972"] == "0755" + assert model["owner_30821"] == "web_admin" + + result = """ + file { '/var/www/customers/public_html/index.php': + path => '/var/www/customers/public_html/index.php', + ensure => present, + mode => '0755', + owner => 'web_admin' + } + """ + self._patch_solver_apply(solver, model, filesystem, Tech.puppet, result) + + def test_patch_solver_puppet_mode(self) -> None: + filesystem = SystemState() + filesystem.state["/var/www/customers/public_html/index.php"] = State() + filesystem.state["/var/www/customers/public_html/index.php"].attrs[ + "state" + ] = "present" + filesystem.state["/var/www/customers/public_html/index.php"].attrs[ + "mode" + ] = "0777" + filesystem.state["/var/www/customers/public_html/index.php"].attrs[ + "owner" + ] = "web_admin" + filesystem.state["/var/www/customers/public_html/index.php"].attrs[ + "content" + ] = "

Hello World

" + + assert self.statement is not None + solver = PatchSolver(self.statement, filesystem) + models = solver.solve() + assert models is not None + assert len(models) == 1 + model = models[0] + assert model[str(solver.sum_var)] == 4 + assert model[str(solver.unchanged[5])] == 1 + assert model[str(solver.unchanged[6])] == 1 + assert model[str(solver.unchanged[7])] == 1 + assert model[str(solver.unchanged[8])] == 0 + assert ( + model["content_20672"] == "

Hello World

" + ) + assert model["state_14450"] == "present" + assert model["mode_18972"] == "0777" + assert model["owner_30821"] == "web_admin" + + result = """ + file { '/var/www/customers/public_html/index.php': + path => '/var/www/customers/public_html/index.php', + content => '

Hello World

', + ensure => present, + mode => '0777', + owner => 'web_admin' + } + """ + self._patch_solver_apply(solver, model, filesystem, Tech.puppet, result) + + def test_patch_solver_puppet_delete_file(self) -> None: + filesystem = SystemState() + filesystem.state["/var/www/customers/public_html/index.php"] = ( + get_nil_file_state() + ) + + assert self.statement is not None + solver = PatchSolver(self.statement, filesystem) + models = solver.solve() + assert models is not None + assert len(models) == 1 + model = models[0] + assert model[str(solver.sum_var)] == 1 + assert model[str(solver.unchanged[5])] == 1 + assert model[str(solver.unchanged[6])] == 0 + assert model[str(solver.unchanged[7])] == 0 + assert model[str(solver.unchanged[8])] == 0 + assert model[str(solver.unchanged[9])] == 0 + assert model["content_20672"] == UNDEF + assert model["state_14450"] == "absent" + assert model["mode_18972"] == UNDEF + assert model["owner_30821"] == UNDEF + + result = """ + file { '/var/www/customers/public_html/index.php': + path => '/var/www/customers/public_html/index.php', + ensure => absent, + } + """ + self._patch_solver_apply(solver, model, filesystem, Tech.puppet, result) + + def test_patch_solver_puppet_timeout(self) -> None: + filesystem = SystemState() + filesystem.state["/var/www/customers/public_html/index.php"] = State() + filesystem.state["/var/www/customers/public_html/index.php"].attrs[ + "state" + ] = "present" + filesystem.state["/var/www/customers/public_html/index.php"].attrs[ + "mode" + ] = "0777" + filesystem.state["/var/www/customers/public_html/index.php"].attrs[ + "owner" + ] = "web_admin" + filesystem.state["/var/www/customers/public_html/index.php"].attrs[ + "content" + ] = "

Hello World

" + + assert self.statement is not None + solver = PatchSolver(self.statement, filesystem, timeout=0) + pytest.raises(TimeoutError, solver.solve) + + def test_patch_solver_puppet_memory_limit(self) -> None: + filesystem = SystemState() + filesystem.state["/var/www/customers/public_html/index.php"] = State() + filesystem.state["/var/www/customers/public_html/index.php"].attrs[ + "state" + ] = "present" + filesystem.state["/var/www/customers/public_html/index.php"].attrs[ + "mode" + ] = "0777" + filesystem.state["/var/www/customers/public_html/index.php"].attrs[ + "owner" + ] = "web_admin" + filesystem.state["/var/www/customers/public_html/index.php"].attrs[ + "content" + ] = "

Hello World

" + + assert self.statement is not None + solver = PatchSolver(self.statement, filesystem, memory_limit=1024 * 10) + pytest.raises(MemoryError, solver.solve) + + +class TestPatchSolverPuppetScript2(TestPatchSolver): + def setUp(self): + super().setUp() + puppet_script_2 = """ + file { '/etc/icinga2/conf.d/test.conf': + ensure => file, + tag => 'icinga2::config::file', + } + """ + self._setup_patch_solver(puppet_script_2, UnitBlockType.script, Tech.puppet) + + def test_patch_solver_puppet_owner(self) -> None: + filesystem = SystemState() + filesystem.state["/etc/icinga2/conf.d/test.conf"] = get_nil_file_state() + filesystem.state["/etc/icinga2/conf.d/test.conf"].attrs["state"] = "present" + filesystem.state["/etc/icinga2/conf.d/test.conf"].attrs["owner"] = "new" + + assert self.statement is not None + solver = PatchSolver(self.statement, filesystem) + models = solver.solve() + assert models is not None + assert len(models) == 1 + model = models[0] + assert model[str(solver.sum_var)] == 4 + assert model["state_3159"] == "present" + assert model["content_16"] == UNDEF + assert model["owner_256"] == "new" + assert model["mode_1296"] == UNDEF + + result = """ + file { '/etc/icinga2/conf.d/test.conf': + ensure => file, + tag => 'icinga2::config::file', + owner => 'new', + } + """ + self._patch_solver_apply(solver, model, filesystem, Tech.puppet, result) + + +class TestPatchSolverPuppetScript3(TestPatchSolver): + def setUp(self): + super().setUp() + puppet_script_3 = """ + file { 'test1': + ensure => file, + } + + file { 'test2': + ensure => file, + } + """ + self._setup_patch_solver(puppet_script_3, UnitBlockType.script, Tech.puppet) + + def test_patch_solver_puppet_two_files(self) -> None: + filesystem = SystemState() + filesystem.state["test1"] = get_nil_file_state() + filesystem.state["test1"].attrs["state"] = "present" + filesystem.state["test1"].attrs["owner"] = "new" + filesystem.state["test2"] = get_nil_file_state() + filesystem.state["test2"].attrs["state"] = "present" + filesystem.state["test2"].attrs["mode"] = "0666" + + assert self.statement is not None + solver = PatchSolver(self.statement, filesystem) + models = solver.solve() + assert models is not None + assert len(models) == 2 + model = models[0] + assert model[str(solver.sum_var)] == 8 + + result = """ + file { 'test1': + ensure => file, + owner => 'new', + } + + file { 'test2': + ensure => file, + mode => '0666', + } + """ + self._patch_solver_apply(solver, model, filesystem, Tech.puppet, result) + + +class TestPatchSolverPuppetScript4(TestPatchSolver): + def setUp(self): + super().setUp() + puppet_script_4 = """ + if $x == 'absent' { + file {'/usr/sbin/policy-rc.d': + ensure => absent, + } + } else { + file {'/usr/sbin/policy-rc.d': + ensure => present, + } + } + """ + self._setup_patch_solver(puppet_script_4, UnitBlockType.script, Tech.puppet) + + def test_patch_solver_puppet_if(self) -> None: + filesystem = SystemState() + + filesystem.state["/usr/sbin/policy-rc.d"] = get_nil_file_state() + filesystem.state["/usr/sbin/policy-rc.d"].attrs["state"] = "present" + + assert self.statement is not None + solver = PatchSolver(self.statement, filesystem) + models = solver.solve() + assert models is not None + assert len(models) == 3 + + assert models[0][str(solver.sum_var)] == 10 + assert models[0]["dejavu-condition-1"] + assert not models[0]["dejavu-condition-2"] + + assert models[1][str(solver.sum_var)] == 9 + assert not models[1]["dejavu-condition-1"] + assert models[1]["dejavu-condition-2"] + + result = """ + if $x == 'absent' { + file {'/usr/sbin/policy-rc.d': + ensure => present, + } + } else { + file {'/usr/sbin/policy-rc.d': + ensure => present, + } + } + """ + self._patch_solver_apply( + solver, models[1], filesystem, Tech.puppet, result, n_filesystems=2 + ) + + +class TestPatchSolverPuppetScript5(TestPatchSolver): + def setUp(self): + super().setUp() + puppet_script_5 = """ + file { '/etc/dhcp/dhclient-enter-hooks': + content => 'test', + owner => 'root', + group => 'root', + mode => '0755', + } + """ + self._setup_patch_solver(puppet_script_5, UnitBlockType.script, Tech.puppet) + + def test_patch_solver_puppet_new_attribute_difficult_name(self) -> None: + """ + This test requires the solver to create a new attribute "state". + However, the attribute "state" should be called "ensure" in Puppet, + so it is required to do the translation back. + """ + filesystem = SystemState() + filesystem.state["/etc/dhcp/dhclient-enter-hooks"] = get_nil_file_state() + + assert self.statement is not None + solver = PatchSolver(self.statement, filesystem) + models = solver.solve() + assert models is not None + assert len(models) == 1 + + result = """ + file { '/etc/dhcp/dhclient-enter-hooks': + group => 'root', + ensure => absent, + } + """ + self._patch_solver_apply(solver, models[0], filesystem, Tech.puppet, result) + + +class TestPatchSolverPuppetScript6(TestPatchSolver): + def setUp(self): + super().setUp() + puppet_script_6 = """ + define apache::vhost ( + String[1] $servername = "test", + String[1] $owner = "owner-test", + ) { + file { "${servername}.conf": + ensure => file, + owner => $owner, + group => 'www', + mode => '0644', + content => 'test', + } + } + + apache::vhost { 'test_vhost': + } + + apache::vhost { 'test_vhost_2': + servername => "test2", + owner => 'new_owner', + } + """ + self._setup_patch_solver(puppet_script_6, UnitBlockType.script, Tech.puppet) + + def test_patch_solver_puppet_defined_resource_delete(self) -> None: + filesystem = SystemState() + filesystem.state["test.conf"] = get_nil_file_state() + filesystem.state["test2.conf"] = get_nil_file_state() + + assert self.statement is not None + solver = PatchSolver(self.statement, filesystem) + models = solver.solve() + assert models is not None + assert len(models) == 4 + + result = """ + define apache::vhost ( + String[1] $servername = "test", + ) { + file { "${servername}.conf": + ensure => absent, + group => 'www', + } + } + + apache::vhost { 'test_vhost': + } + + apache::vhost { 'test_vhost_2': + servername => "test2", + } + """ + self._patch_solver_apply(solver, models[0], filesystem, Tech.puppet, result) + + def test_patch_solver_puppet_defined_resource_change_owner(self) -> None: + filesystem = SystemState() + filesystem.state["test.conf"] = get_default_file_state() + filesystem.state["test.conf"].attrs["owner"] = "owner-test" + filesystem.state["test.conf"].attrs["mode"] = "0644" + filesystem.state["test.conf"].attrs["content"] = "test" + + filesystem.state["test2.conf"] = get_default_file_state() + filesystem.state["test2.conf"].attrs["owner"] = "owner-test2" + filesystem.state["test2.conf"].attrs["mode"] = "0644" + filesystem.state["test2.conf"].attrs["content"] = "test" + + assert self.statement is not None + solver = PatchSolver(self.statement, filesystem) + models = solver.solve() + assert models is not None + assert len(models) == 4 + + result = """ + define apache::vhost ( + String[1] $servername = "test", + String[1] $owner = "owner-test", + ) { + file { "${servername}.conf": + ensure => file, + owner => $owner, + group => 'www', + mode => '0644', + content => 'test', + } + } + + apache::vhost { 'test_vhost': + } + + apache::vhost { 'test_vhost_2': + servername => "test2", + owner => 'owner-test2', + } + """ + self._patch_solver_apply(solver, models[0], filesystem, Tech.puppet, result) + + +class TestPatchSolverPuppetScript7(TestPatchSolver): + def setUp(self): + super().setUp() + puppet_script_7 = """ +$old_config = loadyamlv2('/etc/fuel/cluster/astute.yaml.old','notfound') + +# If it's a redeploy and the file exists we can proceed +if($old_config != 'notfound') +{ + $old_gw_type = $old_config['midonet']['gateway_type'] + if ($old_gw_type == 'bgp') { + + file { 'delete router interfaces script': + ensure => present, + path => '/tmp/delete_router_interfaces_bgp.sh', + content => template('/etc/fuel/plugins/midonet-9.2/puppet/templates/delete_router_interfaces_bgp.sh.erb'), + } + } +} +""" + self._setup_patch_solver(puppet_script_7, UnitBlockType.script, Tech.puppet) + + def test_patch_solver_puppet_inner_if(self) -> None: + filesystem = SystemState() + + filesystem.state["/tmp/delete_router_interfaces_bgp.sh"] = get_nil_file_state() + filesystem.state["/tmp/delete_router_interfaces_bgp.sh"].attrs[ + "state" + ] = "present" + filesystem.state["/tmp/delete_router_interfaces_bgp.sh"].attrs[ + "content" + ] = "test" + + assert self.statement is not None + solver = PatchSolver(self.statement, filesystem) + models = solver.solve() + assert models is not None + assert len(models) == 1 + + result = """ +$old_config = loadyamlv2('/etc/fuel/cluster/astute.yaml.old','notfound') + +# If it's a redeploy and the file exists we can proceed +if($old_config != 'notfound') +{ + $old_gw_type = $old_config['midonet']['gateway_type'] + if ($old_gw_type == 'bgp') { + + file { 'delete router interfaces script': + ensure => present, + path => '/tmp/delete_router_interfaces_bgp.sh', + content => 'test', + } + } +} +""" + self._patch_solver_apply(solver, models[0], filesystem, Tech.puppet, result) + + +class TestPatchSolverPuppetScript8(TestPatchSolver): + def setUp(self): + super().setUp() + puppet_script_8 = """$gitrevision = '$Id$' + +file { '/var/lib/puppet/gitrevision.txt' : + ensure => 'present', + owner => 'root', + group => 'root', + mode => '0444', + content => $gitrevision, + require => File['/var/lib/puppet'], +} + """ + self._setup_patch_solver(puppet_script_8, UnitBlockType.script, Tech.puppet) + + def test_patch_solver_puppet_delete_variable(self) -> None: + filesystem = SystemState() + filesystem.state["/var/lib/puppet/gitrevision.txt"] = get_nil_file_state() + + assert self.statement is not None + solver = PatchSolver(self.statement, filesystem) + models = solver.solve() + assert models is not None + assert len(models) == 1 + + result = """ +file { '/var/lib/puppet/gitrevision.txt' : + ensure => 'absent', + group => 'root', + require => File['/var/lib/puppet'], +} + """ + self._patch_solver_apply(solver, models[0], filesystem, Tech.puppet, result) + + +class TestPatchSolverPuppetScript9(TestPatchSolver): + def setUp(self): + super().setUp() + puppet_script_9 = """ +user { 'mysql': + ensure => present, +} +""" + self._setup_patch_solver(puppet_script_9, UnitBlockType.script, Tech.puppet) + + def test_patch_solver_puppet_user_delete(self) -> None: + filesystem = SystemState() + filesystem.state["user:mysql"] = State() + filesystem.state["user:mysql"].attrs["state"] = "absent" + + assert self.statement is not None + solver = PatchSolver(self.statement, filesystem) + models = solver.solve() + assert models is not None + assert len(models) == 1 + + result = """ +user { 'mysql': + ensure => absent, +} +""" + self._patch_solver_apply(solver, models[0], filesystem, Tech.puppet, result) + + +class TestPatchSolverPuppetScript10(TestPatchSolver): + def setUp(self): + super().setUp() + puppet_script_10 = """ +file { '/etc/plumgrid': + ensure => directory, + mode => 0755, +} +""" + self._setup_patch_solver(puppet_script_10, UnitBlockType.script, Tech.puppet) + + def test_patch_solver_puppet_integer_mode(self) -> None: + filesystem = SystemState() + filesystem.state["/etc/plumgrid"] = get_nil_file_state() + filesystem.state["/etc/plumgrid"].attrs["state"] = "present" + filesystem.state["/etc/plumgrid"].attrs["mode"] = "0756" + + assert self.statement is not None + solver = PatchSolver(self.statement, filesystem) + models = solver.solve() + assert models is not None + assert len(models) == 1 + + result = """ +file { '/etc/plumgrid': + ensure => present, + mode => 0756, +} +""" + self._patch_solver_apply(solver, models[0], filesystem, Tech.puppet, result) + + +class TestPatchSolverPuppetScript11(TestPatchSolver): + def setUp(self): + super().setUp() + puppet_script_11 = """ +file { '/usr/local/bin': + ensure => 'directory', + owner => $boxen_user, + group => 'staff', + mode => '0755' +} +""" + self._setup_patch_solver(puppet_script_11, UnitBlockType.script, Tech.puppet) + + # @unittest.skip("Not implemented yet") + def test_patch_solver_puppet_variable_undefined(self) -> None: + # The problem is that there is no literal to repair and so + # the solver isn't able to get a solution + filesystem = SystemState() + filesystem.state["/usr/local/bin"] = State() + filesystem.state["/usr/local/bin"].attrs["state"] = "directory" + filesystem.state["/usr/local/bin"].attrs["mode"] = "0755" + filesystem.state["/usr/local/bin"].attrs["owner"] = "test" + filesystem.state["/usr/local/bin"].attrs["content"] = UNDEF + + assert self.statement is not None + solver = PatchSolver(self.statement, filesystem) + models = solver.solve() + assert models is not None + assert len(models) == 1 + + result = """ +file { '/usr/local/bin': + ensure => 'directory', + owner => 'test', + group => 'staff', + mode => '0755' +} +""" + self._patch_solver_apply(solver, models[0], filesystem, Tech.puppet, result) + + +class TestPatchSolverPuppetScript12(TestPatchSolver): + def setUp(self): + super().setUp() + puppet_script_12 = """ +$data_dir = '/etc/hiera' +file { 'hiera_data_dir' : + ensure => 'directory', + path => $data_dir, + mode => '0751', +} +""" + self._setup_patch_solver(puppet_script_12, UnitBlockType.script, Tech.puppet) + + def test_patch_solver_puppet_path_variable(self) -> None: + filesystem = SystemState() + filesystem.state["/etc/hiera"] = State() + filesystem.state["/etc/hiera"].attrs["state"] = "directory" + filesystem.state["/etc/hiera"].attrs["mode"] = "0751" + filesystem.state["/etc/hiera"].attrs["owner"] = UNDEF + filesystem.state["/etc/hiera"].attrs["content"] = UNDEF + + assert self.statement is not None + solver = PatchSolver(self.statement, filesystem) + models = solver.solve() + assert models is not None + assert len(models) == 1 + + result = """ +$data_dir = '/etc/hiera' +file { 'hiera_data_dir' : + ensure => 'directory', + path => $data_dir, + mode => '0751', +} +""" + self._patch_solver_apply(solver, models[0], filesystem, Tech.puppet, result) + + +class TestPatchSolverPuppetScript13(TestPatchSolver): + def setUp(self): + super().setUp() + puppet_script_13 = """ +define nginx($includes) { + file { $includes: + ensure => directory, + mode => '0644', + owner => 'root', + group => 'root', + } +} + +$includedir = '/etc/nginx/includes' +nginx { 'nginx': + includes => $includedir, +} +""" + self._setup_patch_solver(puppet_script_13, UnitBlockType.script, Tech.puppet) + + def test_patch_solver_puppet_defined_resource_var_path(self): + filesystem = SystemState() + filesystem.state["/etc/nginx/includes"] = State() + filesystem.state["/etc/nginx/includes"].attrs["state"] = "directory" + filesystem.state["/etc/nginx/includes"].attrs["mode"] = "0751" + filesystem.state["/etc/nginx/includes"].attrs["owner"] = "root" + filesystem.state["/etc/nginx/includes"].attrs["content"] = UNDEF + + assert self.statement is not None + solver = PatchSolver(self.statement, filesystem) + models = solver.solve() + assert models is not None + assert len(models) == 1 + + result = """ +define nginx($includes) { + file { $includes: + ensure => directory, + mode => '0751', + owner => 'root', + group => 'root', + } +} + +$includedir = '/etc/nginx/includes' +nginx { 'nginx': + includes => $includedir, +} +""" + self._patch_solver_apply(solver, models[0], filesystem, Tech.puppet, result) + + +class TestPatchSolverPuppetScript14(TestPatchSolver): + def setUp(self): + super().setUp() + puppet_script_14 = """ +$::bind::params::binduser = 'bind' + +define bind() { + file {'/var/named/chroot/var/log/named': + ensure => directory, + owner => $::bind::params::binduser, + group => $::bind::params::bindgroup, + mode => '0770', + seltype => 'var_log_t', + } +} + +bind { 'bind': +} +""" + self._setup_patch_solver(puppet_script_14, UnitBlockType.script, Tech.puppet) + + def test_patch_solver_puppet_defined_resource_change_outside_owner(self): + filesystem = SystemState() + filesystem.state["/var/named/chroot/var/log/named"] = State() + filesystem.state["/var/named/chroot/var/log/named"].attrs["state"] = "directory" + filesystem.state["/var/named/chroot/var/log/named"].attrs["mode"] = "0770" + filesystem.state["/var/named/chroot/var/log/named"].attrs["owner"] = "new-owner" + filesystem.state["/var/named/chroot/var/log/named"].attrs["content"] = UNDEF + + assert self.statement is not None + solver = PatchSolver(self.statement, filesystem) + models = solver.solve() + assert models is not None + assert len(models) == 1 + result = """ +$::bind::params::binduser = 'new-owner' + +define bind() { + file {'/var/named/chroot/var/log/named': + ensure => directory, + owner => $::bind::params::binduser, + group => $::bind::params::bindgroup, + mode => '0770', + seltype => 'var_log_t', + } +} + +bind { 'bind': +} +""" + self._patch_solver_apply(solver, models[0], filesystem, Tech.puppet, result) + + +class TestPatchSolverPuppetScript15(TestPatchSolver): + def setUp(self): + super().setUp() + puppet_script_15 = """ +package { 'openssl': + ensure => installed, + name => 'openssl', +} +""" + self._setup_patch_solver(puppet_script_15, UnitBlockType.script, Tech.puppet) + + def test_patch_solver_puppet_remove_package(self): + filesystem = SystemState() + filesystem.state["package:openssl"] = State() + filesystem.state["package:openssl"].attrs["state"] = "absent" + + assert self.statement is not None + solver = PatchSolver(self.statement, filesystem) + models = solver.solve() + assert models is not None + assert len(models) == 1 + result = """ +package { 'openssl': + ensure => absent, + name => 'openssl', +} +""" + self._patch_solver_apply(solver, models[0], filesystem, Tech.puppet, result) + + def test_patch_solver_puppet_latest_package(self): + filesystem = SystemState() + filesystem.state["package:openssl"] = State() + filesystem.state["package:openssl"].attrs["state"] = "latest" + + assert self.statement is not None + solver = PatchSolver(self.statement, filesystem) + models = solver.solve() + assert models is not None + assert len(models) == 1 + result = """ +package { 'openssl': + ensure => latest, + name => 'openssl', +} +""" + self._patch_solver_apply(solver, models[0], filesystem, Tech.puppet, result) + + def test_patch_solver_puppet_purge_package(self): + filesystem = SystemState() + filesystem.state["package:openssl"] = State() + filesystem.state["package:openssl"].attrs["state"] = "purged" + + assert self.statement is not None + solver = PatchSolver(self.statement, filesystem) + models = solver.solve() + assert models is not None + assert len(models) == 1 + result = """ +package { 'openssl': + ensure => purged, + name => 'openssl', +} +""" + self._patch_solver_apply(solver, models[0], filesystem, Tech.puppet, result) + + def test_patch_solver_puppet_disabled_package(self): + filesystem = SystemState() + filesystem.state["package:openssl"] = State() + filesystem.state["package:openssl"].attrs["state"] = "disabled" + + assert self.statement is not None + solver = PatchSolver(self.statement, filesystem) + models = solver.solve() + assert models is not None + assert len(models) == 1 + result = """ +package { 'openssl': + ensure => disabled, + name => 'openssl', +} +""" + self._patch_solver_apply(solver, models[0], filesystem, Tech.puppet, result) + + +class TestPatchSolverPuppetScript16(TestPatchSolver): + def setUp(self): + super().setUp() + puppet_script_16 = """ +service { 'openssl': + ensure => running, + name => 'openssl', +} +""" + self._setup_patch_solver(puppet_script_16, UnitBlockType.script, Tech.puppet) + + def test_patch_solver_puppet_stop_service(self): + filesystem = SystemState() + filesystem.state["service:openssl"] = State() + filesystem.state["service:openssl"].attrs["state"] = "stop" + filesystem.state["service:openssl"].attrs["enabled"] = UNDEF + + assert self.statement is not None + solver = PatchSolver(self.statement, filesystem) + models = solver.solve() + assert models is not None + assert len(models) == 1 + result = """ +service { 'openssl': + ensure => stopped, + name => 'openssl', +} +""" + self._patch_solver_apply(solver, models[0], filesystem, Tech.puppet, result) + + def test_patch_solver_puppet_enable_service(self): + filesystem = SystemState() + filesystem.state["service:openssl"] = State() + filesystem.state["service:openssl"].attrs["state"] = "start" + filesystem.state["service:openssl"].attrs["enabled"] = "true" + + assert self.statement is not None + solver = PatchSolver(self.statement, filesystem) + models = solver.solve() + assert models is not None + assert len(models) == 1 + result = """ +service { 'openssl': + ensure => running, + name => 'openssl', + enable => true, +} +""" + self._patch_solver_apply(solver, models[0], filesystem, Tech.puppet, result) + + +class TestPatchSolverPuppetScript17(TestPatchSolver): + def setUp(self): + super().setUp() + puppet_script_17 = """ +$::bind::params::packagenameprefix = 'bind9' + +define bind::package ( + $packagenameprefix = $::bind::params::packagenameprefix, + $packagenamesuffix = '', +) { + package { "$packagenameprefix$packagenamesuffix": + ensure => present + } +} + +if $chroot == 'true' { + bind::package { 'bind::package': + packagenamesuffix => "-chroot" + } +} else { + bind::package { 'bind::package': + packagenamesuffix => "" + } +} +""" + self._setup_patch_solver(puppet_script_17, UnitBlockType.script, Tech.puppet) + + def test_patch_solver_puppet_if_with_defined(self): + filesystem = SystemState() + filesystem.state["package:$packagenameprefix$packagenamesuffix"] = State() + filesystem.state["package:$packagenameprefix$packagenamesuffix"].attrs[ + "state" + ] = "latest" + + assert self.statement is not None + self.statement = PStatement.minimize( + self.statement, ["package:$packagenameprefix$packagenamesuffix"] + ) + solver = PatchSolver(self.statement, filesystem) + models = solver.solve() + assert models is not None + assert len(models) == 2 + result = """ +$::bind::params::packagenameprefix = 'bind9' + +define bind::package ( + $packagenameprefix = $::bind::params::packagenameprefix, + $packagenamesuffix = '', +) { + package { "$packagenameprefix$packagenamesuffix": + ensure => latest + } +} + +if $chroot == 'true' { + bind::package { 'bind::package': + packagenamesuffix => "-chroot" + } +} else { + bind::package { 'bind::package': + packagenamesuffix => "" + } +} +""" + self._patch_solver_apply( + solver, models[0], filesystem, Tech.puppet, result, n_filesystems=2 + ) + + +class TestPatchSolverPuppetScript18(TestPatchSolver): + def setUp(self): + super().setUp() + puppet_script_18 = """ +service { 'keystone': + ensure => running, + enable => true, +} +""" + self._setup_patch_solver(puppet_script_18, UnitBlockType.script, Tech.puppet) + + def test_patch_solver_puppet_service(self): + filesystem = SystemState() + filesystem.state["service:keystone"] = State() + filesystem.state["service:keystone"].attrs["state"] = "stop" + filesystem.state["service:keystone"].attrs["enabled"] = "true" + + assert self.statement is not None + solver = PatchSolver(self.statement, filesystem) + models = solver.solve() + assert models is not None + assert len(models) == 1 + result = """ +service { 'keystone': + ensure => stopped, + enable => true, +} +""" + self._patch_solver_apply(solver, models[0], filesystem, Tech.puppet, result) + + def test_patch_solver_puppet_disable_service(self): + filesystem = SystemState() + filesystem.state["service:keystone"] = State() + filesystem.state["service:keystone"].attrs["state"] = "start" + filesystem.state["service:keystone"].attrs["enabled"] = "false" + + assert self.statement is not None + solver = PatchSolver(self.statement, filesystem) + models = solver.solve() + assert models is not None + assert len(models) == 1 + result = """ +service { 'keystone': + ensure => running, + enable => false, +} +""" + self._patch_solver_apply(solver, models[0], filesystem, Tech.puppet, result) + + +class TestPatchSolverAnsibleScript1(TestPatchSolver): + def setUp(self): + super().setUp() + ansible_script_1 = """ +--- +- ansible.builtin.file: + path: "/var/www/customers/public_html/index.php" + state: file + owner: "web_admin" + mode: '0755' +""" + self._setup_patch_solver(ansible_script_1, UnitBlockType.tasks, Tech.ansible) + + def test_patch_solver_ansible_mode(self) -> None: + filesystem = SystemState() + filesystem.state["/var/www/customers/public_html/index.php"] = ( + get_default_file_state() + ) + filesystem.state["/var/www/customers/public_html/index.php"].attrs[ + "mode" + ] = "0777" + filesystem.state["/var/www/customers/public_html/index.php"].attrs[ + "owner" + ] = "web_admin" + filesystem.state["/var/www/customers/public_html/index.php"].attrs[ + "content" + ] = UNDEF + + assert self.statement is not None + solver = PatchSolver(self.statement, filesystem) + models = solver.solve() + assert models is not None + assert len(models) == 1 + model = models[0] + + result = """ +--- +- ansible.builtin.file: + path: "/var/www/customers/public_html/index.php" + state: file + owner: "web_admin" + mode: '0777' +""" + self._patch_solver_apply(solver, model, filesystem, Tech.ansible, result) + + +class TestPatchSolverAnsibleScript2(TestPatchSolver): + def setUp(self): + super().setUp() + self.ansible_script_2 = """ +--- +- host: localhost + vars: + name: index.php + owner: admin + tasks: + - ansible.builtin.file: + path: "/var/www/customers/public_html/{{ name }}" + state: file + owner: "web_{{ owner }}" + mode: '0755' +""" + self._setup_patch_solver( + self.ansible_script_2, UnitBlockType.script, Tech.ansible + ) + + def test_patch_solver_ansible_owner(self) -> None: + filesystem = SystemState() + filesystem.state["/var/www/customers/public_html/index.php"] = ( + get_default_file_state() + ) + filesystem.state["/var/www/customers/public_html/index.php"].attrs[ + "mode" + ] = "0755" + filesystem.state["/var/www/customers/public_html/index.php"].attrs[ + "owner" + ] = "web_user" + filesystem.state["/var/www/customers/public_html/index.php"].attrs[ + "content" + ] = UNDEF + + assert self.statement is not None + solver = PatchSolver(self.statement, filesystem) + models = solver.solve() + assert models is not None + assert len(models) == 6 + model = models[0] + assert model[str(solver.sum_var)] == 5 + assert model[str(solver.unchanged[8])] == 1 + assert model[str(solver.unchanged[9])] == 1 + assert model[str(solver.unchanged[10])] == 0 + assert model[str(solver.unchanged[11])] == 1 + assert model[str(solver.unchanged[12])] == 1 + assert model[str(solver.unchanged[13])] == 0 + assert model[str(solver.unchanged[14])] == 1 + assert model["state_18000"] == "present" + assert model["owner_35937"] == "web_user" + assert model["mode_27216"] == "0755" + assert model["content_16"] == UNDEF + + result = """ +--- +- host: localhost + vars: + name: index.php + owner: + tasks: + - ansible.builtin.file: + path: "/var/www/customers/public_html/{{ name }}" + state: file + owner: "web_user{{ owner }}" + mode: '0755' +""" + check = False + for model in models: + try: + self._patch_solver_apply( + solver, model, filesystem, Tech.ansible, result + ) + check = True + break + except AssertionError: + check = False + with open(self.f.name, "w") as f: + f.write(self.ansible_script_2) + assert check + + +class TestPatchSolverAnsibleScript3(TestPatchSolver): + def setUp(self): + super().setUp() + ansible_script_3 = """--- +- name: Add the user 'johnd' + ansible.builtin.user: + name: johnd +""" + self._setup_patch_solver(ansible_script_3, UnitBlockType.unknown, Tech.ansible) + + def test_patch_solver_ansible_user_delete(self) -> None: + filesystem = SystemState() + filesystem.state["user:johnd"] = State() + filesystem.state["user:johnd"].attrs["state"] = "absent" + + assert self.statement is not None + solver = PatchSolver(self.statement, filesystem) + models = solver.solve() + assert models is not None + assert len(models) == 1 + + result = """--- +- name: Add the user 'johnd' + ansible.builtin.user: + name: johnd + state: absent +""" + self._patch_solver_apply(solver, models[0], filesystem, Tech.ansible, result) + + +class TestPatchSolverAnsibleScript4(TestPatchSolver): + def setUp(self): + super().setUp() + ansible_script_4 = """--- +- name: Version + hosts: debian + vars: + tests_dir: /molecule/symfony_cli/version + tasks: + - name: Clean tests dir # noqa risky-file-permissions + file: + path: "{{ tests_dir }}" + state: "{{ item }}" + loop: [absent, directory] +""" + self._setup_patch_solver(ansible_script_4, UnitBlockType.script, Tech.ansible) + + @unittest.skip("Not supported yet") + def test_patch_solver_ansible_item(self) -> None: + filesystem = SystemState() + filesystem.state["/molecule/symfony_cli/version"] = get_nil_file_state() + filesystem.state["/molecule/symfony_cli/version"].attrs["state"] = "directory" + filesystem.state["/molecule/symfony_cli/version"].attrs["mode"] = "0755" + filesystem.state["/molecule/symfony_cli/version"].attrs["owner"] = UNDEF + filesystem.state["/molecule/symfony_cli/version"].attrs["content"] = UNDEF + + assert self.statement is not None + solver = PatchSolver(self.statement, filesystem) + models = solver.solve() + assert models is not None + assert len(models) == 1 + + result = """--- +- name: Version + hosts: debian + vars: + tests_dir: /molecule/symfony_cli/version + tasks: + - name: Clean tests dir # noqa risky-file-permissions + file: + path: "{{ tests_dir }}" + state: "directory" + mode: "0755" + loop: [absent, directory] +""" + self._patch_solver_apply(solver, models[0], filesystem, Tech.ansible, result) + + +class TestPatchSolverAnsibleScript5(TestPatchSolver): + def setUp(self): + super().setUp() + ansible_script_5 = """--- +- name: "INITIAL SETUP" + hosts: games.infra.netsoc.co + become: yes + tasks: + - name: "Make /netsoc only readable by root" + file: + path: "/netsoc" + owner: root + group: root + mode: '1770' + +- name: Ensure Minecraft Servers + hosts: games.infra.netsoc.co + roles: + - role: minecraft + vars: + mount: "/netsoc/minecraft" +""" + self._setup_patch_solver(ansible_script_5, UnitBlockType.script, Tech.ansible) + + def test_patch_solver_ansible_unit_block(self) -> None: + filesystem = SystemState() + filesystem.state["/netsoc"] = get_nil_file_state() + + assert self.statement is not None + solver = PatchSolver(self.statement, filesystem) + models = solver.solve() + assert models is not None + assert len(models) == 1 + + result = """--- +- name: "INITIAL SETUP" + hosts: games.infra.netsoc.co + become: yes + tasks: + - name: "Make /netsoc only readable by root" + file: + path: "/netsoc" + group: root + state: absent + +- name: Ensure Minecraft Servers + hosts: games.infra.netsoc.co + roles: + - role: minecraft + vars: + mount: "/netsoc/minecraft" +""" + self._patch_solver_apply(solver, models[0], filesystem, Tech.ansible, result) + + +class TestPatchSolverAnsibleScript6(TestPatchSolver): + def setUp(self): + super().setUp() + ansible_script_6 = """--- +- name: Install ntpdate + ansible.builtin.package: + name: ntpdate + state: present +""" + self._setup_patch_solver(ansible_script_6, UnitBlockType.tasks, Tech.ansible) + + def test_patch_solver_ansible_remove_package(self) -> None: + filesystem = SystemState() + filesystem.state["package:ntpdate"] = State() + filesystem.state["package:ntpdate"].attrs["state"] = "absent" + + assert self.statement is not None + solver = PatchSolver(self.statement, filesystem) + models = solver.solve() + assert models is not None + assert len(models) == 1 + + result = """--- +- name: Install ntpdate + ansible.builtin.package: + name: ntpdate + state: absent +""" + self._patch_solver_apply(solver, models[0], filesystem, Tech.ansible, result) + + def test_patch_solver_ansible_latest_package(self) -> None: + filesystem = SystemState() + filesystem.state["package:ntpdate"] = State() + filesystem.state["package:ntpdate"].attrs["state"] = "latest" + + assert self.statement is not None + solver = PatchSolver(self.statement, filesystem) + models = solver.solve() + assert models is not None + assert len(models) == 1 + + result = """--- +- name: Install ntpdate + ansible.builtin.package: + name: ntpdate + state: latest +""" + self._patch_solver_apply(solver, models[0], filesystem, Tech.ansible, result) + + +class TestPatchSolverAnsibleScript7(TestPatchSolver): + def setUp(self): + super().setUp() + ansible_script_7 = """--- +- name: Start service httpd, if not started + ansible.builtin.service: + name: httpd + state: started +""" + self._setup_patch_solver(ansible_script_7, UnitBlockType.tasks, Tech.ansible) + + def test_patch_solver_ansible_stop_service(self) -> None: + filesystem = SystemState() + filesystem.state["service:httpd"] = State() + filesystem.state["service:httpd"].attrs["state"] = "stop" + filesystem.state["service:httpd"].attrs["enabled"] = UNDEF + + assert self.statement is not None + solver = PatchSolver(self.statement, filesystem) + models = solver.solve() + assert models is not None + assert len(models) == 1 + + result = """--- +- name: Start service httpd, if not started + ansible.builtin.service: + name: httpd + state: stopped +""" + self._patch_solver_apply(solver, models[0], filesystem, Tech.ansible, result) + + def test_patch_solver_ansible_enable_service(self) -> None: + filesystem = SystemState() + filesystem.state["service:httpd"] = State() + filesystem.state["service:httpd"].attrs["state"] = "start" + filesystem.state["service:httpd"].attrs["enabled"] = "true" + + assert self.statement is not None + solver = PatchSolver(self.statement, filesystem) + models = solver.solve() + assert models is not None + assert len(models) == 1 + + result = """--- +- name: Start service httpd, if not started + ansible.builtin.service: + name: httpd + state: started + enabled: true +""" + self._patch_solver_apply(solver, models[0], filesystem, Tech.ansible, result) + + +class TestPatchSolverAnsibleScript8(TestPatchSolver): + def setUp(self): + super().setUp() + ansible_script_8 = """--- +- name: create-user + user: + name: teamcity + home: /opt/teamcity +- name: add user to docker group + user: + name: teamcity + group: docker + append: yes +""" + self._setup_patch_solver(ansible_script_8, UnitBlockType.unknown, Tech.ansible) + + def test_patch_solver_ansible_overrided_user(self) -> None: + filesystem = SystemState() + filesystem.state["user:teamcity"] = State() + filesystem.state["user:teamcity"].attrs["state"] = "absent" + + assert self.statement is not None + self.statement = PStatement.minimize(self.statement, ["user:teamcity"]) + solver = PatchSolver(self.statement, filesystem, timeout=10) + models = solver.solve() + assert models is not None + assert len(models) == 1 + + result = """--- +- name: create-user + user: + name: teamcity + home: /opt/teamcity +- name: add user to docker group + user: + name: teamcity + group: docker + append: yes + state: absent +""" + self._patch_solver_apply(solver, models[0], filesystem, Tech.ansible, result) + + +class TestPatchSolverAnsibleScript9(TestPatchSolver): + def setUp(self): + super().setUp() + ansible_script_9 = """--- +- name: Upgrade all the things! + hosts: all:!localhost + become: True + + vars: + tgt_tmp_dir: "/tmp" + + tasks: + - name: Create a lockfile + file: + path: '{{ tgt_tmp_dir }}/HELLO' + state: 'touch' + when: dist_upgrade_register_replace is defined and dist_upgrade_register_replace|changed + + - name: Remove the lockfile + file: + path: '{{ tgt_tmp_dir }}/HELLO' + state: 'absent' +""" + self._setup_patch_solver(ansible_script_9, UnitBlockType.script, Tech.ansible) + + def test_patch_solver_ansible_redefined_file(self) -> None: + filesystem = SystemState() + filesystem.state["/tmp/HELLO"] = get_default_file_state() + + assert self.statement is not None + solver = PatchSolver(self.statement, filesystem, timeout=10) + models = solver.solve() + assert models is not None + assert len(models) == 3 + + result = """--- +- name: Upgrade all the things! + hosts: all:!localhost + become: True + + vars: + tgt_tmp_dir: "/tmp" + + tasks: + - name: Create a lockfile + file: + path: '{{ tgt_tmp_dir }}/HELLO' + state: 'touch' + when: dist_upgrade_register_replace is defined and dist_upgrade_register_replace|changed + + - name: Remove the lockfile + file: + path: '{{ tgt_tmp_dir }}/HELLO' + state: 'file' +""" + + self._patch_solver_apply(solver, models[0], filesystem, Tech.ansible, result) + + +class TestPatchSolverAnsibleScript10(TestPatchSolver): + def setUp(self): + super().setUp() + ansible_script_10 = """--- +- name: Install the latest version of Apache + ansible.builtin.yum: + name: httpd + state: latest +""" + + self._setup_patch_solver(ansible_script_10, UnitBlockType.tasks, Tech.ansible) + + def test_patch_solver_ansible_present_package_yum(self) -> None: + filesystem = SystemState() + filesystem.state["package:httpd"] = State() + filesystem.state["package:httpd"].attrs["state"] = "present" + + assert self.statement is not None + solver = PatchSolver(self.statement, filesystem, timeout=10) + models = solver.solve() + assert models is not None + assert len(models) == 1 + + result = """--- +- name: Install the latest version of Apache + ansible.builtin.yum: + name: httpd + state: present +""" + self._patch_solver_apply(solver, models[0], filesystem, Tech.ansible, result) + + +class TestPatchSolverAnsibleScript11(TestPatchSolver): + def setUp(self): + super().setUp() + ansible_script_11 = """--- +- name: Install the latest version of Apache + ansible.builtin.apt: + name: httpd + state: latest +""" + + self._setup_patch_solver(ansible_script_11, UnitBlockType.tasks, Tech.ansible) + + def test_patch_solver_ansible_present_package_apt(self) -> None: + filesystem = SystemState() + filesystem.state["package:httpd"] = State() + filesystem.state["package:httpd"].attrs["state"] = "present" + + assert self.statement is not None + solver = PatchSolver(self.statement, filesystem, timeout=10) + models = solver.solve() + assert models is not None + assert len(models) == 1 + + result = """--- +- name: Install the latest version of Apache + ansible.builtin.apt: + name: httpd + state: present +""" + self._patch_solver_apply(solver, models[0], filesystem, Tech.ansible, result) + + +class TestPatchSolverAnsibleScript12(TestPatchSolver): + def setUp(self): + super().setUp() + ansible_script_12 = """--- +- amazon.aws.s3_bucket: + name: mys3bucket + state: present +""" + self._setup_patch_solver(ansible_script_12, UnitBlockType.tasks, Tech.ansible) + + def test_patch_solver_ansible_aws_bucket(self): + filesystem = SystemState() + filesystem.state["aws_s3_bucket:new_bucket"] = State() + filesystem.state["aws_s3_bucket:new_bucket"].attrs["state"] = "present" + filesystem.state["aws_s3_bucket:new_bucket"].attrs["acl"] = UNDEF + assert self.statement is not None + solver = PatchSolver(self.statement, filesystem, timeout=10) + models = solver.solve() + assert models is not None + assert len(models) == 1 + + result = """--- +- amazon.aws.s3_bucket: + name: new_bucket + state: present +""" + self._patch_solver_apply(solver, models[0], filesystem, Tech.ansible, result) + + +class TestPatchSolverAnsibleScript13(TestPatchSolver): + def setUp(self): + super().setUp() + ansible_script_13 = """--- +- name: restart a particular instance by its ID + amazon.aws.ec2_instance: + name: my_instance + state: present + availability_zone: us-west-2a +""" + self._setup_patch_solver(ansible_script_13, UnitBlockType.tasks, Tech.ansible) + + def test_patch_solver_ansible_aws_instance(self): + filesystem = SystemState() + filesystem.state["aws_instance:my_instance"] = State() + filesystem.state["aws_instance:my_instance"].attrs["state"] = "present" + filesystem.state["aws_instance:my_instance"].attrs[ + "availability_zone" + ] = "us-west-2b" + filesystem.state["aws_instance:my_instance"].attrs["instance_type"] = "t2.micro" + assert self.statement is not None + solver = PatchSolver(self.statement, filesystem, timeout=10) + models = solver.solve() + assert models is not None + assert len(models) == 1 + + result = """--- +- name: restart a particular instance by its ID + amazon.aws.ec2_instance: + name: my_instance + state: present + availability_zone: us-west-2b + instance_type: 't2.micro' +""" + self._patch_solver_apply(solver, models[0], filesystem, Tech.ansible, result) + + +class TestPatchSolverAnsibleScript14(TestPatchSolver): + def setUp(self): + super().setUp() + ansible_script_14 = """--- +- name: Create a role with description and tags + amazon.aws.iam_role: + name: mynewrole + state: present + assume_role_policy_document: | + { + "Version": "2012-10-17", + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Sid": "", + "Principal": { + "Service": "ec2.amazonaws.com" + } + } + ] + } +""" + self._setup_patch_solver(ansible_script_14, UnitBlockType.tasks, Tech.ansible) + + def test_patch_solver_ansible_aws_role(self): + filesystem = SystemState() + filesystem.state["aws_iam_role:mynewrole"] = State() + filesystem.state["aws_iam_role:mynewrole"].attrs["state"] = "present" + filesystem.state["aws_iam_role:mynewrole"].attrs[ + "assume_role_policy" + ] = """{ + "Version": "2012-10-17", + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Deny", + "Sid": "", + "Principal": { + "Service": "ec2.amazonaws.com" + } + } + ] +}""" + assert self.statement is not None + solver = PatchSolver(self.statement, filesystem, timeout=10) + models = solver.solve() + assert models is not None + assert len(models) == 1 + model = models[0] + + result = """--- +- name: Create a role with description and tags + amazon.aws.iam_role: + name: mynewrole + state: present + assume_role_policy_document: | + { + "Version": "2012-10-17", + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Deny", + "Sid": "", + "Principal": { + "Service": "ec2.amazonaws.com" + } + } + ] +} +""" + self._patch_solver_apply(solver, model, filesystem, Tech.ansible, result) + + +class TestPatchSolverChefScript1(TestPatchSolver): + def setUp(self): + super().setUp() + chef_script_1 = """ + file '/tmp/something' do + mode '0755' + action :create_if_missing + end + """ + self._setup_patch_solver(chef_script_1, UnitBlockType.script, Tech.chef) + + def test_patch_solver_chef_mode(self) -> None: + filesystem = SystemState() + filesystem.state["/tmp/something"] = get_default_file_state() + filesystem.state["/tmp/something"].attrs["mode"] = "0777" + filesystem.state["/tmp/something"].attrs["owner"] = UNDEF + filesystem.state["/tmp/something"].attrs["content"] = UNDEF + + assert self.statement is not None + solver = PatchSolver(self.statement, filesystem) + models = solver.solve() + assert models is not None + assert len(models) == 1 + model = models[0] + assert model[str(solver.sum_var)] == 4 + assert model["mode_2808"] == "0777" + assert model["state_7904"] == "present" + assert model["content_16"] == UNDEF + assert model["owner_256"] == UNDEF + + result = """ + file '/tmp/something' do + mode '0777' + action :create_if_missing + end + """ + self._patch_solver_apply(solver, model, filesystem, Tech.chef, result) + + def test_patch_solver_chef_delete(self) -> None: + filesystem = SystemState() + filesystem.state["/tmp/something"] = get_nil_file_state() + + assert self.statement is not None + solver = PatchSolver(self.statement, filesystem) + models = solver.solve() + assert models is not None + assert len(models) == 1 + model = models[0] + assert model[str(solver.sum_var)] == 3 + assert model["mode_2808"] == UNDEF + assert model["state_7904"] == "absent" + assert model["content_16"] == UNDEF + assert model["owner_256"] == UNDEF + + result = """ + file '/tmp/something' do + action :delete + end + """ + self._patch_solver_apply(solver, model, filesystem, Tech.chef, result) + + @unittest.skip("Not implemented yet") + def test_patch_solver_chef_modify_to_directory(self) -> None: + filesystem = SystemState() + filesystem.state["/tmp/something"] = get_nil_file_state() + filesystem.state["/tmp/something"].attrs["state"] = "directory" + filesystem.state["/tmp/something"].attrs["mode"] = "0777" + + assert self.statement is not None + solver = PatchSolver(self.statement, filesystem) + models = solver.solve() + assert models is not None + assert len(models) == 1 + model = models[0] + assert model[str(solver.sum_var)] == 2 + assert model[str(solver.unchanged[0])] == 0 + assert model[str(solver.unchanged[1])] == 0 + assert model[str(solver.unchanged[2])] == 1 + assert model[str(solver.unchanged[3])] == 1 + assert model["mode-0"] == "0777" + assert model["sketched-state-1"] == "directory" + assert model["sketched-content-2"] == UNDEF + assert model["sketched-owner-3"] == UNDEF + + result = """ + directory '/tmp/something' do + mode '0777' + action :create + end + """ + self._patch_solver_apply(solver, model, filesystem, Tech.chef, result) + + +class TestPatchSolverChefScript2(TestPatchSolver): + def setUp(self): + super().setUp() + chef_script_2 = """ + directory '/tmp/something' do + action :delete + end + """ + self._setup_patch_solver(chef_script_2, UnitBlockType.script, Tech.chef) + + def test_patch_solver_chef_directory(self) -> None: + filesystem = SystemState() + filesystem.state["/tmp/something"] = get_nil_file_state() + filesystem.state["/tmp/something"].attrs["state"] = "present" + filesystem.state["/tmp/something"].attrs["mode"] = "0777" + + assert self.statement is not None + solver = PatchSolver(self.statement, filesystem) + models = solver.solve() + assert models is not None + assert len(models) == 1 + model = models[0] + assert model[str(solver.sum_var)] == 3 + assert model["state_3159"] == "present" + assert model["content_16"] == UNDEF + assert model["owner_256"] == UNDEF + assert model["mode_1296"] == "0777" + + result = """ + directory '/tmp/something' do + action :create + mode '0777' + end + """ + self._patch_solver_apply(solver, model, filesystem, Tech.chef, result) + + +class TestPatchSolverChefScript3(TestPatchSolver): + def setUp(self): + super().setUp() + chef_script_2 = """ + user 'test' do + name 'test' + action :create + end + """ + self._setup_patch_solver(chef_script_2, UnitBlockType.script, Tech.chef) + + def test_patch_solver_chef_user_delete(self) -> None: + filesystem = SystemState() + filesystem.state["user:test"] = State() + filesystem.state["user:test"].attrs["state"] = "absent" + + assert self.statement is not None + solver = PatchSolver(self.statement, filesystem) + models = solver.solve() + assert models is not None + assert len(models) == 1 + + result = """ + user 'test' do + name 'test' + action :delete + end + """ + self._patch_solver_apply(solver, models[0], filesystem, Tech.chef, result) + + def test_patch_solver_chef_user_change(self) -> None: + filesystem = SystemState() + filesystem.state["user:test2"] = State() + filesystem.state["user:test2"].attrs["state"] = "present" + + assert self.statement is not None + solver = PatchSolver(self.statement, filesystem) + models = solver.solve() + assert models is not None + assert len(models) == 1 + + result = """ + user 'test' do + name 'test2' + action :create + end + """ + self._patch_solver_apply(solver, models[0], filesystem, Tech.chef, result) + + +class TestPatchSolverChefScript4(TestPatchSolver): + def setUp(self): + super().setUp() + chef_script_1 = """ + y = '0755' + x = y + + file '/tmp/something' do + mode x + action :nothing + end + """ + self._setup_patch_solver(chef_script_1, UnitBlockType.script, Tech.chef) + + def test_patch_solver_chef_variable_mode(self): + filesystem = SystemState() + filesystem.state["/tmp/something"] = get_nil_file_state() + filesystem.state["/tmp/something"].attrs["state"] = "present" + filesystem.state["/tmp/something"].attrs["mode"] = "0777" + + assert self.statement is not None + solver = PatchSolver(self.statement, filesystem) + models = solver.solve() + assert models is not None + assert len(models) == 1 + model = models[0] + assert model[str(solver.sum_var)] == 4 + assert model[str(solver.vars["x"])] == "0777" + assert model[str(solver.vars["y"])] == "0777" + assert model["mode_8892"] == "0777" + assert model["state_17836"] == "present" + assert model["content_16"] == UNDEF + assert model["owner_256"] == UNDEF + + result = """ + y = '0777' + x = y + + file '/tmp/something' do + mode x + action :nothing + end + """ + self._patch_solver_apply(solver, model, filesystem, Tech.chef, result) + + +class TestPatchSolverChefScript5(TestPatchSolver): + def setUp(self): + super().setUp() + chef_script_2 = """ +file '/var/www/customers/public_html/index.php' do + content 'This is a placeholder for the home page.' + mode '0755' + owner 'web_admin' + group 'web_admin' +end + +file '/var/www/customers/public_html/index2.php' do + content 'This is a placeholder for the home page.' + mode '0755' + owner 'web_admin' + group 'web_admin' +end + """ + self._setup_patch_solver(chef_script_2, UnitBlockType.script, Tech.chef) + + def test_patch_solver_chef_minimize(self) -> None: + filesystem = SystemState() + filesystem.state["/var/www/customers/public_html/index.php"] = ( + get_default_file_state() + ) + filesystem.state["/var/www/customers/public_html/index.php"].attrs[ + "mode" + ] = "0755" + filesystem.state["/var/www/customers/public_html/index.php"].attrs[ + "owner" + ] = "test" + filesystem.state["/var/www/customers/public_html/index.php"].attrs[ + "content" + ] = "This is a placeholder for the home page." + + filesystem.state["/var/www/customers/public_html/index2.php"] = State() + filesystem.state["/var/www/customers/public_html/index2.php"].attrs[ + "state" + ] = UNDEF + filesystem.state["/var/www/customers/public_html/index2.php"].attrs[ + "mode" + ] = "0755" + filesystem.state["/var/www/customers/public_html/index2.php"].attrs[ + "owner" + ] = "web_admin" + filesystem.state["/var/www/customers/public_html/index2.php"].attrs[ + "content" + ] = "This is a placeholder for the home page." + + assert self.statement is not None + self.statement: PStatement = PStatement.minimize( + self.statement, ["/var/www/customers/public_html/index.php"] + ) + minimized_filesystem = SystemState() + minimized_filesystem.state["/var/www/customers/public_html/index.php"] = ( + get_default_file_state() + ) + minimized_filesystem.state["/var/www/customers/public_html/index.php"].attrs[ + "mode" + ] = "0755" + minimized_filesystem.state["/var/www/customers/public_html/index.php"].attrs[ + "owner" + ] = "test" + minimized_filesystem.state["/var/www/customers/public_html/index.php"].attrs[ + "content" + ] = "This is a placeholder for the home page." + + solver = PatchSolver(self.statement, minimized_filesystem) + models = solver.solve() + assert models is not None + assert len(models) == 1 + + result = """ +file '/var/www/customers/public_html/index.php' do + content 'This is a placeholder for the home page.' + mode '0755' + owner 'test' + group 'web_admin' + action :create +end + +file '/var/www/customers/public_html/index2.php' do + content 'This is a placeholder for the home page.' + mode '0755' + owner 'web_admin' + group 'web_admin' +end + """ + self._patch_solver_apply(solver, models[0], filesystem, Tech.chef, result) + + +class TestPatchSolverChefScript6(TestPatchSolver): + def setUp(self): + super().setUp() + chef_script_6 = """ +test_user = 'leia' +test_user_home = "/home/#{test_user}" + +directory "#{test_user_home}/.ssh" do + mode '0700' + owner test_user + group test_user +end +""" + self._setup_patch_solver(chef_script_6, UnitBlockType.script, Tech.chef) + + def test_patch_solver_chef_vars(self) -> None: + filesystem = SystemState() + filesystem.state["/home/leia/.ssh"] = get_default_file_state() + filesystem.state["/home/leia/.ssh"].attrs["mode"] = "0766" + filesystem.state["/home/leia/.ssh"].attrs["owner"] = "leia" + filesystem.state["/home/leia/.ssh"].attrs["content"] = "leia" + assert self.statement is not None + + solver = PatchSolver(self.statement, filesystem) + models = solver.solve() + assert models is not None + assert len(models) == 1 + + result = """ +test_user = 'leia' +test_user_home = "/home/#{test_user}" + +directory "#{test_user_home}/.ssh" do + mode '0766' + owner test_user + group test_user + action :create + content 'leia' +end +""" + self._patch_solver_apply(solver, models[0], filesystem, Tech.chef, result) + + +class TestPatchSolverChefScript7(TestPatchSolver): + def setUp(self): + super().setUp() + chef_script_7 = """ +package 'tar' do + action :install +end +""" + self._setup_patch_solver(chef_script_7, UnitBlockType.script, Tech.chef) + + def test_patch_solver_chef_remove_package(self) -> None: + filesystem = SystemState() + filesystem.state["package:tar"] = State() + filesystem.state["package:tar"].attrs["state"] = "absent" + assert self.statement is not None + + solver = PatchSolver(self.statement, filesystem) + models = solver.solve() + assert models is not None + assert len(models) == 1 + + result = """ +package 'tar' do + action :remove +end +""" + self._patch_solver_apply(solver, models[0], filesystem, Tech.chef, result) + + def test_patch_solver_chef_latest_package(self) -> None: + filesystem = SystemState() + filesystem.state["package:tar"] = State() + filesystem.state["package:tar"].attrs["state"] = "latest" + assert self.statement is not None + + solver = PatchSolver(self.statement, filesystem) + models = solver.solve() + assert models is not None + assert len(models) == 1 + + result = """ +package 'tar' do + action :upgrade +end +""" + self._patch_solver_apply(solver, models[0], filesystem, Tech.chef, result) + + +class TestPatchSolverChefScript8(TestPatchSolver): + def setUp(self): + super().setUp() + chef_script_8 = """ +package 'tar' do + action :purge +end +""" + self._setup_patch_solver(chef_script_8, UnitBlockType.script, Tech.chef) + + def test_patch_solver_chef_create_package(self) -> None: + filesystem = SystemState() + filesystem.state["package:tar"] = State() + filesystem.state["package:tar"].attrs["state"] = "present" + assert self.statement is not None + + solver = PatchSolver(self.statement, filesystem) + models = solver.solve() + assert models is not None + assert len(models) == 1 + + result = """ +package 'tar' do + action :install +end +""" + self._patch_solver_apply(solver, models[0], filesystem, Tech.chef, result) + + def test_patch_solver_chef_reconfig_package(self) -> None: + filesystem = SystemState() + filesystem.state["package:tar"] = State() + filesystem.state["package:tar"].attrs["state"] = "reconfig" + assert self.statement is not None + + solver = PatchSolver(self.statement, filesystem) + models = solver.solve() + assert models is not None + assert len(models) == 1 + + result = """ +package 'tar' do + action :reconfig +end +""" + self._patch_solver_apply(solver, models[0], filesystem, Tech.chef, result) + + def test_patch_solver_chef_nothing_package(self) -> None: + filesystem = SystemState() + filesystem.state["package:tar"] = State() + filesystem.state["package:tar"].attrs["state"] = "nothing" + assert self.statement is not None + + solver = PatchSolver(self.statement, filesystem) + models = solver.solve() + assert models is not None + assert len(models) == 1 + + result = """ +package 'tar' do + action :nothing +end +""" + self._patch_solver_apply(solver, models[0], filesystem, Tech.chef, result) + + +class TestPatchSolverChefScript9(TestPatchSolver): + def setUp(self): + super().setUp() + chef_script_9 = """ +service 'example_service' do + action :start +end +""" + self._setup_patch_solver(chef_script_9, UnitBlockType.script, Tech.chef) + + def test_patch_solver_chef_stop_service(self) -> None: + filesystem = SystemState() + filesystem.state["service:example_service"] = State() + filesystem.state["service:example_service"].attrs["state"] = "stop" + filesystem.state["service:example_service"].attrs["enabled"] = UNDEF + assert self.statement is not None + + solver = PatchSolver(self.statement, filesystem) + models = solver.solve() + assert models is not None + assert len(models) == 1 + + result = """ +service 'example_service' do + action :stop +end +""" + self._patch_solver_apply(solver, models[0], filesystem, Tech.chef, result) + + def test_patch_solver_chef_enable_service(self) -> None: + filesystem = SystemState() + filesystem.state["service:example_service"] = State() + filesystem.state["service:example_service"].attrs["state"] = UNDEF + filesystem.state["service:example_service"].attrs["enabled"] = "true" + assert self.statement is not None + + solver = PatchSolver(self.statement, filesystem) + models = solver.solve() + assert models is not None + assert len(models) == 1 + + result = """ +service 'example_service' do + action :enable +end +""" + self._patch_solver_apply(solver, models[0], filesystem, Tech.chef, result) + + +class TestPatchSolverChefScript10(TestPatchSolver): + def setUp(self): + super().setUp() + chef_script_10 = """ +app_dir = '/var/www/customers/public_html' +file "#{app_dir}/index.html" do + owner lazy { default_apache_user } + group lazy { default_apache_group } +end +""" + self._setup_patch_solver(chef_script_10, UnitBlockType.script, Tech.chef) + + def test_patch_solver_chef_non_string(self) -> None: + filesystem = SystemState() + filesystem.state["/var/www/customers/public_html/index.html"] = ( + get_default_file_state() + ) + filesystem.state["/var/www/customers/public_html/index.html"].attrs[ + "state" + ] = UNDEF + filesystem.state["/var/www/customers/public_html/index.html"].attrs[ + "owner" + ] = "test" + + assert self.statement is not None + + solver = PatchSolver(self.statement, filesystem) + models = solver.solve() + assert models is not None + + assert len(models) == 3 + + result = """ +app_dir = '/var/www/customers/public_html' +file "#{app_dir}/index.html" do + owner 'test' + group lazy { default_apache_group } +end +""" + self._patch_solver_apply(solver, models[0], filesystem, Tech.chef, result) + + +class TestPatchSolverChefScript11(TestPatchSolver): + def setUp(self): + super().setUp() + chef_script_2 = """ + link '/tmp/file' do + to '/etc/file' + action :delete + end + """ + self._setup_patch_solver(chef_script_2, UnitBlockType.script, Tech.chef) + + def test_patch_solver_chef_link(self) -> None: + filesystem = SystemState() + filesystem.state["/tmp/file"] = get_nil_file_state() + filesystem.state["/tmp/file"].attrs["state"] = "present" + + assert self.statement is not None + solver = PatchSolver(self.statement, filesystem) + models = solver.solve() + assert models is not None + assert len(models) == 1 + model = models[0] + + result = """ + link '/tmp/file' do + to '/etc/file' + action :create + end + """ + self._patch_solver_apply(solver, model, filesystem, Tech.chef, result) + + +class TestPatchSolverTerraformScript1(TestPatchSolver): + def setUp(self): + super().setUp() + terraform_script_1 = """ +resource "aws_iam_role" "test_role" { + name = "test_role" + assume_role_policy = < None: + filesystem = SystemState() + filesystem.state["aws_iam_role:test_role"] = State() + filesystem.state["aws_iam_role:test_role"].attrs["state"] = "present" + filesystem.state["aws_iam_role:test_role"].attrs[ + "assume_role_policy" + ] = """{ + "Version": "2012-10-17", + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Deny", + "Sid": "", + "Principal": { + "Service": "ec2.amazonaws.com" + } + } + ] +}""" + + assert self.statement is not None + solver = PatchSolver(self.statement, filesystem) + models = solver.solve() + assert models is not None + assert len(models) == 1 + + model = models[0] + result = """ +resource "aws_iam_role" "test_role" { + name = "test_role" + assume_role_policy = < None: + filesystem = SystemState() + filesystem.state["aws_instance:web"] = State() + filesystem.state["aws_instance:web"].attrs["state"] = "present" + filesystem.state["aws_instance:web"].attrs["instance_type"] = "t2.micro" + filesystem.state["aws_instance:web"].attrs["availability_zone"] = "us-west-2a" + + assert self.statement is not None + solver = PatchSolver(self.statement, filesystem) + models = solver.solve() + assert models is not None + assert len(models) == 1 + + model = models[0] + result = """ +resource "aws_instance" "web" { + instance_type = "t2.micro" + availability_zone = "us-west-2a" +} +""" + self._patch_solver_apply(solver, model, filesystem, Tech.terraform, result) + + +class TestPatchSolverTerraformScript3(TestPatchSolver): + def setUp(self): + super().setUp() + terraform_script_3 = """ +resource "aws_s3_bucket" "example" { + bucket = "my-tf-test-bucket" +} +""" + self._setup_patch_solver( + terraform_script_3, UnitBlockType.script, Tech.terraform + ) + + def test_patch_solver_terraform_aws_bucket(self): + filesystem = SystemState() + filesystem.state["aws_s3_bucket:different-test-bucket"] = State() + filesystem.state["aws_s3_bucket:different-test-bucket"].attrs[ + "state" + ] = "present" + filesystem.state["aws_s3_bucket:different-test-bucket"].attrs[ + "acl" + ] = "public-read" + + assert self.statement is not None + solver = PatchSolver(self.statement, filesystem) + models = solver.solve() + assert models is not None + assert len(models) == 1 + + model = models[0] + result = """ +resource "aws_s3_bucket" "example" { + bucket = "different-test-bucket" + acl = "public-read" +} +""" + self._patch_solver_apply(solver, model, filesystem, Tech.terraform, result) + + +class TestPatchSolverTerraformScript4(TestPatchSolver): + def setUp(self): + super().setUp() + terraform_script_4 = """ +resource "aws_instance" "stdby_vThunder" { + instance_type = "m4.xlarge" + availability_zone = "${var.region}a" +} +""" + self._setup_patch_solver( + terraform_script_4, UnitBlockType.script, Tech.terraform + ) + + def test_patch_solver_terraform_unsupported(self): + filesystem = SystemState() + filesystem.state["aws_instance:stdby_vThunder"] = State() + filesystem.state["aws_instance:stdby_vThunder"].attrs["state"] = "present" + filesystem.state["aws_instance:stdby_vThunder"].attrs[ + "instance_type" + ] = "m3.xlarge" + filesystem.state["aws_instance:stdby_vThunder"].attrs[ + "availability_zone" + ] = "us-west-2a" + + assert self.statement is not None + solver = PatchSolver(self.statement, filesystem) + models = solver.solve() + assert models is not None + assert len(models) == 1 + + model = models[0] + result = """ +resource "aws_instance" "stdby_vThunder" { + instance_type = "m3.xlarge" + availability_zone = "us-west-2a" +} +""" + self._patch_solver_apply(solver, model, filesystem, Tech.terraform, result) diff --git a/glitch/tests/repair/interactive/test_tracer_model.py b/tests/repair/interactive/test_tracer_model.py similarity index 100% rename from glitch/tests/repair/interactive/test_tracer_model.py rename to tests/repair/interactive/test_tracer_model.py diff --git a/glitch/tests/repair/interactive/test_tracer_parser.py b/tests/repair/interactive/test_tracer_parser.py similarity index 100% rename from glitch/tests/repair/interactive/test_tracer_parser.py rename to tests/repair/interactive/test_tracer_parser.py diff --git a/glitch/tests/repair/interactive/test_tracer_transform.py b/tests/repair/interactive/test_tracer_transform.py similarity index 60% rename from glitch/tests/repair/interactive/test_tracer_transform.py rename to tests/repair/interactive/test_tracer_transform.py index da4b4f58..8cc01a2e 100644 --- a/glitch/tests/repair/interactive/test_tracer_transform.py +++ b/tests/repair/interactive/test_tracer_transform.py @@ -1,3 +1,4 @@ +import getpass import os import pytest import shutil @@ -8,11 +9,12 @@ get_file_system_state, ) from glitch.repair.interactive.tracer.model import * -from glitch.repair.interactive.filesystem import * +from glitch.repair.interactive.system import * +from glitch.repair.interactive.values import UNDEF def test_get_affected_paths() -> None: - sys_calls = [ + sys_calls: List[Syscall] = [ SOpen("open", ["file1", [OpenFlag.O_WRONLY]], 0), SOpenAt("openat", ["0", "file2", [OpenFlag.O_WRONLY]], 0), SRename("rename", ["file3", "file8"], 0), @@ -60,18 +62,37 @@ def setup_file_system(): @pytest.fixture def teardown_file_system(): yield + assert temp_dir is not None shutil.rmtree(temp_dir.name) -def test_get_file_system_state(setup_file_system, teardown_file_system) -> None: +def test_get_file_system_state(setup_file_system, teardown_file_system) -> None: # type: ignore file4 = os.path.join(dir1, "file4") files = {dir1, file2, file3, file4} fs_state = get_file_system_state(files) - assert fs_state.state == { - dir1: Dir("775", os.getlogin()), - file2: File("664", os.getlogin(), ""), - file3: File("664", os.getlogin(), ""), - file4: Nil(), - } + assert len(fs_state.state) == 4 + + assert dir1 in fs_state.state + assert fs_state.state[dir1].attrs["state"] == "directory" + assert fs_state.state[dir1].attrs["mode"] == "0775" + assert fs_state.state[dir1].attrs["owner"] == getpass.getuser() + + assert file2 in fs_state.state + assert fs_state.state[file2].attrs["state"] == "present" + assert fs_state.state[file2].attrs["mode"] == "0664" + assert fs_state.state[file2].attrs["owner"] == getpass.getuser() + assert fs_state.state[file2].attrs["content"] == "" + + assert file3 in fs_state.state + assert fs_state.state[file3].attrs["state"] == "present" + assert fs_state.state[file3].attrs["mode"] == "0664" + assert fs_state.state[file3].attrs["owner"] == getpass.getuser() + assert fs_state.state[file3].attrs["content"] == "" + + assert file4 in fs_state.state + assert fs_state.state[file4].attrs["state"] == "absent" + assert fs_state.state[file4].attrs["mode"] == UNDEF + assert fs_state.state[file4].attrs["owner"] == UNDEF + assert fs_state.state[file4].attrs["content"] == UNDEF diff --git a/glitch/tests/security/terraform/__init__.py b/tests/security/__init__.py similarity index 100% rename from glitch/tests/security/terraform/__init__.py rename to tests/security/__init__.py diff --git a/tests/security/ansible/__init__.py b/tests/security/ansible/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/glitch/tests/security/ansible/files/admin.yml b/tests/security/ansible/files/admin.yml similarity index 100% rename from glitch/tests/security/ansible/files/admin.yml rename to tests/security/ansible/files/admin.yml diff --git a/glitch/tests/security/ansible/files/empty.yml b/tests/security/ansible/files/empty.yml similarity index 93% rename from glitch/tests/security/ansible/files/empty.yml rename to tests/security/ansible/files/empty.yml index a92bf919..40d3fbcc 100644 --- a/glitch/tests/security/ansible/files/empty.yml +++ b/tests/security/ansible/files/empty.yml @@ -4,7 +4,7 @@ url: "https://{{ hostvars[inventory_hostname|regex_replace('ansible-1', 'checkpoint_mgmt')]['private_ip'] }}/web_api/login" method: POST body: - user: {{ admin }} + user: "{{ admin }}" password: body_format: json validate_certs: false diff --git a/glitch/tests/security/ansible/files/full_permission.yml b/tests/security/ansible/files/full_permission.yml similarity index 100% rename from glitch/tests/security/ansible/files/full_permission.yml rename to tests/security/ansible/files/full_permission.yml diff --git a/glitch/tests/security/ansible/files/hard_secr.yml b/tests/security/ansible/files/hard_secr.yml similarity index 93% rename from glitch/tests/security/ansible/files/hard_secr.yml rename to tests/security/ansible/files/hard_secr.yml index 3a190ccf..99a239b8 100644 --- a/glitch/tests/security/ansible/files/hard_secr.yml +++ b/tests/security/ansible/files/hard_secr.yml @@ -5,7 +5,7 @@ method: POST body: user: test - password: 123 + password: "abc123" body_format: json validate_certs: false register: login_data diff --git a/glitch/tests/security/ansible/files/http.yml b/tests/security/ansible/files/http.yml similarity index 100% rename from glitch/tests/security/ansible/files/http.yml rename to tests/security/ansible/files/http.yml diff --git a/glitch/tests/security/ansible/files/int_check.yml b/tests/security/ansible/files/int_check.yml similarity index 100% rename from glitch/tests/security/ansible/files/int_check.yml rename to tests/security/ansible/files/int_check.yml diff --git a/glitch/tests/security/ansible/files/inv_bind.yml b/tests/security/ansible/files/inv_bind.yml similarity index 100% rename from glitch/tests/security/ansible/files/inv_bind.yml rename to tests/security/ansible/files/inv_bind.yml diff --git a/glitch/tests/security/ansible/files/obs_command.yml b/tests/security/ansible/files/obs_command.yml similarity index 100% rename from glitch/tests/security/ansible/files/obs_command.yml rename to tests/security/ansible/files/obs_command.yml diff --git a/glitch/tests/security/ansible/files/susp.yml b/tests/security/ansible/files/susp.yml similarity index 100% rename from glitch/tests/security/ansible/files/susp.yml rename to tests/security/ansible/files/susp.yml diff --git a/glitch/tests/security/ansible/files/weak_crypt.yml b/tests/security/ansible/files/weak_crypt.yml similarity index 100% rename from glitch/tests/security/ansible/files/weak_crypt.yml rename to tests/security/ansible/files/weak_crypt.yml diff --git a/glitch/tests/security/ansible/test_security.py b/tests/security/ansible/test_security.py similarity index 58% rename from glitch/tests/security/ansible/test_security.py rename to tests/security/ansible/test_security.py index 7778f3ad..35756bed 100644 --- a/glitch/tests/security/ansible/test_security.py +++ b/tests/security/ansible/test_security.py @@ -1,46 +1,31 @@ -import unittest - -from glitch.analysis.security import SecurityVisitor -from glitch.parsers.ansible import AnsibleParser +from tests.security.security_helper import BaseSecurityTest from glitch.tech import Tech -class TestSecurity(unittest.TestCase): - def __help_test(self, path, type, n_errors: int, codes, lines) -> None: - parser = AnsibleParser() - inter = parser.parse(path, type, False) - analysis = SecurityVisitor(Tech.ansible) - analysis.config("configs/default.ini") - errors = list( - filter(lambda e: e.code.startswith("sec_"), set(analysis.check(inter))) - ) - errors = sorted(errors, key=lambda e: (e.path, e.line, e.code)) - self.assertEqual(len(errors), n_errors) - for i in range(n_errors): - self.assertEqual(errors[i].code, codes[i]) - self.assertEqual(errors[i].line, lines[i]) +class TestSecurity(BaseSecurityTest): + TECH = Tech.ansible def test_ansible_http(self) -> None: - self.__help_test( + self._help_test( "tests/security/ansible/files/http.yml", "tasks", 1, ["sec_https"], [4] ) def test_ansible_susp_comment(self) -> None: - self.__help_test( + self._help_test( "tests/security/ansible/files/susp.yml", "vars", 1, ["sec_susp_comm"], [9] ) def test_ansible_def_admin(self) -> None: - self.__help_test( + self._help_test( "tests/security/ansible/files/admin.yml", "tasks", - 3, - ["sec_def_admin", "sec_hard_secr", "sec_hard_user"], - [3, 3, 3], + 2, + ["sec_def_admin", "sec_hard_user"], + [3, 3], ) def test_ansible_empt_pass(self) -> None: - self.__help_test( + self._help_test( "tests/security/ansible/files/empty.yml", "tasks", 1, @@ -49,7 +34,7 @@ def test_ansible_empt_pass(self) -> None: ) def test_ansible_weak_crypt(self) -> None: - self.__help_test( + self._help_test( "tests/security/ansible/files/weak_crypt.yml", "tasks", 2, @@ -58,16 +43,16 @@ def test_ansible_weak_crypt(self) -> None: ) def test_ansible_hard_secr(self) -> None: - self.__help_test( + self._help_test( "tests/security/ansible/files/hard_secr.yml", "tasks", - 4, - ["sec_hard_secr", "sec_hard_user", "sec_hard_pass", "sec_hard_secr"], - [7, 7, 8, 8], + 2, + ["sec_hard_user", "sec_hard_pass"], + [7, 8], ) def test_ansible_invalid_bind(self) -> None: - self.__help_test( + self._help_test( "tests/security/ansible/files/inv_bind.yml", "tasks", 1, @@ -76,7 +61,7 @@ def test_ansible_invalid_bind(self) -> None: ) def test_ansible_int_check(self) -> None: - self.__help_test( + self._help_test( "tests/security/ansible/files/int_check.yml", "tasks", 1, @@ -85,7 +70,7 @@ def test_ansible_int_check(self) -> None: ) def test_ansible_full_perm(self) -> None: - self.__help_test( + self._help_test( "tests/security/ansible/files/full_permission.yml", "tasks", 1, @@ -94,7 +79,7 @@ def test_ansible_full_perm(self) -> None: ) def test_ansible_obs_command(self) -> None: - self.__help_test( + self._help_test( "tests/security/ansible/files/obs_command.yml", "tasks", 1, diff --git a/tests/security/chef/__init__.py b/tests/security/chef/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/glitch/tests/security/chef/files/admin.rb b/tests/security/chef/files/admin.rb similarity index 100% rename from glitch/tests/security/chef/files/admin.rb rename to tests/security/chef/files/admin.rb diff --git a/glitch/tests/security/chef/files/empty.rb b/tests/security/chef/files/empty.rb similarity index 100% rename from glitch/tests/security/chef/files/empty.rb rename to tests/security/chef/files/empty.rb diff --git a/glitch/tests/security/chef/files/full_permission.rb b/tests/security/chef/files/full_permission.rb similarity index 100% rename from glitch/tests/security/chef/files/full_permission.rb rename to tests/security/chef/files/full_permission.rb diff --git a/glitch/tests/security/chef/files/hard_secr.rb b/tests/security/chef/files/hard_secr.rb similarity index 100% rename from glitch/tests/security/chef/files/hard_secr.rb rename to tests/security/chef/files/hard_secr.rb diff --git a/glitch/tests/security/chef/files/http.rb b/tests/security/chef/files/http.rb similarity index 100% rename from glitch/tests/security/chef/files/http.rb rename to tests/security/chef/files/http.rb diff --git a/glitch/tests/security/chef/files/int_check.rb b/tests/security/chef/files/int_check.rb similarity index 100% rename from glitch/tests/security/chef/files/int_check.rb rename to tests/security/chef/files/int_check.rb diff --git a/glitch/tests/security/chef/files/inv_bind.rb b/tests/security/chef/files/inv_bind.rb similarity index 100% rename from glitch/tests/security/chef/files/inv_bind.rb rename to tests/security/chef/files/inv_bind.rb diff --git a/glitch/tests/security/chef/files/missing_default.rb b/tests/security/chef/files/missing_default.rb similarity index 100% rename from glitch/tests/security/chef/files/missing_default.rb rename to tests/security/chef/files/missing_default.rb diff --git a/glitch/tests/security/chef/files/obs_command.rb b/tests/security/chef/files/obs_command.rb similarity index 100% rename from glitch/tests/security/chef/files/obs_command.rb rename to tests/security/chef/files/obs_command.rb diff --git a/glitch/tests/security/chef/files/susp.rb b/tests/security/chef/files/susp.rb similarity index 100% rename from glitch/tests/security/chef/files/susp.rb rename to tests/security/chef/files/susp.rb diff --git a/glitch/tests/security/chef/files/weak_crypt.rb b/tests/security/chef/files/weak_crypt.rb similarity index 100% rename from glitch/tests/security/chef/files/weak_crypt.rb rename to tests/security/chef/files/weak_crypt.rb diff --git a/tests/security/chef/test_security.py b/tests/security/chef/test_security.py new file mode 100644 index 00000000..9bae0720 --- /dev/null +++ b/tests/security/chef/test_security.py @@ -0,0 +1,95 @@ +from tests.security.security_helper import BaseSecurityTest +from glitch.parsers.chef import ChefParser +from glitch.tech import Tech + + +class TestSecurity(BaseSecurityTest): + PARSER_CLASS = ChefParser + TECH = Tech.chef + + def test_chef_http(self) -> None: + self._help_test( + "tests/security/chef/files/http.rb", "script", 1, ["sec_https"], [3] + ) + + def test_chef_susp_comment(self) -> None: + self._help_test( + "tests/security/chef/files/susp.rb", "script", 1, ["sec_susp_comm"], [1] + ) + + def test_chef_def_admin(self) -> None: + self._help_test( + "tests/security/chef/files/admin.rb", + "script", + 2, + ["sec_def_admin", "sec_hard_user"], + [8, 8], + ) + + def test_chef_empt_pass(self) -> None: + self._help_test( + "tests/security/chef/files/empty.rb", "script", 1, ["sec_empty_pass"], [1] + ) + + def test_chef_weak_crypt(self) -> None: + self._help_test( + "tests/security/chef/files/weak_crypt.rb", + "script", + 1, + ["sec_weak_crypt"], + [4], + ) + + def test_chef_hard_secr(self) -> None: + self._help_test( + "tests/security/chef/files/hard_secr.rb", + "script", + 1, + ["sec_hard_pass"], + [8], + ) + + def test_chef_invalid_bind(self) -> None: + self._help_test( + "tests/security/chef/files/inv_bind.rb", + "script", + 1, + ["sec_invalid_bind"], + [7], + ) + + def test_chef_int_check(self) -> None: + self._help_test( + "tests/security/chef/files/int_check.rb", + "script", + 1, + ["sec_no_int_check"], + [1], + ) + + def test_chef_missing_default(self) -> None: + self._help_test( + "tests/security/chef/files/missing_default.rb", + "script", + 1, + ["sec_no_default_switch"], + [1], + ) + + def test_chef_full_permission(self) -> None: + self._help_test( + "tests/security/chef/files/full_permission.rb", + "script", + 1, + ["sec_full_permission_filesystem"], + [3], + ) + + def test_chef_obs_command(self) -> None: + self._help_test( + "tests/security/chef/files/obs_command.rb", + "script", + 1, + ["sec_obsolete_command"], + [2], + ) diff --git a/tests/security/puppet/__init__.py b/tests/security/puppet/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/glitch/tests/security/puppet/files/admin.pp b/tests/security/puppet/files/admin.pp similarity index 100% rename from glitch/tests/security/puppet/files/admin.pp rename to tests/security/puppet/files/admin.pp diff --git a/glitch/tests/security/puppet/files/empty.pp b/tests/security/puppet/files/empty.pp similarity index 100% rename from glitch/tests/security/puppet/files/empty.pp rename to tests/security/puppet/files/empty.pp diff --git a/glitch/tests/security/puppet/files/full_permission.pp b/tests/security/puppet/files/full_permission.pp similarity index 100% rename from glitch/tests/security/puppet/files/full_permission.pp rename to tests/security/puppet/files/full_permission.pp diff --git a/glitch/tests/security/puppet/files/hard_secr.pp b/tests/security/puppet/files/hard_secr.pp similarity index 100% rename from glitch/tests/security/puppet/files/hard_secr.pp rename to tests/security/puppet/files/hard_secr.pp diff --git a/glitch/tests/security/puppet/files/http.pp b/tests/security/puppet/files/http.pp similarity index 100% rename from glitch/tests/security/puppet/files/http.pp rename to tests/security/puppet/files/http.pp diff --git a/glitch/tests/security/puppet/files/int_check.pp b/tests/security/puppet/files/int_check.pp similarity index 100% rename from glitch/tests/security/puppet/files/int_check.pp rename to tests/security/puppet/files/int_check.pp diff --git a/glitch/tests/security/puppet/files/inv_bind.pp b/tests/security/puppet/files/inv_bind.pp similarity index 97% rename from glitch/tests/security/puppet/files/inv_bind.pp rename to tests/security/puppet/files/inv_bind.pp index f8633f5a..22b7abba 100644 --- a/glitch/tests/security/puppet/files/inv_bind.pp +++ b/tests/security/puppet/files/inv_bind.pp @@ -4,11 +4,11 @@ state => ['NEW'], action => 'accept', source => $::openstack::config::network_data, - } -> + } firewall { '9999 - Reject remaining traffic': proto => 'all', action => 'reject', reject => 'icmp-host-prohibited', source => '0.0.0.0/0', } -} \ No newline at end of file +} diff --git a/glitch/tests/security/puppet/files/missing_default.pp b/tests/security/puppet/files/missing_default.pp similarity index 100% rename from glitch/tests/security/puppet/files/missing_default.pp rename to tests/security/puppet/files/missing_default.pp diff --git a/glitch/tests/security/puppet/files/obs_command.pp b/tests/security/puppet/files/obs_command.pp similarity index 100% rename from glitch/tests/security/puppet/files/obs_command.pp rename to tests/security/puppet/files/obs_command.pp diff --git a/glitch/tests/security/puppet/files/susp.pp b/tests/security/puppet/files/susp.pp similarity index 100% rename from glitch/tests/security/puppet/files/susp.pp rename to tests/security/puppet/files/susp.pp diff --git a/glitch/tests/security/puppet/files/weak_crypt.pp b/tests/security/puppet/files/weak_crypt.pp similarity index 100% rename from glitch/tests/security/puppet/files/weak_crypt.pp rename to tests/security/puppet/files/weak_crypt.pp diff --git a/tests/security/puppet/test_security.py b/tests/security/puppet/test_security.py new file mode 100644 index 00000000..26345311 --- /dev/null +++ b/tests/security/puppet/test_security.py @@ -0,0 +1,93 @@ +from tests.security.security_helper import BaseSecurityTest +from glitch.tech import Tech + + +class TestSecurity(BaseSecurityTest): + TECH = Tech.puppet + + def test_puppet_http(self) -> None: + self._help_test( + "tests/security/puppet/files/http.pp", "script", 1, ["sec_https"], [2] + ) + + def test_puppet_susp_comment(self) -> None: + self._help_test( + "tests/security/puppet/files/susp.pp", "script", 1, ["sec_susp_comm"], [19] + ) + + def test_puppet_def_admin(self) -> None: + self._help_test( + "tests/security/puppet/files/admin.pp", + "script", + 2, + ["sec_def_admin", "sec_hard_user"], + [7, 7], + ) + + def test_puppet_empt_pass(self) -> None: + self._help_test( + "tests/security/puppet/files/empty.pp", "script", 1, ["sec_empty_pass"], [1] + ) + + def test_puppet_weak_crypt(self) -> None: + self._help_test( + "tests/security/puppet/files/weak_crypt.pp", + "script", + 1, + ["sec_weak_crypt"], + [12], + ) + + def test_puppet_hard_secr(self) -> None: + self._help_test( + "tests/security/puppet/files/hard_secr.pp", + "script", + 1, + ["sec_hard_pass"], + [2], + ) + + def test_puppet_invalid_bind(self) -> None: + self._help_test( + "tests/security/puppet/files/inv_bind.pp", + "script", + 1, + ["sec_invalid_bind"], + [12], + ) + + def test_puppet_int_check(self) -> None: + self._help_test( + "tests/security/puppet/files/int_check.pp", + "script", + 1, + ["sec_no_int_check"], + [5], + ) + + def test_puppet_missing_default(self) -> None: + self._help_test( + "tests/security/puppet/files/missing_default.pp", + "script", + 2, + ["sec_no_default_switch", "sec_no_default_switch"], + [1, 6], + ) + + def test_puppet_full_perm(self) -> None: + self._help_test( + "tests/security/puppet/files/full_permission.pp", + "script", + 1, + ["sec_full_permission_filesystem"], + [4], + ) + + def test_puppet_obs_command(self) -> None: + self._help_test( + "tests/security/puppet/files/obs_command.pp", + "script", + 1, + ["sec_obsolete_command"], + [2], + ) diff --git a/tests/security/security_helper.py b/tests/security/security_helper.py new file mode 100644 index 00000000..f43e8f94 --- /dev/null +++ b/tests/security/security_helper.py @@ -0,0 +1,43 @@ +from glitch.__main__ import lint +from click.testing import CliRunner +from tests.base_test import BaseTest + + +class BaseSecurityTest(BaseTest): + TECH = None # subclass must override + + def _help_test( + self, path: str, type: str, n_errors: int, codes: list[str], lines: list[int] + ) -> None: + assert self.TECH is not None, "Subclasses must define TECH" + + output_path = "tests/security/dump.csv" + runner = CliRunner() + result = runner.invoke( + lint, + [ + "--tech", + self.TECH.value[0], + "--type", + type, + "--csv", + "--smell-types", + "security", + path, + output_path, + ], + ) + if result.exception: + raise result.exception + + errors = self.read_lint_csv(output_path) + + errors = [e for e in errors if e["code"].startswith("sec_")] # type: ignore + + errors = sorted( + errors, key=lambda e: (e["path"] or "", e["line"], e["code"] or "") + ) + self.assertEqual(len(errors), n_errors) + for i in range(n_errors): + self.assertEqual(errors[i]["code"], codes[i]) + self.assertEqual(errors[i]["line"], lines[i]) diff --git a/tests/security/terraform/__init__.py b/tests/security/terraform/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/glitch/tests/security/terraform/files/admin.tf b/tests/security/terraform/files/admin.tf similarity index 100% rename from glitch/tests/security/terraform/files/admin.tf rename to tests/security/terraform/files/admin.tf diff --git a/glitch/tests/security/terraform/files/attached-resource/aws_route53_attached_resource.tf b/tests/security/terraform/files/attached-resource/aws_route53_attached_resource.tf similarity index 100% rename from glitch/tests/security/terraform/files/attached-resource/aws_route53_attached_resource.tf rename to tests/security/terraform/files/attached-resource/aws_route53_attached_resource.tf diff --git a/glitch/tests/security/terraform/files/disabled-authentication/azure-app-service-authentication-activated.tf b/tests/security/terraform/files/disabled-authentication/azure-app-service-authentication-activated.tf similarity index 100% rename from glitch/tests/security/terraform/files/disabled-authentication/azure-app-service-authentication-activated.tf rename to tests/security/terraform/files/disabled-authentication/azure-app-service-authentication-activated.tf diff --git a/glitch/tests/security/terraform/files/disabled-authentication/contained-database-disabled.tf b/tests/security/terraform/files/disabled-authentication/contained-database-disabled.tf similarity index 100% rename from glitch/tests/security/terraform/files/disabled-authentication/contained-database-disabled.tf rename to tests/security/terraform/files/disabled-authentication/contained-database-disabled.tf diff --git a/glitch/tests/security/terraform/files/disabled-authentication/disable-password-authentication.tf b/tests/security/terraform/files/disabled-authentication/disable-password-authentication.tf similarity index 100% rename from glitch/tests/security/terraform/files/disabled-authentication/disable-password-authentication.tf rename to tests/security/terraform/files/disabled-authentication/disable-password-authentication.tf diff --git a/glitch/tests/security/terraform/files/disabled-authentication/gke-basic-auth.tf b/tests/security/terraform/files/disabled-authentication/gke-basic-auth.tf similarity index 100% rename from glitch/tests/security/terraform/files/disabled-authentication/gke-basic-auth.tf rename to tests/security/terraform/files/disabled-authentication/gke-basic-auth.tf diff --git a/glitch/tests/security/terraform/files/disabled-authentication/iam-group-with-mfa.tf b/tests/security/terraform/files/disabled-authentication/iam-group-with-mfa.tf similarity index 91% rename from glitch/tests/security/terraform/files/disabled-authentication/iam-group-with-mfa.tf rename to tests/security/terraform/files/disabled-authentication/iam-group-with-mfa.tf index b778608a..9b80081e 100644 --- a/glitch/tests/security/terraform/files/disabled-authentication/iam-group-with-mfa.tf +++ b/tests/security/terraform/files/disabled-authentication/iam-group-with-mfa.tf @@ -2,7 +2,7 @@ resource "aws_iam_group" "support" { name = "support" } -resource aws_iam_group_policy mfa { +resource "aws_iam_group_policy" "mfa" { group = aws_iam_group.support.name policy = < None: - parser = TerraformParser() - inter = parser.parse(path, "script", False) - analysis = SecurityVisitor(Tech.terraform) - analysis.config("configs/terraform.ini") - errors = list( - filter(lambda e: e.code.startswith("sec_"), set(analysis.check(inter))) - ) - errors = sorted(errors, key=lambda e: (e.path, e.line, e.code)) - self.assertEqual(len(errors), n_errors) - for i in range(n_errors): - self.assertEqual(errors[i].code, codes[i]) - self.assertEqual(errors[i].line, lines[i]) +class TestSecurity(BaseSecurityTest): + TECH = Tech.terraform # testing previous implemented code smells def test_terraform_http(self) -> None: - self.__help_test( - "tests/security/terraform/files/http.tf", 1, ["sec_https"], [2] + self._help_test( + "tests/security/terraform/files/http.tf", "script", 1, ["sec_https"], [2] ) def test_terraform_susp_comment(self) -> None: - self.__help_test( - "tests/security/terraform/files/susp.tf", 1, ["sec_susp_comm"], [8] + self._help_test( + "tests/security/terraform/files/susp.tf", + "script", + 1, + ["sec_susp_comm"], + [8], ) def test_terraform_def_admin(self) -> None: - self.__help_test( + self._help_test( "tests/security/terraform/files/admin.tf", - 3, - ["sec_def_admin", "sec_hard_secr", "sec_hard_user"], - [2, 2, 2], + "script", + 2, + ["sec_def_admin", "sec_hard_user"], + [2, 2], ) def test_terraform_empt_pass(self) -> None: - self.__help_test( - "tests/security/terraform/files/empty.tf", 1, ["sec_empty_pass"], [5] + self._help_test( + "tests/security/terraform/files/empty.tf", + "script", + 1, + ["sec_empty_pass"], + [5], ) def test_terraform_weak_crypt(self) -> None: - self.__help_test( - "tests/security/terraform/files/weak_crypt.tf", 1, ["sec_weak_crypt"], [4] + self._help_test( + "tests/security/terraform/files/weak_crypt.tf", + "script", + 1, + ["sec_weak_crypt"], + [4], ) def test_terraform_hard_secr(self) -> None: - self.__help_test( + self._help_test( "tests/security/terraform/files/hard_secr.tf", - 2, - ["sec_hard_pass", "sec_hard_secr"], - [5, 5], + "script", + 1, + ["sec_hard_pass"], + [5], ) def test_terraform_invalid_bind(self) -> None: - self.__help_test( - "tests/security/terraform/files/inv_bind.tf", 1, ["sec_invalid_bind"], [19] + self._help_test( + "tests/security/terraform/files/inv_bind.tf", + "script", + 1, + ["sec_invalid_bind"], + [19], ) # testing new implemented code smells, or previous ones with new rules for Terraform def test_terraform_insecure_access_control(self) -> None: - self.__help_test( + self._help_test( "tests/security/terraform/files/insecure-access-control/access-to-bigquery-dataset.tf", + "script", 1, ["sec_access_control"], [3], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/insecure-access-control/aks-ip-ranges-enabled.tf", + "script", 1, ["sec_access_control"], [1], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/insecure-access-control/associated-access-block-to-s3-bucket.tf", + "script", 1, ["sec_access_control"], [1], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/insecure-access-control/aws-database-instance-publicly-accessible.tf", + "script", 2, ["sec_access_control", "sec_access_control"], [2, 18], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/insecure-access-control/aws-sqs-no-wildcards-in-policy.tf", + "script", 1, ["sec_access_control"], [4], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/insecure-access-control/azure-authorization-wildcard-action.tf", + "script", 1, ["sec_access_control"], [7], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/insecure-access-control/azure-container-use-rbac-permissions.tf", + "script", 1, ["sec_access_control"], [2], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/insecure-access-control/azure-database-not-publicly-accessible.tf", + "script", 2, ["sec_access_control", "sec_access_control"], [1, 6], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/insecure-access-control/bucket-public-read-acl.tf", + "script", 3, ["sec_access_control", "sec_access_control", "sec_access_control"], [1, 8, 25], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/insecure-access-control/cidr-range-public-access-eks-cluster.tf", + "script", 1, ["sec_access_control"], [1], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/insecure-access-control/cross-db-ownership-chaining.tf", + "script", 3, ["sec_access_control", "sec_access_control", "sec_access_control"], [1, 50, 97], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/insecure-access-control/data-factory-public-access.tf", + "script", 2, ["sec_access_control", "sec_access_control"], [1, 5], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/insecure-access-control/google-compute-no-default-service-account.tf", + "script", 2, ["sec_access_control", "sec_access_control"], [1, 19], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/insecure-access-control/google-gke-use-rbac-permissions.tf", + "script", 1, ["sec_access_control"], [17], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/insecure-access-control/google-storage-enable-ubla.tf", + "script", 2, ["sec_access_control", "sec_access_control"], [1, 8], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/insecure-access-control/google-storage-no-public-access.tf", + "script", 3, ["sec_access_control", "sec_access_control", "sec_access_control"], [4, 10, 22], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/insecure-access-control/mq-broker-publicly-exposed.tf", + "script", 1, ["sec_access_control"], [2], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/insecure-access-control/prevent-client-disable-encryption.tf", + "script", 1, ["sec_access_control"], [13], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/insecure-access-control/private-cluster-nodes.tf", + "script", 2, ["sec_access_control", "sec_access_control"], [1, 19], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/insecure-access-control/public-access-eks-cluster.tf", + "script", 1, ["sec_access_control"], [10], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/insecure-access-control/public-access-policy.tf", + "script", 1, ["sec_access_control"], [4], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/insecure-access-control/public-github-repo.tf", + "script", 3, ["sec_access_control", "sec_access_control", "sec_access_control"], [1, 6, 18], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/insecure-access-control/s3-access-through-acl.tf", + "script", 1, ["sec_access_control"], [7], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/insecure-access-control/s3-block-public-acl.tf", + "script", 2, ["sec_access_control", "sec_access_control"], [1, 10], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/insecure-access-control/s3-block-public-policy.tf", + "script", 2, ["sec_access_control", "sec_access_control"], [1, 11], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/insecure-access-control/s3-ignore-public-acl.tf", + "script", 2, ["sec_access_control", "sec_access_control"], [1, 13], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/insecure-access-control/s3-restrict-public-bucket.tf", + "script", 2, ["sec_access_control", "sec_access_control"], [1, 12], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/insecure-access-control/specify-source-lambda-permission.tf", + "script", 1, ["sec_access_control"], [1], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/insecure-access-control/storage-containers-public-access.tf", + "script", 1, ["sec_access_control"], [26], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/insecure-access-control/unauthorized-access-api-gateway-methods.tf", + "script", 2, ["sec_access_control", "sec_access_control"], [37, 44], ) def test_terraform_invalid_ip_binding(self) -> None: - self.__help_test( + self._help_test( "tests/security/terraform/files/invalid-ip-binding/aws-ec2-vpc-no-public-egress-sgr.tf", + "script", 2, ["sec_invalid_bind", "sec_invalid_bind"], [5, 20], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/invalid-ip-binding/aws-ec2-vpc-no-public-ingress-acl.tf", + "script", 1, ["sec_invalid_bind"], [7], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/invalid-ip-binding/aws-ec2-vpc-no-public-ingress-sgr.tf", + "script", 2, ["sec_invalid_bind", "sec_invalid_bind"], [4, 17], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/invalid-ip-binding/azure-network-no-public-egress.tf", + "script", 1, ["sec_invalid_bind"], [3], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/invalid-ip-binding/azure-network-no-public-ingress.tf", + "script", 1, ["sec_invalid_bind"], [3], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/invalid-ip-binding/cloud-sql-database-publicly-exposed.tf", + "script", 1, ["sec_invalid_bind"], - [14], + [12], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/invalid-ip-binding/compute-firewall-inbound-rule-public-ip.tf", + "script", 1, ["sec_invalid_bind"], [9], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/invalid-ip-binding/compute-firewall-outbound-rule-public-ip.tf", + "script", 1, ["sec_invalid_bind"], [9], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/invalid-ip-binding/eks-cluster-open-cidr-range.tf", + "script", 1, ["sec_invalid_bind"], [11], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/invalid-ip-binding/gke-control-plane-publicly-accessible.tf", + "script", 1, ["sec_invalid_bind"], [8], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/invalid-ip-binding/openstack-networking-no-public-egress.tf", + "script", 1, ["sec_invalid_bind"], [8], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/invalid-ip-binding/openstack-networking-no-public-ingress.tf", + "script", 1, ["sec_invalid_bind"], [8], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/invalid-ip-binding/public-egress-network-policy.tf", + "script", 1, ["sec_invalid_bind"], [27], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/invalid-ip-binding/public-ingress-network-policy.tf", + "script", 1, ["sec_invalid_bind"], [27], ) def test_terraform_disabled_authentication(self) -> None: - self.__help_test( + self._help_test( "tests/security/terraform/files/disabled-authentication/azure-app-service-authentication-activated.tf", + "script", 2, ["sec_authentication", "sec_authentication"], [1, 11], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/disabled-authentication/contained-database-disabled.tf", + "script", 1, ["sec_authentication"], [1], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/disabled-authentication/disable-password-authentication.tf", + "script", 3, ["sec_authentication", "sec_authentication", "sec_authentication"], [2, 13, 18], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/disabled-authentication/gke-basic-auth.tf", + "script", 1, ["sec_authentication"], [4], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/disabled-authentication/iam-group-with-mfa.tf", + "script", 2, ["sec_authentication", "sec_authentication"], [7, 53], ) def test_terraform_missing_encryption(self) -> None: - self.__help_test( + self._help_test( "tests/security/terraform/files/missing-encryption/athena-enable-at-rest-encryption.tf", + "script", 2, ["sec_missing_encryption", "sec_missing_encryption"], [1, 10], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/missing-encryption/aws-codebuild-enable-encryption.tf", + "script", 2, ["sec_missing_encryption", "sec_missing_encryption"], [3, 9], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/missing-encryption/aws-ecr-encrypted.tf", + "script", 2, ["sec_missing_encryption", "sec_missing_encryption"], [1, 17], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/missing-encryption/aws-neptune-at-rest-encryption.tf", + "script", 2, ["sec_missing_encryption", "sec_missing_encryption"], [1, 9], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/missing-encryption/documentdb-storage-encryption.tf", + "script", 2, ["sec_missing_encryption", "sec_missing_encryption"], [1, 9], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/missing-encryption/dynamodb-rest-encryption.tf", + "script", 2, ["sec_missing_encryption", "sec_missing_encryption"], [1, 9], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/missing-encryption/ecs-task-definitions-in-transit-encryption.tf", + "script", 2, ["sec_missing_encryption", "sec_missing_encryption"], [1, 29], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/missing-encryption/efs-encryption.tf", + "script", 2, ["sec_missing_encryption", "sec_missing_encryption"], [1, 6], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/missing-encryption/eks-encryption-secrets-enabled.tf", + "script", 5, [ "sec_missing_encryption", @@ -425,68 +486,79 @@ def test_terraform_missing_encryption(self) -> None: ], [1, 1, 9, 23, 34], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/missing-encryption/elasticache-enable-at-rest-encryption.tf", + "script", 2, ["sec_missing_encryption", "sec_missing_encryption"], [1, 6], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/missing-encryption/elasticache-enable-in-transit-encryption.tf", + "script", 2, ["sec_missing_encryption", "sec_missing_encryption"], [1, 7], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/missing-encryption/elasticsearch-domain-encrypted.tf", + "script", 2, ["sec_missing_encryption", "sec_missing_encryption"], [1, 17], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/missing-encryption/elasticsearch-in-transit-encryption.tf", + "script", 2, ["sec_missing_encryption", "sec_missing_encryption"], [1, 16], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/missing-encryption/emr-enable-at-rest-encryption.tf", + "script", 1, ["sec_missing_encryption"], [4], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/missing-encryption/emr-enable-in-transit-encryption.tf", + "script", 1, ["sec_missing_encryption"], [4], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/missing-encryption/emr-enable-local-disk-encryption.tf", + "script", 1, ["sec_missing_encryption"], [4], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/missing-encryption/emr-s3encryption-mode-sse-kms.tf", + "script", 1, ["sec_missing_encryption"], [4], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/missing-encryption/enable-cache-encryption.tf", + "script", 2, ["sec_missing_encryption", "sec_missing_encryption"], [1, 6], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/missing-encryption/encrypted-ebs-volume.tf", + "script", 2, ["sec_missing_encryption", "sec_missing_encryption"], [1, 7], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/missing-encryption/encrypted-root-block-device.tf", + "script", 4, [ "sec_missing_encryption", @@ -496,20 +568,23 @@ def test_terraform_missing_encryption(self) -> None: ], [1, 13, 23, 27], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/missing-encryption/instance-encrypted-block-device.tf", + "script", 1, ["sec_missing_encryption"], [14], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/missing-encryption/kinesis-stream-encryption.tf", + "script", 2, ["sec_missing_encryption", "sec_missing_encryption"], [1, 6], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/missing-encryption/msk-enable-in-transit-encryption.tf", + "script", 3, [ "sec_missing_encryption", @@ -518,32 +593,37 @@ def test_terraform_missing_encryption(self) -> None: ], [1, 14, 15], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/missing-encryption/rds-encrypt-cluster-storage-data.tf", + "script", 2, ["sec_missing_encryption", "sec_missing_encryption"], [1, 6], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/missing-encryption/rds-encrypt-instance-storage-data.tf", + "script", 2, ["sec_missing_encryption", "sec_missing_encryption"], [1, 8], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/missing-encryption/redshift-cluster-rest-encryption.tf", + "script", 2, ["sec_missing_encryption", "sec_missing_encryption"], [1, 6], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/missing-encryption/unencrypted-s3-bucket.tf", + "script", 2, ["sec_missing_encryption", "sec_missing_encryption"], [25, 64], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/missing-encryption/workspaces-disk-encryption.tf", + "script", 6, [ "sec_missing_encryption", @@ -557,216 +637,236 @@ def test_terraform_missing_encryption(self) -> None: ) def test_terraform_hard_coded_secrets(self) -> None: - self.__help_test( + self._help_test( "tests/security/terraform/files/hard-coded-secrets/encryption-key-in-plaintext.tf", + "script", 1, ["sec_hard_secr"], [3], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/hard-coded-secrets/plaintext-password.tf", - 2, - ["sec_hard_pass", "sec_hard_secr"], - [2, 2], - ) - self.__help_test( - "tests/security/terraform/files/hard-coded-secrets/plaintext-value-github-actions.tf", + "script", 1, - ["sec_hard_secr"], - [5], + ["sec_hard_pass"], + [2], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/hard-coded-secrets/sensitive-credentials-in-vm-custom-data.tf", - 2, - ["sec_hard_pass", "sec_hard_secr"], - [3, 3], - ) - self.__help_test( - "tests/security/terraform/files/hard-coded-secrets/sensitive-data-in-plaintext.tf", - 2, - ["sec_hard_pass", "sec_hard_secr"], - [8, 8], + "script", + 1, + ["sec_hard_pass"], + [3], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/hard-coded-secrets/sensitive-data-stored-in-user-data.tf", - 4, - ["sec_hard_pass", "sec_hard_secr", "sec_hard_pass", "sec_hard_secr"], - [2, 2, 14, 14], + "script", + 2, + ["sec_hard_pass", "sec_hard_pass"], + [2, 14], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/hard-coded-secrets/sensitive-environment-variables.tf", - 2, - ["sec_hard_pass", "sec_hard_secr"], - [2, 2], + "script", + 1, + ["sec_hard_pass"], + [2], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/hard-coded-secrets/user-data-contains-sensitive-aws-keys.tf", + "script", 1, ["sec_hard_secr"], [9], ) def test_terraform_public_ip(self) -> None: - self.__help_test( + self._help_test( "tests/security/terraform/files/public-ip/google-compute-intance-with-public-ip.tf", + "script", 1, ["sec_public_ip"], [4], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/public-ip/lauch-configuration-public-ip-addr.tf", + "script", 1, ["sec_public_ip"], [2], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/public-ip/oracle-compute-no-public-ip.tf", + "script", 1, ["sec_public_ip"], [3], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/public-ip/subnet-public-ip-address.tf", + "script", 1, ["sec_public_ip"], [3], ) def test_terraform_use_of_http_without_tls(self) -> None: - self.__help_test( + self._help_test( "tests/security/terraform/files/use-of-http-without-tls/azure-appservice-enforce-https.tf", + "script", 2, ["sec_https", "sec_https"], [1, 8], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/use-of-http-without-tls/azure-storage-enforce-https.tf", + "script", 1, ["sec_https"], [2], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/use-of-http-without-tls/cloudfront-enforce-https.tf", + "script", 2, ["sec_https", "sec_https"], [1, 13], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/use-of-http-without-tls/digitalocean-compute-enforce-https.tf", + "script", 1, ["sec_https"], [7], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/use-of-http-without-tls/elastic-search-enforce-https.tf", + "script", 2, ["sec_https", "sec_https"], [1, 19], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/use-of-http-without-tls/elb-use-plain-http.tf", + "script", 2, ["sec_https", "sec_https"], [1, 6], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/use-of-http-without-tls/aws-ssm-avoid-leaks-via-http.tf", + "script", 1, ["sec_https"], [8], ) def test_terraform_ssl_tls_mtls_policy(self) -> None: - self.__help_test( + self._help_test( "tests/security/terraform/files/ssl-tls-mtls-policy/api-gateway-secure-tls-policy.tf", + "script", 2, ["sec_ssl_tls_policy", "sec_ssl_tls_policy"], [1, 5], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/ssl-tls-mtls-policy/azure-appservice-require-client-cert.tf", + "script", 2, ["sec_ssl_tls_policy", "sec_ssl_tls_policy"], [1, 9], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/ssl-tls-mtls-policy/azure-appservice-secure-tls-policy.tf", + "script", 1, ["sec_ssl_tls_policy"], [3], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/ssl-tls-mtls-policy/azure-storage-use-secure-tls-policy.tf", + "script", 1, ["sec_ssl_tls_policy"], [2], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/ssl-tls-mtls-policy/cloudfront-secure-tls-policy.tf", + "script", 2, ["sec_ssl_tls_policy", "sec_ssl_tls_policy"], [1, 13], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/ssl-tls-mtls-policy/database-enable.ssl-eforcement.tf", + "script", 2, ["sec_ssl_tls_policy", "sec_ssl_tls_policy"], [1, 8], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/ssl-tls-mtls-policy/database-secure-tls-policy.tf", + "script", 2, ["sec_ssl_tls_policy", "sec_ssl_tls_policy"], [2, 22], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/ssl-tls-mtls-policy/elastic-search-secure-tls-policy.tf", + "script", 2, ["sec_ssl_tls_policy", "sec_ssl_tls_policy"], [1, 20], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/ssl-tls-mtls-policy/elb-secure-tls-policy.tf", + "script", 2, ["sec_ssl_tls_policy", "sec_ssl_tls_policy"], [1, 6], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/ssl-tls-mtls-policy/google-compute-secure-tls-policy.tf", + "script", 2, ["sec_ssl_tls_policy", "sec_ssl_tls_policy"], [1, 5], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/ssl-tls-mtls-policy/sql-encrypt-in-transit-data.tf", + "script", 2, ["sec_ssl_tls_policy", "sec_ssl_tls_policy"], [1, 45], ) def test_terraform_use_of_dns_without_dnssec(self) -> None: - self.__help_test( + self._help_test( "tests/security/terraform/files/use-of-dns-without-dnssec/cloud-dns-without-dnssec.tf", + "script", 2, ["sec_dnssec", "sec_dnssec"], [1, 6], ) def test_terraform_firewall_misconfiguration(self) -> None: - self.__help_test( + self._help_test( "tests/security/terraform/files/firewall-misconfiguration/alb-drop-invalid-headers.tf", + "script", 2, ["sec_firewall_misconfig", "sec_firewall_misconfig"], [1, 7], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/firewall-misconfiguration/alb-exposed-to-internet.tf", + "script", 2, ["sec_firewall_misconfig", "sec_firewall_misconfig"], [1, 7], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/firewall-misconfiguration/azure-keyvault-specify-network-acl.tf", + "script", 3, [ "sec_firewall_misconfig", @@ -775,38 +875,44 @@ def test_terraform_firewall_misconfiguration(self) -> None: ], [1, 1, 13], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/firewall-misconfiguration/cloudfront-use-waf.tf", + "script", 2, ["sec_firewall_misconfig", "sec_firewall_misconfig"], [1, 14], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/firewall-misconfiguration/config-master-authorized-networks.tf", + "script", 1, ["sec_firewall_misconfig"], [1], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/firewall-misconfiguration/google-compute-inbound-rule-traffic.tf", + "script", 1, ["sec_firewall_misconfig"], [1], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/firewall-misconfiguration/google-compute-no-ip-forward.tf", + "script", 1, ["sec_firewall_misconfig"], [2], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/firewall-misconfiguration/google-compute-outbound-rule-traffic.tf", + "script", 1, ["sec_firewall_misconfig"], [1], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/firewall-misconfiguration/openstack-compute-no-public-access.tf", + "script", 3, [ "sec_firewall_misconfig", @@ -817,134 +923,155 @@ def test_terraform_firewall_misconfiguration(self) -> None: ) def test_terraform_missing_threats_detection_and_alerts(self) -> None: - self.__help_test( + self._help_test( "tests/security/terraform/files/missing-threats-detection-and-alerts/azure-database-disabled-alerts.tf", + "script", 1, ["sec_threats_detection_alerts"], [2], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/missing-threats-detection-and-alerts/azure-database-email-admin.tf", + "script", 2, ["sec_threats_detection_alerts", "sec_threats_detection_alerts"], [1, 7], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/missing-threats-detection-and-alerts/azure-database-email-for-alerts.tf", + "script", 1, ["sec_threats_detection_alerts"], [1], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/missing-threats-detection-and-alerts/azure-security-center-alert-notifications.tf", + "script", 2, ["sec_threats_detection_alerts", "sec_threats_detection_alerts"], [5, 6], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/missing-threats-detection-and-alerts/azure-security-require-contact-phone.tf", + "script", 2, ["sec_threats_detection_alerts", "sec_threats_detection_alerts"], [1, 10], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/missing-threats-detection-and-alerts/github-repo-vulnerability-alerts.tf", + "script", 2, ["sec_threats_detection_alerts", "sec_threats_detection_alerts"], [1, 16], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/missing-threats-detection-and-alerts/aws-ecr-enable-image-scans.tf", + "script", 2, ["sec_threats_detection_alerts", "sec_threats_detection_alerts"], [1, 19], ) def test_terraform_weak_password_key_policy(self) -> None: - self.__help_test( + self._help_test( "tests/security/terraform/files/weak-password-key-policy/aws-iam-no-password-reuse.tf", + "script", 2, ["sec_weak_password_key_policy", "sec_weak_password_key_policy"], [1, 11], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/weak-password-key-policy/aws-iam-require-lowercase-in-passwords.tf", + "script", 2, ["sec_weak_password_key_policy", "sec_weak_password_key_policy"], [1, 11], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/weak-password-key-policy/aws-iam-require-numbers-in-passwords.tf", + "script", 2, ["sec_weak_password_key_policy", "sec_weak_password_key_policy"], [1, 11], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/weak-password-key-policy/aws-iam-require-symbols-in-passwords.tf", + "script", 2, ["sec_weak_password_key_policy", "sec_weak_password_key_policy"], [1, 11], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/weak-password-key-policy/aws-iam-require-uppercase-in-passwords.tf", + "script", 2, ["sec_weak_password_key_policy", "sec_weak_password_key_policy"], [1, 11], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/weak-password-key-policy/aws-iam-set-max-password-age.tf", + "script", 2, ["sec_weak_password_key_policy", "sec_weak_password_key_policy"], [1, 11], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/weak-password-key-policy/aws-iam-set-minimum-password-length.tf", + "script", 2, ["sec_weak_password_key_policy", "sec_weak_password_key_policy"], [1, 11], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/weak-password-key-policy/azure-keyvault-ensure-secret-expiry.tf", + "script", 2, ["sec_weak_password_key_policy", "sec_weak_password_key_policy"], [1, 11], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/weak-password-key-policy/azure-keyvault-no-purge.tf", + "script", 2, ["sec_weak_password_key_policy", "sec_weak_password_key_policy"], [1, 11], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/weak-password-key-policy/azure-keyvault-ensure-key-expiration-date.tf", + "script", 2, ["sec_weak_password_key_policy", "sec_weak_password_key_policy"], [1, 5], ) def test_terraform_integrity_policy(self) -> None: - self.__help_test( + self._help_test( "tests/security/terraform/files/integrity-policy/aws-ecr-immutable-repo.tf", + "script", 2, ["sec_integrity_policy", "sec_integrity_policy"], [1, 13], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/integrity-policy/google-compute-enable-integrity-monitoring.tf", + "script", 1, ["sec_integrity_policy"], [3], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/integrity-policy/google-compute-enable-virtual-tpm.tf", + "script", 1, ["sec_integrity_policy"], [3], ) def test_terraform_sensitive_action_by_iam(self) -> None: - self.__help_test( + self._help_test( "tests/security/terraform/files/sensitive-action-by-iam/aws-iam-no-policy-wildcards.tf", + "script", 3, [ "sec_sensitive_iam_action", @@ -955,178 +1082,207 @@ def test_terraform_sensitive_action_by_iam(self) -> None: ) def test_terraform_key_management(self) -> None: - self.__help_test( + self._help_test( "tests/security/terraform/files/key-management/aws-cloudtrail-encryption-use-cmk.tf", + "script", 2, ["sec_key_management", "sec_key_management"], [1, 8], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/key-management/aws-cloudwatch-log-group-customer-key.tf", + "script", 2, ["sec_key_management", "sec_key_management"], [1, 6], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/key-management/aws-documentdb-use-cmk.tf", + "script", 2, ["sec_key_management", "sec_key_management"], [1, 7], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/key-management/aws-dynamodb-table-use-cmk.tf", + "script", 2, ["sec_key_management", "sec_key_management"], [1, 10], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/key-management/aws-ebs-use-cmk.tf", + "script", 2, ["sec_key_management", "sec_key_management"], [1, 7], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/key-management/aws-ecr-use-cmk.tf", + "script", 2, ["sec_key_management", "sec_key_management"], [1, 18], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/key-management/aws-kinesis-stream-use-cmk.tf", + "script", 2, ["sec_key_management", "sec_key_management"], [1, 7], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/key-management/aws-kms-auto-rotate-keys.tf", + "script", 2, ["sec_key_management", "sec_key_management"], [1, 5], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/key-management/aws-neptune-use-cmk.tf", + "script", 2, ["sec_key_management", "sec_key_management"], [1, 9], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/key-management/aws-sns-topic-encryption-use-cmk.tf", + "script", 2, ["sec_key_management", "sec_key_management"], [1, 5], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/key-management/aws-sqs-queue-encryption-use-cmk.tf", + "script", 2, ["sec_key_management", "sec_key_management"], [1, 5], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/key-management/aws-ssm-secret-use-cmk.tf", + "script", 2, ["sec_key_management", "sec_key_management"], [1, 7], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/key-management/azure-storage-account-use-cmk.tf", + "script", 1, ["sec_key_management"], [1], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/key-management/google-compute-disk-encryption-use-cmk.tf", + "script", 2, ["sec_key_management", "sec_key_management"], [1, 6], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/key-management/google-compute-no-project-wide-ssh-keys.tf", + "script", 2, ["sec_key_management", "sec_key_management"], [1, 12], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/key-management/google-compute-vm-disk-encryption-use-cmk.tf", + "script", 2, ["sec_key_management", "sec_key_management"], [1, 12], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/key-management/google-kms-rotate-kms-keys.tf", + "script", 3, ["sec_key_management", "sec_key_management", "sec_key_management"], [1, 9, 15], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/key-management/rds-cluster-use-cmk.tf", + "script", 2, ["sec_key_management", "sec_key_management"], [1, 7], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/key-management/rds-instance-use-cmk.tf", + "script", 2, ["sec_key_management", "sec_key_management"], [1, 8], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/key-management/rds-performance-insights-use-cmk.tf", + "script", 2, ["sec_key_management", "sec_key_management"], [1, 9], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/key-management/redshift-cluster-use-cmk.tf", + "script", 2, ["sec_key_management", "sec_key_management"], [1, 7], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/key-management/s3-encryption-customer-key.tf", + "script", 2, ["sec_key_management", "sec_key_management"], [9, 47], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/key-management/digitalocean-compute-use-ssh-keys.tf", + "script", 1, ["sec_key_management"], [1], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/key-management/google-storage-bucket-encryption-customer-key.tf", + "script", 2, ["sec_key_management", "sec_key_management"], [1, 8], ) def test_terraform_network_security_rules(self) -> None: - self.__help_test( + self._help_test( "tests/security/terraform/files/network-security-rules/aws-vpc-ec2-use-tcp.tf", + "script", 1, ["sec_network_security_rules"], [2], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/network-security-rules/azure-container-configured-network-policy.tf", + "script", 2, ["sec_network_security_rules", "sec_network_security_rules"], [1, 15], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/network-security-rules/azure-network-disable-rdp-from-internet.tf", + "script", 2, ["sec_network_security_rules", "sec_network_security_rules"], [8, 32], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/network-security-rules/azure-network-ssh-blocked-from-internet.tf", + "script", 2, ["sec_network_security_rules", "sec_network_security_rules"], [8, 32], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/network-security-rules/azure-storage-default-action-deny.tf", + "script", 3, [ "sec_network_security_rules", @@ -1135,228 +1291,265 @@ def test_terraform_network_security_rules(self) -> None: ], [1, 8, 21], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/network-security-rules/azure-synapse-virtual-network-enabled.tf", + "script", 2, ["sec_network_security_rules", "sec_network_security_rules"], [1, 5], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/network-security-rules/google-compute-no-serial-port.tf", + "script", 1, ["sec_network_security_rules"], [4], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/network-security-rules/google-gke-enable-ip-aliasing.tf", + "script", 1, ["sec_network_security_rules"], [1], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/network-security-rules/google-gke-enable-network-policy.tf", + "script", 2, ["sec_network_security_rules", "sec_network_security_rules"], [1, 19], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/network-security-rules/google-iam-no-default-network.tf", + "script", 2, ["sec_network_security_rules", "sec_network_security_rules"], [1, 5], ) def test_terraform_permission_of_iam_policies(self) -> None: - self.__help_test( + self._help_test( "tests/security/terraform/files/permission-of-iam-policies/default-service-account-not-used-at-folder-level.tf", + "script", 2, ["sec_permission_iam_policies", "sec_permission_iam_policies"], [4, 10], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/permission-of-iam-policies/default-service-account-not-used-at-organization-level.tf", + "script", 2, ["sec_permission_iam_policies", "sec_permission_iam_policies"], [4, 10], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/permission-of-iam-policies/default-service-account-not-used-at-project-level.tf", + "script", 2, ["sec_permission_iam_policies", "sec_permission_iam_policies"], [4, 10], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/permission-of-iam-policies/google-iam-no-folder-level-service-account-impersonation.tf", + "script", 1, ["sec_permission_iam_policies"], [3], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/permission-of-iam-policies/google-iam-no-organization-level-service-account-impersonation.tf", + "script", 1, ["sec_permission_iam_policies"], [3], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/permission-of-iam-policies/google-iam-no-project-level-service-account-impersonation.tf", + "script", 1, ["sec_permission_iam_policies"], [3], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/permission-of-iam-policies/google-iam-no-user-granted-permissions.tf", + "script", 2, ["sec_permission_iam_policies", "sec_permission_iam_policies"], [2, 6], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/permission-of-iam-policies/iam-policies-attached-only-to-groups-or-roles.tf", + "script", 1, ["sec_permission_iam_policies"], [7], ) def test_terraform_logging(self) -> None: - self.__help_test( + self._help_test( "tests/security/terraform/files/logging/aws-api-gateway-enable-access-logging.tf", + "script", 4, ["sec_logging", "sec_logging", "sec_logging", "sec_logging"], [1, 4, 10, 17], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/logging/aws-api-gateway-enable-tracing.tf", + "script", 2, ["sec_logging", "sec_logging"], [1, 9], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/logging/aws-cloudfront-enable-logging.tf", + "script", 2, ["sec_logging", "sec_logging"], [1, 13], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/logging/aws-cloudtrail-enable-log-validation.tf", + "script", 2, ["sec_logging", "sec_logging"], [1, 7], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/logging/aws-cloudtrail-ensure-cloudwatch-integration.tf", + "script", 2, ["sec_logging", "sec_logging"], [1, 7], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/logging/aws-documentdb-enable-log-export.tf", + "script", 2, ["sec_logging", "sec_logging"], [1, 7], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/logging/aws-eks-enable-control-plane-logging.tf", + "script", 2, ["sec_logging", "sec_logging"], [1, 15], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/logging/aws-elastic-search-enable-domain-logging.tf", + "script", 3, ["sec_logging", "sec_logging", "sec_logging"], [1, 17, 36], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/logging/aws-lambda-enable-tracing.tf", + "script", 2, ["sec_logging", "sec_logging"], [1, 6], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/logging/aws-mq-enable-audit-logging.tf", + "script", 2, ["sec_logging", "sec_logging"], [1, 10], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/logging/aws-mq-enable-general-logging.tf", + "script", 2, ["sec_logging", "sec_logging"], [1, 10], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/logging/aws-msk-enable-logging.tf", + "script", 5, ["sec_logging", "sec_logging", "sec_logging", "sec_logging", "sec_logging"], [1, 14, 48, 51, 54], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/logging/aws-neptune-enable-log-export.tf", + "script", 2, ["sec_logging", "sec_logging"], [1, 7], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/logging/aws-rds-enable-performance-insights.tf", + "script", 2, ["sec_logging", "sec_logging"], [1, 7], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/logging/aws-s3-enable-bucket-logging.tf", + "script", 1, ["sec_logging"], [1], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/logging/azure-container-aks-logging-configured.tf", + "script", 2, ["sec_logging", "sec_logging"], [1, 13], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/logging/azure-monitor-activity-log-retention-set.tf", + "script", 2, ["sec_logging", "sec_logging"], [1, 8], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/logging/azure-monitor-capture-all-activities.tf", + "script", 2, ["sec_logging", "sec_logging"], [1, 8], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/logging/azure-mssql-database-enable-audit.tf", + "script", 1, ["sec_logging"], [1], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/logging/azure-mssql-server-and-database-retention-period-set.tf", + "script", 3, ["sec_logging", "sec_logging", "sec_logging"], [3, 13, 18], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/logging/azure-mssql-server-enable-audit.tf", + "script", 1, ["sec_logging"], [1], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/logging/azure-network-retention-policy-set.tf", + "script", 2, ["sec_logging", "sec_logging"], [1, 7], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/logging/azure-postgres-configuration-enabled-logs.tf", + "script", 3, ["sec_logging", "sec_logging", "sec_logging"], [5, 12, 19], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/logging/azure-storage-queue-services-logging-enabled.tf", + "script", 6, [ "sec_logging", @@ -1368,32 +1561,37 @@ def test_terraform_logging(self) -> None: ], [1, 1, 1, 15, 16, 17], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/logging/ensure-cloudwatch-log-group-specifies-retention-days.tf", + "script", 2, ["sec_logging", "sec_logging"], [1, 6], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/logging/google-compute-enable-vpc-flow-logs.tf", + "script", 1, ["sec_logging"], [1], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/logging/google-gke-enable-stackdriver-logging.tf", + "script", 1, ["sec_logging"], [2], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/logging/google-gke-enable-stackdriver-monitoring.tf", + "script", 1, ["sec_logging"], [2], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/logging/google-sql-database-log-flags.tf", + "script", 12, [ "sec_logging", @@ -1411,88 +1609,101 @@ def test_terraform_logging(self) -> None: ], [1, 1, 1, 1, 1, 36, 40, 44, 48, 52, 56, 60], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/logging/storage-logging-enabled-for-blob-service-for-read-requests.tf", + "script", 4, ["sec_logging", "sec_logging", "sec_logging", "sec_logging"], [1, 8, 49, 79], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/logging/aws-ecs-enable-container-insight.tf", + "script", 3, ["sec_logging", "sec_logging", "sec_logging"], [1, 7, 11], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/logging/aws-vpc-flow-logs-enabled.tf", + "script", 1, ["sec_logging"], [11], ) def test_terraform_attached_resource(self) -> None: - self.__help_test( + self._help_test( "tests/security/terraform/files/attached-resource/aws_route53_attached_resource.tf", + "script", 2, ["sec_attached_resource", "sec_attached_resource"], [12, 16], ) def test_terraform_versioning(self) -> None: - self.__help_test( + self._help_test( "tests/security/terraform/files/versioning/aws-s3-enable-versioning.tf", + "script", 2, ["sec_versioning", "sec_versioning"], [1, 8], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/versioning/digitalocean-spaces-versioning-enabled.tf", + "script", 2, ["sec_versioning", "sec_versioning"], [1, 7], ) def test_terraform_naming(self) -> None: - self.__help_test( + self._help_test( "tests/security/terraform/files/naming/aws-ec2-description-to-security-group-rule.tf", + "script", 2, ["sec_naming", "sec_naming"], [1, 14], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/naming/aws-ec2-description-to-security-group.tf", + "script", 2, ["sec_naming", "sec_naming"], [1, 5], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/naming/aws-elasticache-description-for-security-group.tf", + "script", 2, ["sec_naming", "sec_naming"], [1, 7], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/naming/naming-rules-storage-accounts.tf", + "script", 2, ["sec_naming", "sec_naming"], [2, 21], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/naming/openstack-networking-describe-security-group.tf", + "script", 2, ["sec_naming", "sec_naming"], [1, 5], ) - self.__help_test( + self._help_test( "tests/security/terraform/files/naming/google-gke-use-cluster-labels.tf", + "script", 2, ["sec_naming", "sec_naming"], [1, 19], ) def test_terraform_replication(self) -> None: - self.__help_test( + self._help_test( "tests/security/terraform/files/replication/s3-bucket-cross-region-replication.tf", + "script", 2, ["sec_replication", "sec_replication"], [9, 16], diff --git a/vscode-extension/glitch/package-lock.json b/vscode-extension/glitch/package-lock.json index c007794e..97ee1e3f 100644 --- a/vscode-extension/glitch/package-lock.json +++ b/vscode-extension/glitch/package-lock.json @@ -1,12 +1,12 @@ { - "name": "glitch", - "version": "0.0.1", + "name": "glitch-iac", + "version": "0.0.3", "lockfileVersion": 2, "requires": true, "packages": { "": { - "name": "glitch", - "version": "0.0.1", + "name": "glitch-iac", + "version": "0.0.3", "dependencies": { "child_process": "^1.0.2" }, diff --git a/vscode-extension/glitch/package.json b/vscode-extension/glitch/package.json index 9385dc45..5fb6c3ac 100644 --- a/vscode-extension/glitch/package.json +++ b/vscode-extension/glitch/package.json @@ -8,7 +8,7 @@ "type": "git", "url": "https://github.com/sr-lab/GLITCH.git" }, - "version": "0.0.2", + "version": "0.0.3", "engines": { "vscode": "^1.68.0" }, @@ -17,29 +17,35 @@ "Linters" ], "keywords": [ - "puppet", "ansible", "chef", "IaC", "infrastruture" + "puppet", "ansible", "chef", "terraform", "IaC", "infrastructure" ], "activationEvents": [ "onLanguage:ruby", - "onLanguage:puppet", - "onLanguage:yaml" + "onLanguage:puppet", + "onLanguage:yaml", + "onLanguage:terraform" ], "contributes": { "languages": [ { "id": "puppet", - "extensions": [ ".pp"], + "extensions": [".pp"], "aliases": ["Puppet"] }, { "id": "yaml", - "extensions": [ ".yaml", ".yml"], + "extensions": [".yaml", ".yml"], "aliases": ["YAML"] }, { "id": "ruby", - "extensions": [ ".rb"], + "extensions": [".rb"], "aliases": ["Ruby"] + }, + { + "id": "terraform", + "extensions": [".tf"], + "aliases": ["Terraform", "HCL"] } ], "configuration": { @@ -60,11 +66,11 @@ "default": "", "description": "Choose the technology to which you will apply GLITCH. The value should be a valid string for the command line tool's tech option. If you do not define the value, the extension will use the default for the file extension of the current file." }, - "glitch.smells": { + "glitch.smellTypes": { "type": "array", - "items" : {"type": "string"}, + "items": {"type": "string"}, "default": [], - "description": "Select a subset of the type of smells to analyze. The values should be contained in the possible choices for the smells option of the command-line tool. If you do not define any value, it will default to run every analyses." + "description": "Select a subset of smell types to analyze (e.g., 'design', 'security'). If empty, all analyses run." } } } diff --git a/vscode-extension/glitch/src/diagnostics.ts b/vscode-extension/glitch/src/diagnostics.ts index a4f9c876..4c8dbaac 100644 --- a/vscode-extension/glitch/src/diagnostics.ts +++ b/vscode-extension/glitch/src/diagnostics.ts @@ -1,102 +1,203 @@ +import * as cp from 'child_process'; import * as vscode from 'vscode'; -import * as cp from "child_process"; +const DEBOUNCE_MS = 500; -const execShell = (cmd: string) => - new Promise((resolve, reject) => { - cp.exec(cmd, (err, out) => { - if (err) { - return reject(err.message); - } - return resolve(out); - }); +let debounceTimer: NodeJS.Timeout|undefined; +let currentProcess: cp.ChildProcess|undefined; +let glitchNotFoundShown = false; + +function execGlitch( + cmd: string, token: vscode.CancellationToken): Promise { + return new Promise((resolve, reject) => { + const process = cp.exec(cmd, (err, stdout) => { + currentProcess = undefined; + if (token.isCancellationRequested) { + return resolve(''); + } + if (err) { + const message = err.message.toLowerCase(); + if (message.includes('not found') || + message.includes('not recognized') || message.includes('enoent') || + (err as NodeJS.ErrnoException).code === 'ENOENT' || + process.exitCode === 127) { + return reject(new Error('GLITCH_NOT_FOUND')); + } + return reject(err); + } + return resolve(stdout); + }); + + currentProcess = process; + + token.onCancellationRequested(() => { + if (process && !process.killed) { + process.kill(); + } }); + }); +} + +async function refreshDiagnostics( + doc: vscode.TextDocument, glitchDiagnostics: vscode.DiagnosticCollection, + token: vscode.CancellationToken): Promise { + const configuration = vscode.workspace.getConfiguration('glitch'); + const diagnostics: vscode.Diagnostic[] = []; + + if (!configuration.get('enable')) { + glitchDiagnostics.set(doc.uri, []); + return; + } + + let options = ''; + + const config = configuration.get('configurationPath'); + if (config && config !== '') { + options += ` --config "${config}"`; + } + + const tech = configuration.get('tech'); + if (tech && tech !== '') { + options += ` --tech ${tech}`; + } else if (doc.fileName.endsWith('.yaml') || doc.fileName.endsWith('.yml')) { + options += ' --tech ansible'; + } else if (doc.fileName.endsWith('.rb')) { + options += ' --tech chef'; + } else if (doc.fileName.endsWith('.pp')) { + options += ' --tech puppet'; + } else if (doc.fileName.endsWith('.tf')) { + options += ' --tech terraform'; + } else { + return; + } + + const smellTypes = configuration.get('smellTypes') || []; + for (const smellType of smellTypes) { + options += ` --smell-types ${smellType}`; + } + + const cmd = `glitch lint${options} --linter "${doc.fileName}"`; + + try { + const csv = await execGlitch(cmd, token); + + if (token.isCancellationRequested) { + return; + } + + glitchNotFoundShown = false; -export async function refreshDiagnostics(doc: vscode.TextDocument, - glitchDiagnostics: vscode.DiagnosticCollection) { - const configuration = vscode.workspace.getConfiguration('glitch'); - const diagnostics: vscode.Diagnostic[] = []; - - if (!configuration.get('enable')) { - glitchDiagnostics.set(doc.uri, []); - return; - } - - let options = "" - - let config = configuration.get('configurationPath'); - if (config != "") { - options += " --config " + config; - } - - let tech = configuration.get('tech'); - if (tech != "") { - options += " --tech " + tech; - } else if (doc.fileName.endsWith(".yaml") || doc.fileName.endsWith(".yml")) { - options += " --tech ansible" - } else if (doc.fileName.endsWith(".rb")) { - options += " --tech chef" - } else if (doc.fileName.endsWith(".pp")) { - options += " --tech puppet" - } else { - return; - } - - let smells: string[] | undefined = configuration.get('smells'); - for (let i = 0; i < smells!.length; i++) { - options += " --smells " + smells![i]; - } - - execShell( - 'glitch --linter ' + options + " " + doc.fileName, - ).then(csv => { - let lines = csv.split('\n'); - lines = lines.filter(line => line.includes(',')); - for (let l = 0; l < lines.length; l++) { - let split = lines[l].split(',', 5); - - let line = parseInt(split[2]) - if (line < 0) { line = 1; } - - const range = new vscode.Range(line - 1, 0, line, 0); - diagnostics.push( - new vscode.Diagnostic( - range, - split[0], - vscode.DiagnosticSeverity.Warning - ) - ); - } - - glitchDiagnostics.set(doc.uri, diagnostics); - }) - .catch(reason => { - vscode.window.showErrorMessage(reason.split('Error:')[1]); - }); + const lines = csv.split('\n').filter(line => line.includes(',')); + + for (const line of lines) { + const split = line.split(',', 6); + if (split.length < 4) { + continue; + } + + let lineNum = parseInt(split[2], 10); + if (isNaN(lineNum) || lineNum < 1) { + lineNum = 1; + } + + const range = new vscode.Range( + lineNum - 1, 0, lineNum - 1, Number.MAX_SAFE_INTEGER); + const diagnostic = new vscode.Diagnostic( + range, split[0], vscode.DiagnosticSeverity.Warning); + + if (split[3]) { + diagnostic.code = split[3]; + } + + diagnostics.push(diagnostic); + } + + glitchDiagnostics.set(doc.uri, diagnostics); + } catch (err) { + if (err instanceof Error) { + if (err.message === 'GLITCH_NOT_FOUND') { + if (!glitchNotFoundShown) { + glitchNotFoundShown = true; + vscode.window.showErrorMessage( + 'GLITCH not found. Install with: pip install glitch'); + } + } else if (err.message.includes('rego') && err.message.includes('not exist')) { + if (!glitchNotFoundShown) { + glitchNotFoundShown = true; + vscode.window.showErrorMessage( + 'GLITCH: Rego query files not found. Run from GLITCH project root or reinstall.'); + } + } else if (err.message.includes('Rego library is not available')) { + if (!glitchNotFoundShown) { + glitchNotFoundShown = true; + vscode.window.showErrorMessage( + 'GLITCH: Rego shared library missing. See README for build instructions.'); + } + } else { + const errorParts = err.message.split('Error:'); + const errorMessage = + errorParts.length > 1 ? errorParts[1].trim() : err.message; + vscode.window.showErrorMessage(`GLITCH: ${errorMessage}`); + } + } + glitchDiagnostics.set(doc.uri, []); + } } -export function subscribeToDocumentChanges(context: vscode.ExtensionContext, - glitchDiagnostics: vscode.DiagnosticCollection): void { - - if (vscode.window.activeTextEditor) { - refreshDiagnostics(vscode.window.activeTextEditor.document, - glitchDiagnostics); - } - context.subscriptions.push( - vscode.window.onDidChangeActiveTextEditor(editor => { - if (editor) { - refreshDiagnostics(editor.document, - glitchDiagnostics); - } - }) - ); - - context.subscriptions.push( - vscode.workspace.onDidChangeTextDocument(e => - refreshDiagnostics(e.document, glitchDiagnostics)) - ); - - context.subscriptions.push( - vscode.workspace.onDidCloseTextDocument(doc => glitchDiagnostics.delete(doc.uri)) - ); +export function subscribeToDocumentChanges( + context: vscode.ExtensionContext, + glitchDiagnostics: vscode.DiagnosticCollection): void { + let cancellationTokenSource: vscode.CancellationTokenSource|undefined; + + function triggerRefresh(doc: vscode.TextDocument) { + if (debounceTimer) { + clearTimeout(debounceTimer); + } + + if (cancellationTokenSource) { + cancellationTokenSource.cancel(); + cancellationTokenSource.dispose(); + } + + cancellationTokenSource = new vscode.CancellationTokenSource(); + const token = cancellationTokenSource.token; + + debounceTimer = setTimeout(() => { + refreshDiagnostics(doc, glitchDiagnostics, token); + }, DEBOUNCE_MS); + } + + if (vscode.window.activeTextEditor) { + triggerRefresh(vscode.window.activeTextEditor.document); + } + + context.subscriptions.push( + vscode.window.onDidChangeActiveTextEditor(editor => { + if (editor) { + triggerRefresh(editor.document); + } + })); + + context.subscriptions.push(vscode.workspace.onDidChangeTextDocument(e => { + triggerRefresh(e.document); + })); + + context.subscriptions.push(vscode.workspace.onDidCloseTextDocument(doc => { + glitchDiagnostics.delete(doc.uri); + })); + + context.subscriptions.push({ + dispose: () => { + if (debounceTimer) { + clearTimeout(debounceTimer); + } + if (cancellationTokenSource) { + cancellationTokenSource.cancel(); + cancellationTokenSource.dispose(); + } + if (currentProcess && !currentProcess.killed) { + currentProcess.kill(); + } + } + }); } \ No newline at end of file