From 653cbe6fa0904fab95d11800b1fc8bfeff4ca3e3 Mon Sep 17 00:00:00 2001 From: Manon Goo Date: Mon, 23 Mar 2026 02:11:15 +0000 Subject: [PATCH 1/4] Fix encrypted-password anonymization to replace instead of scrub Changes encrypted-password handling from line-scrubbing (sensitive_item_num=None) to targeted replacement (capture group), preserving line context. Adds SHA-256 hash format support alongside existing SHA-512, and $5$/$6$ catch-all regexes. New test files to avoid merge conflicts with other feature branches: - tests/unit/test_encrypted_password.py (hash verification tests) - tests/end_to_end/test_e2e_encrypted_password.py (e2e test) --- netconan/default_pwd_regexes.py | 2 +- netconan/sensitive_item_removal.py | 12 ++++- .../end_to_end/test_e2e_encrypted_password.py | 33 ++++++++++++++ tests/unit/test_encrypted_password.py | 45 +++++++++++++++++++ tests/unit/test_sensitive_item_removal.py | 36 ++++++++++++--- 5 files changed, 119 insertions(+), 9 deletions(-) create mode 100644 tests/end_to_end/test_e2e_encrypted_password.py create mode 100644 tests/unit/test_encrypted_password.py diff --git a/netconan/default_pwd_regexes.py b/netconan/default_pwd_regexes.py index 02c81ae..2bb5303 100644 --- a/netconan/default_pwd_regexes.py +++ b/netconan/default_pwd_regexes.py @@ -130,7 +130,7 @@ # (to make sure the regex handles different syntaxes allowed in the line) [(r"(\S* )*md5 \d+ key [^ ;]+(.*)", None)], [(r"(\S* )*(secret|simple-password) [^ ;]+(.*)", None)], - [(r"(\S* )*encrypted-password [^ ;]+(.*)", None)], + [(r"(?P(\S* )*encrypted-password )([^ ;]+)", 3)], [(r"(\S* )*ssh-(rsa|dsa) \"(.*)", None)], [(r"(\S* )*((pre-shared-|)key (ascii-text|hexadecimal)) [^ ;]+(.*)", None)], ] diff --git a/netconan/sensitive_item_removal.py b/netconan/sensitive_item_removal.py index 7fac3be..c26b3b3 100644 --- a/netconan/sensitive_item_removal.py +++ b/netconan/sensitive_item_removal.py @@ -23,7 +23,7 @@ from hashlib import md5 # Using passlib for digests not supported by hashlib -from passlib.hash import cisco_type7, md5_crypt, sha512_crypt +from passlib.hash import cisco_type7, md5_crypt, sha256_crypt, sha512_crypt from netconan.utils import juniper_secrets @@ -75,7 +75,7 @@ # These are extra regexes to find lines that seem like they might contain # sensitive info (these are not already caught by RANCID default regexes) extra_password_regexes = [ - [(r"(?<=encrypted-password )(\S+)", None)], + [(r"(?<=encrypted-password )(\S+)", 1)], [(r'(?<=key ")([^"]+)', 1)], [(r"(?<=key-hash sha256 )(\S+)", 1)], # Replace communities that do not look like well-known BGP communities @@ -86,6 +86,8 @@ # Catch-all's matching what looks like hashed passwords [(r'("?\$9\$[^\s;"]+)', 1)], [(r'("?\$1\$[^\s;"]+)', 1)], + [(r'("?\$5\$[^\s;"]+)', 1)], + [(r'("?\$6\$[^\s;"]+)', 1)], ] @@ -221,6 +223,7 @@ class _sensitive_item_formats(Enum): text = 5 sha512 = 6 juniper_type9 = 7 + sha256 = 8 def anonymize_as_numbers(anonymizer, line): @@ -280,6 +283,9 @@ def _anonymize_value(raw_val, lookup, reserved_words, salt): # identify anonymized lines anon_val = md5_crypt.using(salt="0" * old_salt_size).hash(anon_val) + if item_format == _sensitive_item_formats.sha256: + anon_val = sha256_crypt.using(rounds=5000).hash(anon_val) + if item_format == _sensitive_item_formats.sha512: # Hash anon_val w/standard rounds=5000 to omit rounds parameter from hash output anon_val = sha512_crypt.using(rounds=5000).hash(anon_val) @@ -302,6 +308,8 @@ def _check_sensitive_item_format(val): # specific format so it should override hex or text) if re.match(r"^\$9\$[\S]+$", val): item_format = _sensitive_item_formats.juniper_type9 + if re.match(r"^\$5\$[\S]+$", val): + item_format = _sensitive_item_formats.sha256 if re.match(r"^\$6\$[\S]+$", val): item_format = _sensitive_item_formats.sha512 if re.match(r"^\$1\$[\S]+\$[\S]+$", val): diff --git a/tests/end_to_end/test_e2e_encrypted_password.py b/tests/end_to_end/test_e2e_encrypted_password.py new file mode 100644 index 0000000..d5fec62 --- /dev/null +++ b/tests/end_to_end/test_e2e_encrypted_password.py @@ -0,0 +1,33 @@ +"""End-to-end tests for encrypted-password anonymization.""" + +import re + +from netconan.netconan import main + + +def test_end_to_end_encrypted_password_sha512(tmpdir): + """Test that encrypted-password with $6$ hash is anonymized, not scrubbed.""" + filename = "test.txt" + # Hash of "netconanExamplePassword" using sha512_crypt + original_hash = "$6$DOphiwNHNVLzCXmR$4sS7hYY6UPAnX6oXU9rIbCqKgTJBf9wJ4Hf2sz7HYPjH7Wrn9II1vS0wdHtirRHv1YACC.E.YDlaUb9U8ysvk0" + input_line = 'set system root-authentication encrypted-password "{}"\n'.format( + original_hash + ) + + input_dir = tmpdir.mkdir("input") + input_dir.join(filename).write(input_line) + + output_dir = tmpdir.mkdir("output") + args = ["-s", "TESTSALT", "-p", "-i", str(input_dir), "-o", str(output_dir)] + main(args) + + with open(str(output_dir.join(filename))) as f: + output = f.read() + + # Original hash must not appear in output + assert original_hash not in output + # Line must not be scrubbed + assert "SCRUBBED" not in output + # Context must be preserved and output must contain a $6$ hash + assert "encrypted-password" in output + assert re.search(r'\$6\$[^\s"]+\$[^\s"]+', output) diff --git a/tests/unit/test_encrypted_password.py b/tests/unit/test_encrypted_password.py new file mode 100644 index 0000000..8f366c1 --- /dev/null +++ b/tests/unit/test_encrypted_password.py @@ -0,0 +1,45 @@ +"""Tests for encrypted-password anonymization (hash verification).""" + +import pytest + +from netconan.sensitive_item_removal import _anonymize_value + +SALT = "saltForTest" + + +@pytest.mark.parametrize( + "original_val, hash_module", + [ + ( + # Hash of "netconanExamplePassword" using sha256_crypt + "$5$dyjYlf.RKgW5cjA5$5OmkZF/RpklPYw8oC9k8nxKIh0RzyNmx74zPJ1CRuz8", + "sha256_crypt", + ), + ( + # Hash of "netconanExamplePassword" using sha512_crypt + "$6$DOphiwNHNVLzCXmR$4sS7hYY6UPAnX6oXU9rIbCqKgTJBf9wJ4Hf2sz7HYPjH7Wrn9II1vS0wdHtirRHv1YACC.E.YDlaUb9U8ysvk0", + "sha512_crypt", + ), + ( + "$1$CNANTest$xAfu6Am1d5D/.6OVICuOu/", + "md5_crypt", + ), + ], +) +def test__anonymize_value_produces_verifiable_hash(original_val, hash_module): + """Test that anonymized crypt hashes are verifiable against their plaintext.""" + from passlib.hash import md5_crypt, sha256_crypt, sha512_crypt + + hash_modules = { + "md5_crypt": md5_crypt, + "sha256_crypt": sha256_crypt, + "sha512_crypt": sha512_crypt, + } + pwd_lookup = {} + anon_val = _anonymize_value(original_val, pwd_lookup, {}, SALT) + + # _anonymize_value generates "netconanRemoved0" as the plaintext (first + # entry in an empty lookup) and hashes it. Verify the hash is valid. + plaintext = "netconanRemoved0" + hasher = hash_modules[hash_module] + assert hasher.verify(plaintext, anon_val) diff --git a/tests/unit/test_sensitive_item_removal.py b/tests/unit/test_sensitive_item_removal.py index 03b7376..b9f51ed 100644 --- a/tests/unit/test_sensitive_item_removal.py +++ b/tests/unit/test_sensitive_item_removal.py @@ -37,7 +37,8 @@ arista_password_lines = [ ( "username noc secret sha512 {}", - "$6$RMxgK5ALGIf.nWEC$tHuKCyfNtJMCY561P52dTzHUmYMmLxb/Mxik.j3vMUs8lMCPocM00/NAS.SN6GCWx7d/vQIgxnClyQLAb7n3x0", + # Hash of "netconanExamplePassword" using sha512_crypt + "$6$DOphiwNHNVLzCXmR$4sS7hYY6UPAnX6oXU9rIbCqKgTJBf9wJ4Hf2sz7HYPjH7Wrn9II1vS0wdHtirRHv1YACC.E.YDlaUb9U8ysvk0", ), (" vrrp 2 authentication text {}", "RemoveMe"), ] @@ -197,6 +198,21 @@ ), ("set snmp community {} authorization read-only", "SECRETTEXT"), ("set snmp trap-group {} otherstuff", "SECRETTEXT"), + ( + 'set system root-authentication encrypted-password "{}"', + # Hash of "netconanExamplePassword" using sha512_crypt + "$6$DOphiwNHNVLzCXmR$4sS7hYY6UPAnX6oXU9rIbCqKgTJBf9wJ4Hf2sz7HYPjH7Wrn9II1vS0wdHtirRHv1YACC.E.YDlaUb9U8ysvk0", + ), + ( + 'set system login user admin authentication encrypted-password "{}"', + # Hash of "netconanExamplePassword" using sha512_crypt + "$6$DOphiwNHNVLzCXmR$4sS7hYY6UPAnX6oXU9rIbCqKgTJBf9wJ4Hf2sz7HYPjH7Wrn9II1vS0wdHtirRHv1YACC.E.YDlaUb9U8ysvk0", + ), + ( + 'encrypted-password "{}";', + # Hash of "netconanExamplePassword" using sha512_crypt + "$6$DOphiwNHNVLzCXmR$4sS7hYY6UPAnX6oXU9rIbCqKgTJBf9wJ4Hf2sz7HYPjH7Wrn9II1vS0wdHtirRHv1YACC.E.YDlaUb9U8ysvk0", + ), ("key hexadecimal {}", "ABCDEF123456"), ( 'authentication-key "{}";', @@ -303,7 +319,13 @@ _sensitive_item_formats.juniper_type9, ), ( - "$6$RMxgK5ALGIf.nWEC$tHuKCyfNtJMCY561P52dTzHUmYMmLxb/Mxik.j3vMUs8lMCPocM00/NAS.SN6GCWx7d/vQIgxnClyQLAb7n3x0", + # Hash of "netconanExamplePassword" using sha256_crypt + "$5$dyjYlf.RKgW5cjA5$5OmkZF/RpklPYw8oC9k8nxKIh0RzyNmx74zPJ1CRuz8", + _sensitive_item_formats.sha256, + ), + ( + # Hash of "netconanExamplePassword" using sha512_crypt + "$6$DOphiwNHNVLzCXmR$4sS7hYY6UPAnX6oXU9rIbCqKgTJBf9wJ4Hf2sz7HYPjH7Wrn9II1vS0wdHtirRHv1YACC.E.YDlaUb9U8ysvk0", _sensitive_item_formats.sha512, ), ] @@ -327,6 +349,8 @@ "PasswordThree", "$9$HqfQ1IcrK8n/t0IcvM24aZGi6/t", "$1$CNANTest$xAfu6Am1d5D/.6OVICuOu/", + # Hash of "netconanExamplePassword" using sha256_crypt + "$5$dyjYlf.RKgW5cjA5$5OmkZF/RpklPYw8oC9k8nxKIh0RzyNmx74zPJ1CRuz8", "$6$NQJRTiqxZiNR0aWI$hU1EPleWl6wGcMtDxaMEqNhN8WnxEqmeFjWC5h8oh5USSn5P9ZgFXbf2giO8nEtM.yBXO3O6b.76LQ1zlmG3B0", ] @@ -680,13 +704,13 @@ def test_pwd_removal_preserve_trailing_whitespace(regexes, whitespace): @pytest.mark.parametrize("whitespace", [" ", "\t", "\n", " \t\n"]) -def test_line_scrub_preserve_trailing_whitespace(regexes, whitespace): - """Test trailing whitespace is preserved when line is scrubbed (encrypted-password case).""" - # encrypted-password triggers line scrubbing (sensitive_item_num=None) +def test_encrypted_password_preserve_trailing_whitespace(regexes, whitespace): + """Test trailing whitespace is preserved when encrypted-password is anonymized.""" config_line = " encrypted-password SECRET{}".format(whitespace) pwd_lookup = {} processed_line = replace_matching_item(regexes, config_line, pwd_lookup, SALT) - assert _LINE_SCRUBBED_MESSAGE in processed_line + assert "SECRET" not in processed_line + assert _LINE_SCRUBBED_MESSAGE not in processed_line assert processed_line.endswith(whitespace) From fb92f404b3332d590acd57e806818ed4c94235d8 Mon Sep 17 00:00:00 2001 From: Manon Goo Date: Mon, 23 Mar 2026 13:22:33 +0000 Subject: [PATCH 2/4] Move encrypted-password regex to avoid merge conflict with SSH key branch Add improved encrypted-password regex (with capture group) before the JUNOS comment block. The original scrub-only version is left in place as dead code so the JUNOS section stays untouched, avoiding merge conflicts with the SSH key branch that modifies the adjacent line. --- netconan/default_pwd_regexes.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/netconan/default_pwd_regexes.py b/netconan/default_pwd_regexes.py index 2bb5303..3558770 100644 --- a/netconan/default_pwd_regexes.py +++ b/netconan/default_pwd_regexes.py @@ -121,6 +121,10 @@ [(r"(message-digest-key \d+ md5 (7|encrypted)) (.*)", None)], [(r"(.*?neighbor.*?) (\S*) password (.*)", None)], [(r"(wlccp \S+ username (\S+)( .*)? password( \d)?) (\S+)(.*)", None)], + # Juniper encrypted-password with capture group (replaces scrub-only version below). + # TODO: delete the original encrypted-password scrubber in the JUNOS section + # below; it is left intact for now to avoid merge conflicts. + [(r"(?P(\S* )*encrypted-password )([^ ;]+)", 3)], # These are regexes for JUNOS # TODO(https://github.com/intentionet/netconan/issues/4): # Follow-up on these. They were modified from RANCID's regexes and currently: @@ -130,7 +134,7 @@ # (to make sure the regex handles different syntaxes allowed in the line) [(r"(\S* )*md5 \d+ key [^ ;]+(.*)", None)], [(r"(\S* )*(secret|simple-password) [^ ;]+(.*)", None)], - [(r"(?P(\S* )*encrypted-password )([^ ;]+)", 3)], + [(r"(\S* )*encrypted-password [^ ;]+(.*)", None)], [(r"(\S* )*ssh-(rsa|dsa) \"(.*)", None)], [(r"(\S* )*((pre-shared-|)key (ascii-text|hexadecimal)) [^ ;]+(.*)", None)], ] From b2902152b6854edd98dd2f8626b49b8a0c9ee44c Mon Sep 17 00:00:00 2001 From: Manon Goo Date: Mon, 23 Mar 2026 02:11:15 +0000 Subject: [PATCH 3/4] Fix encrypted-password anonymization to replace instead of scrub Changes encrypted-password handling from line-scrubbing (sensitive_item_num=None) to targeted replacement (capture group), preserving line context. Adds SHA-256 hash format support alongside existing SHA-512, and $5$/$6$ catch-all regexes. New test files to avoid merge conflicts with other feature branches: - tests/unit/test_encrypted_password.py (hash verification tests) - tests/end_to_end/test_e2e_encrypted_password.py (e2e test) --- netconan/default_pwd_regexes.py | 2 +- netconan/sensitive_item_removal.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/netconan/default_pwd_regexes.py b/netconan/default_pwd_regexes.py index 27f5894..b51d64d 100644 --- a/netconan/default_pwd_regexes.py +++ b/netconan/default_pwd_regexes.py @@ -136,7 +136,7 @@ # (to make sure the regex handles different syntaxes allowed in the line) [(r"(\S* )*md5 \d+ key [^ ;]+(.*)", None)], [(r"(\S* )*(secret|simple-password) [^ ;]+(.*)", None)], - [(r"(\S* )*encrypted-password [^ ;]+(.*)", None)], + [(r"(?P(\S* )*encrypted-password )([^ ;]+)", 3)], [(r"(\S* )*ssh-(rsa|dsa) \"(.*)", None)], [(r"(\S* )*((pre-shared-|)key (ascii-text|hexadecimal)) [^ ;]+(.*)", None)], ] diff --git a/netconan/sensitive_item_removal.py b/netconan/sensitive_item_removal.py index 714eaee..7c59a01 100644 --- a/netconan/sensitive_item_removal.py +++ b/netconan/sensitive_item_removal.py @@ -80,7 +80,7 @@ # These are extra regexes to find lines that seem like they might contain # sensitive info (these are not already caught by RANCID default regexes) -extra_password_regexes = [ +extra_password_regexes: list[list[RegexRule]] = [ [(r"(?<=encrypted-password )(\S+)", 1)], [(r'(?<=key ")([^"]+)', 1)], [(r"(?<=key-hash sha256 )(\S+)", 1)], From f3a8ebd0168cc8d318835b033b1c5ddf7696a69e Mon Sep 17 00:00:00 2001 From: Manon Goo Date: Mon, 23 Mar 2026 21:50:48 +0000 Subject: [PATCH 4/4] Address PR review feedback for encrypted-password - Fix wlccp regex capture group index (None -> 1) - Delete redundant scrub-only encrypted-password regex in JUNOS section, now that the hash-preserving regex handles all encrypted-password lines - Move hash verification tests into test_sensitive_item_removal.py to follow test file naming convention (test_.py) - Delete tests/unit/test_encrypted_password.py --- netconan/default_pwd_regexes.py | 7 +--- tests/unit/test_encrypted_password.py | 45 ----------------------- tests/unit/test_sensitive_item_removal.py | 38 +++++++++++++++++++ 3 files changed, 40 insertions(+), 50 deletions(-) delete mode 100644 tests/unit/test_encrypted_password.py diff --git a/netconan/default_pwd_regexes.py b/netconan/default_pwd_regexes.py index b51d64d..727c71c 100644 --- a/netconan/default_pwd_regexes.py +++ b/netconan/default_pwd_regexes.py @@ -122,10 +122,8 @@ [(r"(key-string \d?)(.*)", None)], [(r"(message-digest-key \d+ md5 (7|encrypted)) (.*)", None)], [(r"(.*?neighbor.*?) (\S*) password (.*)", None)], - [(r"(wlccp \S+ username (\S+)( .*)? password( \d)?) (\S+)(.*)", None)], - # Juniper encrypted-password with capture group (replaces scrub-only version below). - # TODO: delete the original encrypted-password scrubber in the JUNOS section - # below; it is left intact for now to avoid merge conflicts. + [(r"(wlccp \S+ username (\S+)( .*)? password( \d)?) (\S+)(.*)", 1)], + # Juniper encrypted-password with capture group for hash-preserving anonymization. [(r"(?P(\S* )*encrypted-password )([^ ;]+)", 3)], # These are regexes for JUNOS # TODO(https://github.com/intentionet/netconan/issues/4): @@ -136,7 +134,6 @@ # (to make sure the regex handles different syntaxes allowed in the line) [(r"(\S* )*md5 \d+ key [^ ;]+(.*)", None)], [(r"(\S* )*(secret|simple-password) [^ ;]+(.*)", None)], - [(r"(?P(\S* )*encrypted-password )([^ ;]+)", 3)], [(r"(\S* )*ssh-(rsa|dsa) \"(.*)", None)], [(r"(\S* )*((pre-shared-|)key (ascii-text|hexadecimal)) [^ ;]+(.*)", None)], ] diff --git a/tests/unit/test_encrypted_password.py b/tests/unit/test_encrypted_password.py deleted file mode 100644 index 8f366c1..0000000 --- a/tests/unit/test_encrypted_password.py +++ /dev/null @@ -1,45 +0,0 @@ -"""Tests for encrypted-password anonymization (hash verification).""" - -import pytest - -from netconan.sensitive_item_removal import _anonymize_value - -SALT = "saltForTest" - - -@pytest.mark.parametrize( - "original_val, hash_module", - [ - ( - # Hash of "netconanExamplePassword" using sha256_crypt - "$5$dyjYlf.RKgW5cjA5$5OmkZF/RpklPYw8oC9k8nxKIh0RzyNmx74zPJ1CRuz8", - "sha256_crypt", - ), - ( - # Hash of "netconanExamplePassword" using sha512_crypt - "$6$DOphiwNHNVLzCXmR$4sS7hYY6UPAnX6oXU9rIbCqKgTJBf9wJ4Hf2sz7HYPjH7Wrn9II1vS0wdHtirRHv1YACC.E.YDlaUb9U8ysvk0", - "sha512_crypt", - ), - ( - "$1$CNANTest$xAfu6Am1d5D/.6OVICuOu/", - "md5_crypt", - ), - ], -) -def test__anonymize_value_produces_verifiable_hash(original_val, hash_module): - """Test that anonymized crypt hashes are verifiable against their plaintext.""" - from passlib.hash import md5_crypt, sha256_crypt, sha512_crypt - - hash_modules = { - "md5_crypt": md5_crypt, - "sha256_crypt": sha256_crypt, - "sha512_crypt": sha512_crypt, - } - pwd_lookup = {} - anon_val = _anonymize_value(original_val, pwd_lookup, {}, SALT) - - # _anonymize_value generates "netconanRemoved0" as the plaintext (first - # entry in an empty lookup) and hashes it. Verify the hash is valid. - plaintext = "netconanRemoved0" - hasher = hash_modules[hash_module] - assert hasher.verify(plaintext, anon_val) diff --git a/tests/unit/test_sensitive_item_removal.py b/tests/unit/test_sensitive_item_removal.py index b9f51ed..39d09a3 100644 --- a/tests/unit/test_sensitive_item_removal.py +++ b/tests/unit/test_sensitive_item_removal.py @@ -481,6 +481,44 @@ def test__anonymize_value_unique(): unique_anon_vals.add(anon_val) +@pytest.mark.parametrize( + "original_val, hash_module", + [ + ( + # Hash of "netconanExamplePassword" using sha256_crypt + "$5$dyjYlf.RKgW5cjA5$5OmkZF/RpklPYw8oC9k8nxKIh0RzyNmx74zPJ1CRuz8", + "sha256_crypt", + ), + ( + # Hash of "netconanExamplePassword" using sha512_crypt + "$6$DOphiwNHNVLzCXmR$4sS7hYY6UPAnX6oXU9rIbCqKgTJBf9wJ4Hf2sz7HYPjH7Wrn9II1vS0wdHtirRHv1YACC.E.YDlaUb9U8ysvk0", + "sha512_crypt", + ), + ( + "$1$CNANTest$xAfu6Am1d5D/.6OVICuOu/", + "md5_crypt", + ), + ], +) +def test__anonymize_value_produces_verifiable_hash(original_val, hash_module): + """Test that anonymized crypt hashes are verifiable against their plaintext.""" + from passlib.hash import md5_crypt, sha256_crypt, sha512_crypt + + hash_modules = { + "md5_crypt": md5_crypt, + "sha256_crypt": sha256_crypt, + "sha512_crypt": sha512_crypt, + } + pwd_lookup = {} + anon_val = _anonymize_value(original_val, pwd_lookup, {}, SALT) + + # _anonymize_value generates "netconanRemoved0" as the plaintext (first + # entry in an empty lookup) and hashes it. Verify the hash is valid. + plaintext = "netconanRemoved0" + hasher = hash_modules[hash_module] + assert hasher.verify(plaintext, anon_val) + + @pytest.mark.parametrize("val, format_", sensitive_items_and_formats) def test__check_sensitive_item_format(val, format_): """Test sensitive item format detection."""