⚡️ Speed up function validate_parse_dates_presence by 2,303%

[codeflash-ai]

📄 2,303% (23.03x) speedup for validate_parse_dates_presence in pandas/io/parsers/base_parser.py

⏱️ Runtime : 21.9 milliseconds → 909 microseconds (best of 93 runs)

📝 Explanation and details

The optimization replaces O(n) sequence lookups with O(1) set lookups by converting the columns parameter to a set once at the beginning of the function.

Key Change:

• Added columns_set = set(columns) and replaced all col not in columns and col in columns checks with col not in columns_set and col in columns_set.

Why This Works:
In Python, checking membership in a list/sequence requires scanning through elements linearly (O(n) time complexity), while set membership checks use hash lookups (O(1) average time complexity). The original code performed up to two membership checks per iteration in the parse_dates loop, making it O(n×m) where n is the number of columns and m is the number of parse_dates entries.

Performance Impact:
The line profiler shows the critical bottleneck was line if col not in columns taking 79.1% of total runtime (21.8ms out of 27.6ms). After optimization, the equivalent check takes only 20.1% of total runtime (1.27ms out of 6.32ms) - a 17x improvement on the hottest line.

Test Case Analysis:

• Small datasets show modest 4-28% slowdowns due to set conversion overhead
• Large datasets show dramatic speedups: 2788% faster for 1000 columns, 7532% faster with duplicates
• The optimization shines when len(columns) is large, which is common in real pandas data parsing workflows

Context Impact:
Based on the function reference, this is called from _set_noconvert_dtype_columns during pandas CSV parsing when parse_dates is specified. Since CSV files often have many columns and this function validates parse_dates early in the parsing pipeline, the O(1) lookup optimization significantly improves parser initialization time for wide datasets.

✅ Correctness verification report:

Test	Status
⚙️ Existing Unit Tests	🔘 None Found
🌀 Generated Regression Tests	✅ 51 Passed
⏪ Replay Tests	🔘 None Found
🔎 Concolic Coverage Tests	🔘 None Found
📊 Tests Coverage	100.0%

🌀 Generated Regression Tests and Runtime

# imports
import pytest
from pandas.io.parsers.base_parser import validate_parse_dates_presence

# unit tests

# 1. Basic Test Cases


def test_parse_dates_false_returns_empty_set():
    # parse_dates is False, should return empty set
    codeflash_output = validate_parse_dates_presence(
        False, ["a", "b", "c"]
    )  # 961ns -> 906ns (6.07% faster)


def test_parse_dates_true_returns_empty_set():
    # parse_dates is True, should return empty set
    codeflash_output = validate_parse_dates_presence(
        True, ["a", "b", "c"]
    )  # 663ns -> 692ns (4.19% slower)


def test_parse_dates_list_with_existing_columns():
    # parse_dates is a list of column names, all present
    codeflash_output = validate_parse_dates_presence(["a", "b"], ["a", "b", "c"])
    result = codeflash_output  # 1.41μs -> 1.86μs (24.1% slower)


def test_parse_dates_list_with_duplicates():
    # Duplicates in parse_dates list should be deduplicated in the result set
    codeflash_output = validate_parse_dates_presence(["a", "a", "b"], ["a", "b", "c"])
    result = codeflash_output  # 1.63μs -> 1.70μs (4.46% slower)


def test_parse_dates_list_with_non_str_int_index():
    # parse_dates contains an integer index, which is present in columns
    codeflash_output = validate_parse_dates_presence([1], [0, 1, 2])
    result = codeflash_output  # 1.34μs -> 1.70μs (21.1% slower)


def test_parse_dates_list_mixed_str_and_int():
    # parse_dates contains both string and integer, all present
    codeflash_output = validate_parse_dates_presence(["a", 1], ["a", 1, "b"])
    result = codeflash_output  # 1.53μs -> 1.87μs (18.0% slower)


def test_parse_dates_list_with_tuple_column():
    # parse_dates contains a tuple column, which is present
    codeflash_output = validate_parse_dates_presence([("x", "y")], ["a", ("x", "y")])
    result = codeflash_output  # 1.42μs -> 1.70μs (16.5% slower)


# 2. Edge Test Cases


def test_parse_dates_empty_list():
    # parse_dates is an empty list, should return empty set
    codeflash_output = validate_parse_dates_presence(
        [], ["a", "b"]
    )  # 856ns -> 1.20μs (28.9% slower)


def test_parse_dates_list_all_missing():
    # All columns in parse_dates are missing, should raise ValueError
    with pytest.raises(ValueError) as excinfo:
        validate_parse_dates_presence(
            ["x", "y"], ["a", "b"]
        )  # 3.34μs -> 3.30μs (1.24% faster)


def test_parse_dates_list_some_missing():
    # Some columns in parse_dates are missing, should raise ValueError with only missing ones
    with pytest.raises(ValueError) as excinfo:
        validate_parse_dates_presence(
            ["a", "x"], ["a", "b"]
        )  # 2.94μs -> 2.98μs (1.51% slower)


def test_parse_dates_list_with_non_str_non_int():
    # parse_dates contains an object not present in columns, but as index
    columns = ["a", "b", (1, 2)]
    codeflash_output = validate_parse_dates_presence([(1, 2)], columns)
    result = codeflash_output  # 1.51μs -> 1.87μs (19.3% slower)


def test_parse_dates_list_with_index_lookup():
    # parse_dates contains an int not in columns, but columns is long enough to index
    columns = ["a", "b", "c"]
    # 2 is not in columns, so columns[2] is 'c'
    codeflash_output = validate_parse_dates_presence([2], columns)
    result = codeflash_output  # 1.38μs -> 1.70μs (18.5% slower)


def test_parse_dates_list_with_index_out_of_range():
    # parse_dates contains an int not in columns, but out of range, should raise IndexError
    columns = ["a", "b"]
    with pytest.raises(IndexError):
        validate_parse_dates_presence([3], columns)  # 1.69μs -> 1.84μs (8.41% slower)


def test_parse_dates_list_with_non_string_column_missing():
    # parse_dates contains a string not in columns, and a non-string present
    columns = [0, 1, 2]
    with pytest.raises(ValueError) as excinfo:
        validate_parse_dates_presence(
            ["x", 1], columns
        )  # 3.43μs -> 3.57μs (3.92% slower)


def test_parse_dates_list_with_empty_columns():
    # columns is empty, any string in parse_dates should raise ValueError
    with pytest.raises(ValueError):
        validate_parse_dates_presence(["a"], [])  # 2.59μs -> 2.83μs (8.42% slower)


def test_parse_dates_list_with_column_names_as_numbers():
    # columns are numbers, parse_dates is string, should raise ValueError
    with pytest.raises(ValueError):
        validate_parse_dates_presence(
            ["1"], [1, 2, 3]
        )  # 2.67μs -> 2.92μs (8.51% slower)


def test_parse_dates_list_with_column_names_as_booleans():
    # columns are booleans, parse_dates is boolean True, should return empty set
    codeflash_output = validate_parse_dates_presence(
        True, [True, False]
    )  # 825ns -> 852ns (3.17% slower)


def test_parse_dates_list_with_column_names_as_booleans_and_str():
    # columns are booleans, parse_dates is string 'True', should raise ValueError
    with pytest.raises(ValueError):
        validate_parse_dates_presence(
            ["True"], [True, False]
        )  # 2.64μs -> 2.85μs (7.43% slower)


def test_parse_dates_list_with_column_names_as_none():
    # columns contains None, parse_dates references None
    codeflash_output = validate_parse_dates_presence([None], ["a", None])
    result = codeflash_output  # 1.29μs -> 1.74μs (25.8% slower)


def test_parse_dates_list_with_index_as_none():
    # parse_dates contains None, columns is long enough, None is not in columns, so columns[None] raises TypeError
    columns = ["a", "b"]
    with pytest.raises(TypeError):
        validate_parse_dates_presence(
            [None], columns
        )  # 2.02μs -> 2.25μs (10.5% slower)


def test_parse_dates_list_with_negative_index():
    # parse_dates contains negative index, which should select from columns
    columns = ["a", "b", "c"]
    codeflash_output = validate_parse_dates_presence([-1], columns)
    result = codeflash_output  # 1.48μs -> 1.73μs (14.6% slower)


def test_parse_dates_list_with_large_negative_index():
    # parse_dates contains negative index out of range, should raise IndexError
    columns = ["a", "b"]
    with pytest.raises(IndexError):
        validate_parse_dates_presence([-3], columns)  # 1.67μs -> 1.86μs (10.5% slower)


# 3. Large Scale Test Cases


def test_parse_dates_large_list_all_present():
    # Large number of columns and parse_dates, all present
    columns = [f"col{i}" for i in range(1000)]
    parse_dates = [f"col{i}" for i in range(500, 1000)]
    codeflash_output = validate_parse_dates_presence(parse_dates, columns)
    result = codeflash_output  # 2.55ms -> 88.5μs (2788% faster)


def test_parse_dates_large_list_some_missing():
    # Large number of columns, some parse_dates missing
    columns = [f"col{i}" for i in range(1000)]
    parse_dates = [f"col{i}" for i in range(990, 1010)]
    with pytest.raises(ValueError) as excinfo:
        validate_parse_dates_presence(
            parse_dates, columns
        )  # 118μs -> 35.2μs (235% faster)
    # Should mention all missing columns
    for i in range(1000, 1010):
        pass


def test_parse_dates_large_list_with_indices():
    # parse_dates contains many indices, all in range
    columns = [f"col{i}" for i in range(1000)]
    parse_dates = list(range(900, 1000))
    codeflash_output = validate_parse_dates_presence(parse_dates, columns)
    result = codeflash_output  # 643μs -> 48.8μs (1220% faster)


def test_parse_dates_large_list_with_mixed_types():
    # parse_dates contains both indices and names, all present
    columns = [f"col{i}" for i in range(1000)]
    parse_dates = [f"col{i}" for i in range(990, 1000)] + list(range(980, 990))
    codeflash_output = validate_parse_dates_presence(parse_dates, columns)
    result = codeflash_output  # 132μs -> 40.6μs (227% faster)
    # Should deduplicate if overlap
    expected = set(f"col{i}" for i in range(980, 1000))


def test_parse_dates_large_list_with_duplicate_names():
    # parse_dates contains many duplicates, should deduplicate
    columns = [f"col{i}" for i in range(1000)]
    parse_dates = ["col999"] * 1000
    codeflash_output = validate_parse_dates_presence(parse_dates, columns)
    result = codeflash_output  # 6.81ms -> 89.2μs (7532% faster)


def test_parse_dates_large_list_with_all_indices_out_of_range():
    # parse_dates contains indices all out of range, should raise IndexError
    columns = [f"col{i}" for i in range(100)]
    parse_dates = list(range(100, 110))
    with pytest.raises(IndexError):
        validate_parse_dates_presence(
            parse_dates, columns
        )  # 2.51μs -> 5.46μs (54.0% slower)


# codeflash_output is used to check that the output of the original code is the same as that of the optimized code.

# imports
import pytest  # used for our unit tests
from pandas.io.parsers.base_parser import validate_parse_dates_presence

# unit tests

# ----------- BASIC TEST CASES -----------


def test_parse_dates_false_returns_empty():
    # parse_dates is False, should return empty set
    codeflash_output = validate_parse_dates_presence(
        False, ["a", "b", "c"]
    )  # 643ns -> 703ns (8.53% slower)


def test_parse_dates_true_returns_empty():
    # parse_dates is True, should return empty set
    codeflash_output = validate_parse_dates_presence(
        True, ["a", "b", "c"]
    )  # 632ns -> 635ns (0.472% slower)


def test_parse_dates_empty_list_returns_empty():
    # parse_dates is empty list, should return empty set
    codeflash_output = validate_parse_dates_presence(
        [], ["a", "b", "c"]
    )  # 909ns -> 1.24μs (26.6% slower)


def test_parse_dates_single_str_present():
    # parse_dates is list with a single string present in columns
    codeflash_output = validate_parse_dates_presence(
        ["a"], ["a", "b", "c"]
    )  # 1.25μs -> 1.56μs (19.9% slower)


def test_parse_dates_multiple_str_present():
    # parse_dates is list with multiple strings present in columns
    codeflash_output = validate_parse_dates_presence(
        ["a", "b"], ["a", "b", "c"]
    )  # 1.44μs -> 1.63μs (11.6% slower)


def test_parse_dates_mixed_types_present():
    # parse_dates contains both str and int, both present in columns
    codeflash_output = validate_parse_dates_presence(
        ["a", 1], ["a", 1, "b"]
    )  # 1.53μs -> 1.92μs (20.1% slower)


def test_parse_dates_duplicate_entries():
    # parse_dates contains duplicate entries; result should be unique
    codeflash_output = validate_parse_dates_presence(
        ["a", "a", "b"], ["a", "b", "c"]
    )  # 1.81μs -> 2.12μs (14.5% slower)


def test_parse_dates_all_missing():
    # All columns in parse_dates are missing
    with pytest.raises(ValueError) as excinfo:
        validate_parse_dates_presence(
            ["x", "y"], ["a", "b", "c"]
        )  # 3.43μs -> 3.42μs (0.058% faster)


def test_parse_dates_some_missing_some_present():
    # Some columns present, some missing
    with pytest.raises(ValueError) as excinfo:
        validate_parse_dates_presence(
            ["a", "x", "b", "y"], ["a", "b", "c"]
        )  # 3.35μs -> 3.42μs (2.13% slower)


# ----------- EDGE TEST CASES -----------


def test_parse_dates_empty_columns():
    # columns is empty, any non-empty parse_dates should raise
    with pytest.raises(ValueError) as excinfo:
        validate_parse_dates_presence(["a"], [])  # 2.52μs -> 2.69μs (6.18% slower)


def test_parse_dates_empty_list_and_empty_columns():
    # Both are empty, should return empty set
    codeflash_output = validate_parse_dates_presence(
        [], []
    )  # 868ns -> 1.13μs (23.0% slower)


def test_parse_dates_non_str_hashable():
    # parse_dates contains a tuple, present in columns
    codeflash_output = validate_parse_dates_presence(
        [(1, 2)], ["a", (1, 2), "b"]
    )  # 1.44μs -> 1.85μs (22.2% slower)


def test_parse_dates_str_and_non_str_types():
    # parse_dates contains str and tuple, both present
    codeflash_output = validate_parse_dates_presence(
        ["a", (1, 2)], ["a", (1, 2), "b"]
    )  # 1.92μs -> 2.26μs (15.2% slower)


def test_parse_dates_str_and_non_str_types_missing():
    # parse_dates contains str (missing) and tuple (present)
    with pytest.raises(ValueError) as excinfo:
        validate_parse_dates_presence(
            ["x", (1, 2)], ["a", (1, 2), "b"]
        )  # 3.47μs -> 3.51μs (1.25% slower)


def test_parse_dates_str_and_int_index():
    # parse_dates contains int, which is present as column index
    codeflash_output = validate_parse_dates_presence(
        [0, "a"], [0, "a", "b"]
    )  # 1.63μs -> 1.80μs (9.65% slower)


def test_parse_dates_large_list_all_present():
    # Large parse_dates list, all present in columns
    columns = [f"col{i}" for i in range(1000)]
    parse_dates = [f"col{i}" for i in range(1000)]
    codeflash_output = validate_parse_dates_presence(parse_dates, columns)
    result = codeflash_output  # 3.34ms -> 124μs (2587% faster)


def test_parse_dates_large_list_some_missing():
    # Large parse_dates list, some missing in columns
    columns = [f"col{i}" for i in range(900)]
    parse_dates = [f"col{i}" for i in range(1000)]
    with pytest.raises(ValueError) as excinfo:
        validate_parse_dates_presence(
            parse_dates, columns
        )  # 3.32ms -> 123μs (2576% faster)
    # Should list all missing columns
    missing_cols = [f"col{i}" for i in range(900, 1000)]
    for col in missing_cols:
        pass


def test_parse_dates_large_list_duplicates():
    # Large parse_dates list, with duplicates, all present
    columns = [f"col{i}" for i in range(500)]
    parse_dates = [f"col{i}" for i in range(500)] * 2  # duplicates
    codeflash_output = validate_parse_dates_presence(parse_dates, columns)
    result = codeflash_output  # 1.59ms -> 90.6μs (1653% faster)


def test_parse_dates_large_columns_small_parse_dates():
    # Large columns, small parse_dates, all present
    columns = [f"col{i}" for i in range(1000)]
    parse_dates = ["col1", "col500", "col999"]
    codeflash_output = validate_parse_dates_presence(parse_dates, columns)
    result = codeflash_output  # 11.4μs -> 37.4μs (69.4% slower)


def test_parse_dates_large_columns_empty_parse_dates():
    # Large columns, empty parse_dates
    columns = [f"col{i}" for i in range(1000)]
    parse_dates = []
    codeflash_output = validate_parse_dates_presence(parse_dates, columns)
    result = codeflash_output  # 867ns -> 35.6μs (97.6% slower)


def test_parse_dates_large_columns_large_parse_dates_some_missing():
    # Large columns and parse_dates, some missing
    columns = [f"col{i}" for i in range(1000)]
    parse_dates = [f"col{i}" for i in range(990)] + [
        "missing1",
        "missing2",
        "missing3",
        "missing4",
        "missing5",
        "missing6",
        "missing7",
        "missing8",
        "missing9",
        "missing10",
    ]
    with pytest.raises(ValueError) as excinfo:
        validate_parse_dates_presence(
            parse_dates, columns
        )  # 3.27ms -> 111μs (2829% faster)
    for col in [f"missing{i}" for i in range(1, 11)]:
        pass


# ----------- MISCELLANEOUS TESTS -----------


def test_parse_dates_column_names_are_ints():
    # columns are ints, parse_dates are ints
    columns = list(range(10))
    parse_dates = [1, 3, 5]
    codeflash_output = validate_parse_dates_presence(parse_dates, columns)
    result = codeflash_output  # 1.75μs -> 2.27μs (23.2% slower)


def test_parse_dates_column_names_are_mixed_types():
    # columns are mixed types, parse_dates are mixed types
    columns = ["a", 1, (2, 3)]
    parse_dates = ["a", 1, (2, 3)]
    codeflash_output = validate_parse_dates_presence(parse_dates, columns)
    result = codeflash_output  # 1.91μs -> 2.07μs (7.88% slower)

To edit these changes git checkout codeflash/optimize-validate_parse_dates_presence-mj9ubxlk and push.