diff --git a/docs/3recipes/attack_recipes.rst b/docs/3recipes/attack_recipes.rst index 477cb1e3f..874bd885d 100644 --- a/docs/3recipes/attack_recipes.rst +++ b/docs/3recipes/attack_recipes.rst @@ -36,6 +36,7 @@ Attacks on classification models 14. TextBugger (TextBugger: Generating Adversarial Text Against Real-world Applications) 15. Pruthi (Combating Adversarial Misspellings with Robust Word Recognition 2019) 16. CLARE (Contextualized Perturbation for Textual Adversarial Attack 2020) +17. LEAP (LEAP: Efficient and Automated Test Method for NLP Software 2023) @@ -136,7 +137,9 @@ Attacks on classification models :members: :noindex: - +.. automodule:: textattack.attack_recipes.leap_2023 + :members: + :noindex: diff --git a/docs/api/search_methods.rst b/docs/api/search_methods.rst index 979dadcb1..2aed38cb5 100644 --- a/docs/api/search_methods.rst +++ b/docs/api/search_methods.rst @@ -44,3 +44,7 @@ ParticleSwarmOptimization .. autoclass:: textattack.search_methods.ParticleSwarmOptimization :members: +ParticleSwarmOptimizationLEAP +-------------------------- +.. autoclass:: textattack.search_methods.ParticleSwarmOptimizationLEAP + :members: diff --git a/textattack/attack.py b/textattack/attack.py index 7e05f93ec..47537d1b0 100644 --- a/textattack/attack.py +++ b/textattack/attack.py @@ -372,9 +372,9 @@ def filter_transformations( uncached_texts.append(transformed_text) else: # promote transformed_text to the top of the LRU cache - self.constraints_cache[(current_text, transformed_text)] = ( - self.constraints_cache[(current_text, transformed_text)] - ) + self.constraints_cache[ + (current_text, transformed_text) + ] = self.constraints_cache[(current_text, transformed_text)] if self.constraints_cache[(current_text, transformed_text)]: filtered_texts.append(transformed_text) filtered_texts += self._filter_transformations_uncached( diff --git a/textattack/attack_args.py b/textattack/attack_args.py index b99f6dc58..8697ea529 100644 --- a/textattack/attack_args.py +++ b/textattack/attack_args.py @@ -37,6 +37,7 @@ "checklist": "textattack.attack_recipes.CheckList2020", "clare": "textattack.attack_recipes.CLARE2020", "a2t": "textattack.attack_recipes.A2TYoo2021", + "leap": "textattack.attack_recipes.LEAP2023", } diff --git a/textattack/attack_recipes/__init__.py b/textattack/attack_recipes/__init__.py index 6e865ddee..137c19592 100644 --- a/textattack/attack_recipes/__init__.py +++ b/textattack/attack_recipes/__init__.py @@ -39,6 +39,7 @@ from .pso_zang_2020 import PSOZang2020 from .checklist_ribeiro_2020 import CheckList2020 from .clare_li_2020 import CLARE2020 +from .leap_2023 import LEAP2023 from .french_recipe import FrenchRecipe from .spanish_recipe import SpanishRecipe from .chinese_recipe import ChineseRecipe diff --git a/textattack/attack_recipes/leap_2023.py b/textattack/attack_recipes/leap_2023.py new file mode 100644 index 000000000..cc5076284 --- /dev/null +++ b/textattack/attack_recipes/leap_2023.py @@ -0,0 +1,44 @@ +""" + +LEAP +================================== + +(LEAP: Efficient and Automated Test Method for NLP Software) + +""" +from textattack import Attack +from textattack.constraints.pre_transformation import ( + MaxModificationRate, + StopwordModification, +) +from textattack.goal_functions import UntargetedClassification +from textattack.search_methods import ParticleSwarmOptimizationLEAP +from textattack.transformations import WordSwapWordNet + +from .attack_recipe import AttackRecipe + + +class LEAP2023(AttackRecipe): + @staticmethod + def build(model_wrapper): + # + # Swap words with their synonyms extracted based on the WordNet. + # + transformation = WordSwapWordNet() + # + # MaxModificationRate = 0.16 in AG's News + # + constraints = [MaxModificationRate(max_rate=0.16), StopwordModification()] + # + # + # Use untargeted classification for demo, can be switched to targeted one + # + goal_function = UntargetedClassification(model_wrapper) + # + # Perform word substitution with LEAP algorithm. + # + search_method = ParticleSwarmOptimizationLEAP( + pop_size=60, max_iters=20, post_turn_check=True, max_turn_retries=20 + ) + + return Attack(goal_function, constraints, transformation, search_method) diff --git a/textattack/goal_functions/classification/targeted_classification.py b/textattack/goal_functions/classification/targeted_classification.py index 3b1ad3acd..6041b6258 100644 --- a/textattack/goal_functions/classification/targeted_classification.py +++ b/textattack/goal_functions/classification/targeted_classification.py @@ -11,7 +11,7 @@ class TargetedClassification(ClassificationGoalFunction): """A targeted attack on classification models which attempts to maximize the score of the target label. - Complete when the arget label is the predicted label. + Complete when the target label is the predicted label. """ def __init__(self, *args, target_class=0, **kwargs): diff --git a/textattack/metrics/attack_metrics/words_perturbed.py b/textattack/metrics/attack_metrics/words_perturbed.py index 38c11b293..6104de1b3 100644 --- a/textattack/metrics/attack_metrics/words_perturbed.py +++ b/textattack/metrics/attack_metrics/words_perturbed.py @@ -65,9 +65,9 @@ def calculate(self, results): self.all_metrics["avg_word_perturbed"] = self.avg_number_word_perturbed_num() self.all_metrics["avg_word_perturbed_perc"] = self.avg_perturbation_perc() self.all_metrics["max_words_changed"] = self.max_words_changed - self.all_metrics["num_words_changed_until_success"] = ( - self.num_words_changed_until_success - ) + self.all_metrics[ + "num_words_changed_until_success" + ] = self.num_words_changed_until_success return self.all_metrics diff --git a/textattack/search_methods/__init__.py b/textattack/search_methods/__init__.py index b5e262916..8645c474a 100644 --- a/textattack/search_methods/__init__.py +++ b/textattack/search_methods/__init__.py @@ -15,3 +15,4 @@ from .alzantot_genetic_algorithm import AlzantotGeneticAlgorithm from .improved_genetic_algorithm import ImprovedGeneticAlgorithm from .particle_swarm_optimization import ParticleSwarmOptimization +from .particle_swarm_optimization_leap import ParticleSwarmOptimizationLEAP diff --git a/textattack/search_methods/particle_swarm_optimization.py b/textattack/search_methods/particle_swarm_optimization.py index fdc48aa07..639f513bc 100644 --- a/textattack/search_methods/particle_swarm_optimization.py +++ b/textattack/search_methods/particle_swarm_optimization.py @@ -120,9 +120,9 @@ def _turn(self, source_text, target_text, prob, original_text): & indices_to_replace ) if "last_transformation" in source_text.attacked_text.attack_attrs: - new_text.attack_attrs["last_transformation"] = ( - source_text.attacked_text.attack_attrs["last_transformation"] - ) + new_text.attack_attrs[ + "last_transformation" + ] = source_text.attacked_text.attack_attrs["last_transformation"] if not self.post_turn_check or (new_text.words == source_text.words): break diff --git a/textattack/search_methods/particle_swarm_optimization_leap.py b/textattack/search_methods/particle_swarm_optimization_leap.py new file mode 100644 index 000000000..bec99114b --- /dev/null +++ b/textattack/search_methods/particle_swarm_optimization_leap.py @@ -0,0 +1,277 @@ +""" + +LEAP Particle Swarm Optimization +==================================== + +LEAP, an automated test method that uses LEvy flight-based Adaptive Particle +swarm optimization integrated with textual features to generate adversarial test cases. + +al +``_ +``_ +""" + +import copy + +import numpy as np +from scipy.special import gamma as gamma + +from textattack.goal_function_results import GoalFunctionResultStatus +from textattack.search_methods import ParticleSwarmOptimization + + +def sigmax(alpha): + numerator = gamma(alpha + 1.0) * np.sin(np.pi * alpha / 2.0) + denominator = gamma((alpha + 1) / 2.0) * alpha * np.power(2.0, (alpha - 1.0) / 2.0) + return np.power(numerator / denominator, 1.0 / alpha) + + +def vf(alpha): + x = np.random.normal(0, 1) + y = np.random.normal(0, 1) + + x = x * sigmax(alpha) + + return x / np.power(np.abs(y), 1.0 / alpha) + + +def K(alpha): + k = alpha * gamma((alpha + 1.0) / (2.0 * alpha)) / gamma(1.0 / alpha) + k *= np.power( + alpha + * gamma((alpha + 1.0) / 2.0) + / (gamma(alpha + 1.0) * np.sin(np.pi * alpha / 2.0)), + 1.0 / alpha, + ) + + return k + + +def C(alpha): + x = np.array( + (0.75, 0.8, 0.9, 1.1, 1.2, 1.3, 1.4, 1.5, 1.6, 1.7, 1.8, 1.9, 1.95, 1.99) + ) + y = np.array( + ( + 2.2085, + 2.483, + 2.7675, + 2.945, + 2.941, + 2.9005, + 2.8315, + 2.737, + 2.6125, + 2.4465, + 2.206, + 1.7915, + 1.3925, + 0.6089, + ) + ) + + return np.interp(alpha, x, y) + + +def levy(alpha, gamma=1, n=1): + w = 0 + for i in range(0, n): + v = vf(alpha) + + while v < -10: + v = vf(alpha) + + w += v * ((K(alpha) - 1.0) * np.exp(-v / C(alpha)) + 1.0) + + z = 1.0 / np.power(n, 1.0 / alpha) * w * gamma + + return z + + +def get_one_levy(min, max): + while True: + temp = levy(1.5, 1) + if min <= temp <= max: + break + else: + continue + return temp + + +def softmax(x, axis=1): + row_max = x.max(axis=axis) + + # Each element of the row needs to be subtracted from the corresponding maximum value, otherwise exp(x) will overflow, resulting in the inf case + row_max = row_max.reshape(-1, 1) + x = x - row_max + + # Calculate the exponential power of e + x_exp = np.exp(x) + x_sum = np.sum(x_exp, axis=axis, keepdims=True) + s = x_exp / x_sum + return s + + +class ParticleSwarmOptimizationLEAP(ParticleSwarmOptimization): + """Attacks a model with word substiutitions using a variant of Particle + Swarm Optimization (PSO) algorithm called LEAP.""" + + def _greedy_perturb(self, pop_member, original_result): + best_neighbors, prob_list = self._get_best_neighbors( + pop_member.result, original_result + ) + random_result = best_neighbors[np.argsort(prob_list)[-1]] + pop_member.attacked_text = random_result.attacked_text + pop_member.result = random_result + return True + + def perform_search(self, initial_result): + self._search_over = False + population = self._initialize_population(initial_result, self.pop_size) + + # Initialize velocities + v_init = [] + v_init_rand = np.random.uniform(-self.v_max, self.v_max, self.pop_size) + v_init_levy = [] + while True: + temp = levy(1.5, 1) + if -self.v_max <= temp <= self.v_max: + v_init_levy.append(temp) + else: + continue + if len(v_init_levy) == self.pop_size: + break + for i in range(self.pop_size): + if np.random.uniform( + -self.v_max, + self.v_max, + ) < levy(1.5, 1): + v_init.append(v_init_rand[i]) + else: + v_init.append(v_init_levy[i]) + v_init = np.array(v_init) + + velocities = np.array( + [ + [v_init[t] for _ in range(initial_result.attacked_text.num_words)] + for t in range(self.pop_size) + ] + ) + + global_elite = max(population, key=lambda x: x.score) + if ( + self._search_over + or global_elite.result.goal_status == GoalFunctionResultStatus.SUCCEEDED + ): + return global_elite.result + + local_elites = copy.copy(population) + + pop_fit_list = [] + for i in range(len(population)): + pop_fit_list.append(population[i].score) + pop_fit = np.array(pop_fit_list) + fit_ave = round(pop_fit.mean(), 3) + fit_min = pop_fit.min() + + # start iterations + omega = [] + for i in range(self.max_iters): + for k in range(len(population)): + if population[k].score < fit_ave: + omega.append( + self.omega_2 + + ( + (population[k].score - fit_min) + * (self.omega_1 - self.omega_2) + ) + / (fit_ave - fit_min) + ) + else: + omega.append(get_one_levy(0.5, 0.8)) + C1 = self.c1_origin - i / self.max_iters * (self.c1_origin - self.c2_origin) + C2 = self.c2_origin + i / self.max_iters * (self.c1_origin - self.c2_origin) + P1 = C1 + P2 = C2 + + for k in range(len(population)): + # calculate the probability of turning each word + pop_mem_words = population[k].words + local_elite_words = local_elites[k].words + assert len(pop_mem_words) == len( + local_elite_words + ), "PSO word length mismatch!" + + for d in range(len(pop_mem_words)): + velocities[k][d] = omega[k] * velocities[k][d] + (1 - omega[k]) * ( + self._equal(pop_mem_words[d], local_elite_words[d]) + + self._equal(pop_mem_words[d], global_elite.words[d]) + ) + turn_list = np.array([velocities[k]]) + turn_prob = softmax(turn_list)[0] + + if np.random.uniform() < P1: + # Move towards local elite + population[k] = self._turn( + local_elites[k], + population[k], + turn_prob, + initial_result.attacked_text, + ) + + if np.random.uniform() < P2: + # Move towards global elite + population[k] = self._turn( + global_elite, + population[k], + turn_prob, + initial_result.attacked_text, + ) + + # Check if there is any successful attack in the current population + pop_results, self._search_over = self.get_goal_results( + [p.attacked_text for p in population] + ) + if self._search_over: + # if `get_goal_results` gets cut short by query budget, resize population + population = population[: len(pop_results)] + for k in range(len(pop_results)): + population[k].result = pop_results[k] + + top_member = max(population, key=lambda x: x.score) + if ( + self._search_over + or top_member.result.goal_status == GoalFunctionResultStatus.SUCCEEDED + ): + return top_member.result + + # Mutation based on the current change rate + for k in range(len(population)): + change_ratio = initial_result.attacked_text.words_diff_ratio( + population[k].attacked_text + ) + # Referred from the original source code + p_change = 1 - 2 * change_ratio + if np.random.uniform() < p_change: + self._perturb(population[k], initial_result) + + if self._search_over: + break + + # Check if there is any successful attack in the current population + top_member = max(population, key=lambda x: x.score) + if ( + self._search_over + or top_member.result.goal_status == GoalFunctionResultStatus.SUCCEEDED + ): + return top_member.result + + # Update the elite if the score is increased + for k in range(len(population)): + if population[k].score > local_elites[k].score: + local_elites[k] = copy.copy(population[k]) + + if top_member.score > global_elite.score: + global_elite = copy.copy(top_member) + + return global_elite.result diff --git a/textattack/shared/validators.py b/textattack/shared/validators.py index 45513a2a3..55f4ed08c 100644 --- a/textattack/shared/validators.py +++ b/textattack/shared/validators.py @@ -25,10 +25,7 @@ r"^textattack.models.helpers.word_cnn_for_classification.*", r"^transformers.modeling_\w*\.\w*ForSequenceClassification$", ], - ( - NonOverlappingOutput, - MinimizeBleu, - ): [ + (NonOverlappingOutput, MinimizeBleu,): [ r"^textattack.models.helpers.t5_for_text_to_text.*", ], } diff --git a/textattack/transformations/sentence_transformations/back_transcription.py b/textattack/transformations/sentence_transformations/back_transcription.py index 81cc8aff9..c902b6d52 100644 --- a/textattack/transformations/sentence_transformations/back_transcription.py +++ b/textattack/transformations/sentence_transformations/back_transcription.py @@ -12,8 +12,9 @@ class BackTranscription(SentenceTransformation): - """A type of sentence level transformation that takes in a text input, converts it into - synthesized speech using ASR, and transcribes it back to text using TTS. + """A type of sentence level transformation that takes in a text input, + converts it into synthesized speech using ASR, and transcribes it back to + text using TTS. tts_model: text-to-speech model from huggingface asr_model: automatic speech recognition model from huggingface