How to initialize EntityRuler with sourced tagger, parser, lemmatizer, … components?

[kinghuang]

I'm trying to assemble a model where the tagger, parser, lemmatizer and other components are sourced from another model. This model also has several EntityRulers, which I'm initializing with patterns. Some of the patterns use the LEMMA attribute.

Config Excerpts:

[nlp]
pipeline = ["tok2vec","affix_splitter","tagger","parser","senter","attribute_ruler","lemmatizer","service_code_labeller","num_unit_ruler","unspsc_ruler","unspsc_40140000_ruler","unspsc_40170000_ruler","pied_piper_ruler","ner","ident_ruler","entity_linker","entity_validator","entity_annotator"]

[components.lemmatizer]
source = ${vars.base_model}
component = "lemmatizer"

[components.num_unit_ruler]
factory = "entity_ruler"

[initialize.components.num_unit_ruler.patterns]
@readers = "tsds.Patterns.v1"
sets = ["numeric", "qudt/all", "qudt_alt/all"]

Full Config File

[vars]
base_model = "en_enverus_oi_ner"

[paths]
train = ""
dev = ""
data = ""
raw = null
init_tok2vec = null
vectors = ${vars.base_model}

[system]
gpu_allocator = null
seed = 0

[nlp]
lang = "en"
pipeline = ["tok2vec","affix_splitter","tagger","parser","senter","attribute_ruler","lemmatizer","service_code_labeller","num_unit_ruler","unspsc_ruler","unspsc_40140000_ruler","unspsc_40170000_ruler","pied_piper_ruler","ner","ident_ruler","entity_linker","entity_validator","entity_annotator"]
tokenizer = {"@tokenizers":"oi.Tokenizer.v1"}
before_creation = null
after_creation = null
disabled = []
batch_size = 1000

[components]

[components.affix_splitter]
factory = "affix_splitter"
affix_attr = "service_code"
affix_spans_key = "service_code"
after_suffix = null
before_prefix = ["0+"]

[components.attribute_ruler]
source = ${vars.base_model}
component = "attribute_ruler"

[components.entity_annotator]
factory = "entity_annotator"
value_key = "value"
meta_key = "meta"

[components.entity_linker]
source = ${vars.base_model}
component = "entity_linker"

[components.entity_validator]
factory = "entity_validator"
incorrect_spans_key = "non_entities"

[components.lemmatizer]
source = ${vars.base_model}
component = "lemmatizer"

[components.service_code_labeller]
factory = "span_entity_labeller"
spans_key = "service_code"
set_label = "IDENT"
values = ["(?=.*\\d)[\\da-zA-Z\\-]+"]

[components.num_unit_ruler]
factory = "entity_ruler"

[components.unspsc_ruler]
factory = "entity_ruler"

[components.unspsc_40140000_ruler]
factory = "counting_entity_ruler"
overwrite_ents = true
min_count = 2
labels = ["GOOD","PART","ORG","PRODUCT","AGGREGATE","TYPE","MEASURE","SIZE","SPEC","MATERIAL","UNIT"]

[components.unspsc_40170000_ruler]
factory = "counting_entity_ruler"
overwrite_ents = true
min_count = 2
labels = ["GOOD","PART","ORG","PRODUCT","AGGREGATE","TYPE","MEASURE","SIZE","SPEC","MATERIAL","UNIT"]

[components.pied_piper_ruler]
factory = "entity_ruler"
overwrite_ents = true

[components.ident_ruler]
factory = "entity_ruler"
overwrite_ents = true

[components.ner]
source = ${vars.base_model}
component = "ner"

[components.parser]
source = ${vars.base_model}
component = "parser"

[components.senter]
source = ${vars.base_model}
component = "senter"

[components.tagger]
source = ${vars.base_model}
component = "tagger"

[components.tok2vec]
source = ${vars.base_model}
component = "tok2vec"

[corpora]

[corpora.dev]
@readers = "spacy.Corpus.v1"
path = ${paths.dev}
max_length = 0
gold_preproc = false
limit = 0
augmenter = null

[corpora.train]
@readers = "spacy.Corpus.v1"
path = ${paths.train}
max_length = 2000
gold_preproc = false
limit = 0
augmenter = null

[training]
train_corpus = "corpora.train"
dev_corpus = "corpora.dev"
seed = ${system.seed}
gpu_allocator = ${system.gpu_allocator}
dropout = 0.1
accumulate_gradient = 1
patience = 1600
max_epochs = 1
max_steps = 1
eval_frequency = 200
frozen_components = ["attribute_ruler","lemmatizer","tok2vec","tagger","parser","ner"]
before_to_disk = null
annotating_components = []

[training.batcher]
@batchers = "spacy.batch_by_words.v1"
discard_oversize = false
tolerance = 0.2
get_length = null

[training.batcher.size]
@schedules = "compounding.v1"
start = 100
stop = 1000
compound = 1.001
t = 0.0

[training.logger]
@loggers = "spacy.ConsoleLogger.v1"
progress_bar = false

[training.optimizer]
@optimizers = "Adam.v1"
beta1 = 0.9
beta2 = 0.999
L2_is_weight_decay = true
L2 = 0.01
grad_clip = 1.0
use_averages = false
eps = 0.00000001

[training.optimizer.learn_rate]
@schedules = "warmup_linear.v1"
warmup_steps = 250
total_steps = 20000
initial_rate = 0.00005

[training.score_weights]
tag_acc = null
dep_uas = null
dep_las = null
dep_las_per_type = null
sents_p = null
sents_r = null
sents_f = 0.5
lemma_acc = null
ents_f = 0.5
ents_p = 0.0
ents_r = 0.0
ents_per_type = null

[pretraining]

[initialize]
vectors = ${paths.vectors}
init_tok2vec = ${paths.init_tok2vec}
vocab_data = null
lookups = null
before_init = null
after_init = null

[initialize.components]

[initialize.components.num_unit_ruler]

[initialize.components.num_unit_ruler.patterns]
@readers = "tsds.Patterns.v1"
sets = ["numeric", "qudt/all", "qudt_alt/all"]

[initialize.components.unspsc_ruler]

[initialize.components.unspsc_ruler.patterns]
@readers = "tsds.Patterns.v1"
sets = ["unspsc/40140000", "unspsc_alt/40140000-GOOD", "unspsc/40170000", "unspsc_alt/40170000-GOOD"]

[initialize.components.unspsc_40140000_ruler]

[initialize.components.unspsc_40140000_ruler.patterns]
@readers = "tsds.Patterns.v1"
sets = ["unspsc/40140000", "unspsc_alt/40140000"]

[initialize.components.unspsc_40170000_ruler]

[initialize.components.unspsc_40170000_ruler.patterns]
@readers = "tsds.Patterns.v1"
sets = ["unspsc/40170000", "unspsc_alt/40170000"]

[initialize.components.pied_piper_ruler]

[initialize.components.pied_piper_ruler.patterns]
@readers = "tsds.Patterns.v1"
sets = ["pied_piper/pk", "pied_piper/sku"]

[initialize.components.ident_ruler]

[initialize.components.ident_ruler.patterns]
@readers = "tsds.Patterns.v1"
sets = ["ident/all"]

[initialize.components.entity_validator]

[initialize.components.entity_validator.patterns]
@readers = "srsly.read_yaml.v1"
path = "${paths.data}/entity_validator/patterns.yml"

[initialize.tokenizer]

Example Pattern:

{"id": "unit:AC", "label": "UNIT", "pattern": [{"LEMMA": "acre"}]}

Problem:

When I run spacy assemble, a ValueError occurs. The lemmatizer component is not active in the pipeline.

ValueError: [E155] The pipeline needs to include a lemmatizer in order to use Matcher or PhraseMatcher with the attribute LEMMA. Try using `nlp()` instead of `nlp.make_doc()` or `list(nlp.pipe())` instead of `list(nlp.tokenizer.pipe())`.

Full Stack Trace

Traceback (most recent call last):
  File "/usr/local/lib/python3.9/runpy.py", line 197, in _run_module_as_main
    return _run_code(code, main_globals, None,
  File "/usr/local/lib/python3.9/runpy.py", line 87, in _run_code
    exec(code, run_globals)
  File "/usr/local/lib/python3.9/site-packages/spacy/__main__.py", line 4, in <module>
    setup_cli()
  File "/usr/local/lib/python3.9/site-packages/spacy/cli/_util.py", line 69, in setup_cli
    command(prog_name=COMMAND)
  File "/usr/local/lib/python3.9/site-packages/click/core.py", line 829, in __call__
    return self.main(*args, **kwargs)
  File "/usr/local/lib/python3.9/site-packages/click/core.py", line 782, in main
    rv = self.invoke(ctx)
  File "/usr/local/lib/python3.9/site-packages/click/core.py", line 1259, in invoke
    return _process_result(sub_ctx.command.invoke(sub_ctx))
  File "/usr/local/lib/python3.9/site-packages/click/core.py", line 1066, in invoke
    return ctx.invoke(self.callback, **ctx.params)
  File "/usr/local/lib/python3.9/site-packages/click/core.py", line 610, in invoke
    return callback(*args, **kwargs)
  File "/usr/local/lib/python3.9/site-packages/typer/main.py", line 500, in wrapper
    return callback(**use_params)  # type: ignore
  File "/usr/local/lib/python3.9/site-packages/spacy/cli/assemble.py", line 51, in assemble_cli
    nlp.initialize()
  File "/usr/local/lib/python3.9/site-packages/spacy/language.py", line 1272, in initialize
    proc.initialize(get_examples, nlp=self, **p_settings)
  File "/usr/local/lib/python3.9/site-packages/spacy/pipeline/entityruler.py", line 220, in initialize
    self.add_patterns(patterns)
  File "/usr/local/lib/python3.9/site-packages/spacy/pipeline/entityruler.py", line 299, in add_patterns
    for label, pattern, ent_id in zip(
  File "/usr/local/lib/python3.9/site-packages/spacy/language.py", line 1520, in pipe
    for doc in docs:
  File "/usr/local/lib/python3.9/site-packages/spacy/util.py", line 1516, in _pipe
    yield from proc.pipe(docs, **kwargs)
  File "spacy/pipeline/pipe.pyx", line 53, in pipe
  File "/usr/local/lib/python3.9/site-packages/spacy/util.py", line 1516, in _pipe
    yield from proc.pipe(docs, **kwargs)
  File "spacy/pipeline/pipe.pyx", line 53, in pipe
  File "/usr/local/lib/python3.9/site-packages/spacy/util.py", line 1516, in _pipe
    yield from proc.pipe(docs, **kwargs)
  File "spacy/pipeline/pipe.pyx", line 53, in pipe
  File "/usr/local/lib/python3.9/site-packages/spacy/util.py", line 1516, in _pipe
    yield from proc.pipe(docs, **kwargs)
  File "spacy/pipeline/pipe.pyx", line 58, in pipe
  File "/usr/local/lib/python3.9/site-packages/spacy/util.py", line 1535, in raise_error
    raise e
  File "spacy/pipeline/pipe.pyx", line 55, in spacy.pipeline.pipe.Pipe.pipe
  File "/usr/local/lib/python3.9/site-packages/spacy/pipeline/entityruler.py", line 140, in __call__
    error_handler(self.name, self, [doc], e)
  File "/usr/local/lib/python3.9/site-packages/spacy/util.py", line 1535, in raise_error
    raise e
  File "/usr/local/lib/python3.9/site-packages/spacy/pipeline/entityruler.py", line 136, in __call__
    matches = self.match(doc)
  File "/usr/local/lib/python3.9/site-packages/spacy/pipeline/entityruler.py", line 149, in match
    matches = list(self.matcher(doc)) + list(self.phrase_matcher(doc))
  File "spacy/matcher/matcher.pyx", line 248, in spacy.matcher.matcher.Matcher.__call__
ValueError: [E155] The pipeline needs to include a lemmatizer in order to use Matcher or PhraseMatcher with the attribute LEMMA. Try using `nlp()` instead of `nlp.make_doc()` or `list(nlp.pipe())` instead of `list(nlp.tokenizer.pipe())`.

This is because assemble_cli() disables all sourced pipeline components before calling initialize() on the remaining components.

spaCy/spacy/cli/assemble.py

Lines 47 to 51 in 90ec820

	sourced = get_sourced_components(config)
	# Make sure that listeners are defined before initializing further
	nlp._link_components()
	with nlp.select_pipes(disable=[*sourced]):
	nlp.initialize()

Pipeline Components During EntityRuler.initialize():

['affix_splitter', 'service_code_labeller', 'num_unit_ruler', 'unspsc_ruler', 'unspsc_40140000_ruler', 'unspsc_40170000_ruler']

All the sourced pipeline components, including lemmatizer are disabled.

Question:

How I can initialize EntityRuler components with patterns that use attributes such as LEMMA in a config where the lemmatizer and other components are sourced?

I think what is needed is for the assemble command to not initialize the sourced pipeline components, instead of disabling them during intialization.