198 lines
8.6 KiB
YAML
198 lines
8.6 KiB
YAML
title: "fa_core_news_sm"
|
|
description: >
|
|
A CPU-sized Persian (fa) core pipeline for spaCy 3.8: tagger (XPOS), morphologizer
|
|
(UPOS + FEATS), trainable lemmatizer, dependency parser and NER.
|
|
|
|
UD components are trained on UD_Persian-PerDT (PerUDT v1.0, CC BY-SA 4.0). The NER
|
|
component is trained separately on ParsTwiNER (MIT) and merged in, because no
|
|
redistributably-licensed Persian NER corpus shares a genre with the treebank. See
|
|
docs/MODELS.md for the full source/licence analysis and docs/CONTRIBUTING-GUIDE.md
|
|
for how this gets published.
|
|
|
|
Run everything with: `spacy project run all`
|
|
|
|
vars:
|
|
lang: "fa"
|
|
package_name: "core_news_sm"
|
|
package_version: "3.8.0"
|
|
treebank: "fa_perdt"
|
|
# -1 = CPU. A GTX 940MX (2 GB) is not worth the transfer overhead for an sm pipeline.
|
|
gpu: -1
|
|
n_sents: 10
|
|
|
|
directories:
|
|
- "assets"
|
|
- "corpus"
|
|
- "configs"
|
|
- "scripts"
|
|
- "training"
|
|
- "metrics"
|
|
- "packages"
|
|
|
|
assets:
|
|
- dest: "assets/ud/fa_perdt-ud-train.conllu"
|
|
url: "https://raw.githubusercontent.com/UniversalDependencies/UD_Persian-PerDT/master/fa_perdt-ud-train.conllu"
|
|
checksum: "f5a8ba901a776b4fd1941ecadcc6d506"
|
|
description: "UD_Persian-PerDT train split (CC BY-SA 4.0)"
|
|
- dest: "assets/ud/fa_perdt-ud-dev.conllu"
|
|
url: "https://raw.githubusercontent.com/UniversalDependencies/UD_Persian-PerDT/master/fa_perdt-ud-dev.conllu"
|
|
checksum: "f103020da7c1e917aafb8a8321f4cb84"
|
|
description: "UD_Persian-PerDT dev split (CC BY-SA 4.0)"
|
|
- dest: "assets/ud/fa_perdt-ud-test.conllu"
|
|
url: "https://raw.githubusercontent.com/UniversalDependencies/UD_Persian-PerDT/master/fa_perdt-ud-test.conllu"
|
|
checksum: "b62a66994cef2c50f7e524a1471102d8"
|
|
description: "UD_Persian-PerDT test split (CC BY-SA 4.0)"
|
|
- dest: "assets/ner/ParsTwiNER_corpus_v1.0.0.zip"
|
|
url: "https://github.com/overfit-ir/parstwiner/releases/download/v1.0.0/ParsTwiNER_corpus_v1.0.0.zip"
|
|
checksum: "54615340d76c4d51b8f54751b07a278d"
|
|
description: "ParsTwiNER Persian Twitter NER corpus, IOB2 (MIT)"
|
|
|
|
workflows:
|
|
all:
|
|
- inspect
|
|
- convert-ud
|
|
- convert-ner
|
|
- debug-data
|
|
- train-core
|
|
- train-ner
|
|
- assemble
|
|
- evaluate
|
|
- finalize-meta
|
|
- package
|
|
- smoke
|
|
|
|
commands:
|
|
- name: "inspect"
|
|
help: "Report annotation coverage of the downloaded treebank(s)"
|
|
script:
|
|
- "python scripts/inspect_treebanks.py assets/ud"
|
|
deps:
|
|
- "assets/ud/fa_perdt-ud-train.conllu"
|
|
- "scripts/inspect_treebanks.py"
|
|
|
|
- name: "convert-ud"
|
|
help: >
|
|
CoNLL-U -> DocBin. --merge-subtokens fuses multiword-token clitics into single
|
|
tokens; docs/MODELS.md §5 has the measurement that justifies it.
|
|
script:
|
|
- "python -m spacy convert assets/ud/${vars.treebank}-ud-train.conllu corpus/merged --converter conllu --n-sents ${vars.n_sents} --merge-subtokens"
|
|
- "python -m spacy convert assets/ud/${vars.treebank}-ud-dev.conllu corpus/merged --converter conllu --n-sents ${vars.n_sents} --merge-subtokens"
|
|
- "python -m spacy convert assets/ud/${vars.treebank}-ud-test.conllu corpus/merged --converter conllu --n-sents ${vars.n_sents} --merge-subtokens"
|
|
# Reproduce the measurement in docs/MODELS.md §5: convert the dev split WITHOUT
|
|
# merging and compare both against the tokenizer we actually ship.
|
|
- "python -m spacy convert assets/ud/${vars.treebank}-ud-dev.conllu corpus/split --converter conllu --n-sents ${vars.n_sents}"
|
|
- "python scripts/tokenization_report.py corpus/merged/${vars.treebank}-ud-dev.spacy corpus/split/${vars.treebank}-ud-dev.spacy"
|
|
deps:
|
|
- "assets/ud/${vars.treebank}-ud-train.conllu"
|
|
- "assets/ud/${vars.treebank}-ud-dev.conllu"
|
|
- "assets/ud/${vars.treebank}-ud-test.conllu"
|
|
outputs:
|
|
- "corpus/merged/${vars.treebank}-ud-train.spacy"
|
|
- "corpus/merged/${vars.treebank}-ud-dev.spacy"
|
|
- "corpus/merged/${vars.treebank}-ud-test.spacy"
|
|
|
|
- name: "convert-ner"
|
|
help: "Unpack ParsTwiNER and convert its IOB2 files to DocBin"
|
|
script:
|
|
- "python scripts/extract_parstwiner.py assets/ner/ParsTwiNER_corpus_v1.0.0.zip assets/ner"
|
|
- "python -m spacy convert assets/ner/train.txt corpus/ner --converter ner --n-sents ${vars.n_sents} --lang ${vars.lang}"
|
|
- "python -m spacy convert assets/ner/dev.txt corpus/ner --converter ner --n-sents ${vars.n_sents} --lang ${vars.lang}"
|
|
- "python -m spacy convert assets/ner/test.txt corpus/ner --converter ner --n-sents ${vars.n_sents} --lang ${vars.lang}"
|
|
deps:
|
|
- "assets/ner/ParsTwiNER_corpus_v1.0.0.zip"
|
|
- "scripts/extract_parstwiner.py"
|
|
outputs:
|
|
- "corpus/ner/train.spacy"
|
|
- "corpus/ner/dev.spacy"
|
|
- "corpus/ner/test.spacy"
|
|
|
|
- name: "debug-data"
|
|
help: "Validate both corpora against their configs before burning CPU on training"
|
|
script:
|
|
- "python -m spacy debug data configs/fa_core_news_sm.cfg --paths.train corpus/merged/${vars.treebank}-ud-train.spacy --paths.dev corpus/merged/${vars.treebank}-ud-dev.spacy"
|
|
- "python -m spacy debug data configs/fa_ner_sm.cfg --paths.train corpus/ner/train.spacy --paths.dev corpus/ner/dev.spacy"
|
|
deps:
|
|
- "corpus/merged/${vars.treebank}-ud-train.spacy"
|
|
- "corpus/ner/train.spacy"
|
|
- "configs/fa_core_news_sm.cfg"
|
|
- "configs/fa_ner_sm.cfg"
|
|
|
|
- name: "train-core"
|
|
help: "Train tok2vec + tagger + morphologizer + trainable_lemmatizer + parser on PerDT"
|
|
script:
|
|
- "python -m spacy train configs/fa_core_news_sm.cfg --output training/core --paths.train corpus/merged/${vars.treebank}-ud-train.spacy --paths.dev corpus/merged/${vars.treebank}-ud-dev.spacy --gpu-id ${vars.gpu}"
|
|
deps:
|
|
- "corpus/merged/${vars.treebank}-ud-train.spacy"
|
|
- "corpus/merged/${vars.treebank}-ud-dev.spacy"
|
|
- "configs/fa_core_news_sm.cfg"
|
|
outputs:
|
|
- "training/core/model-best"
|
|
|
|
- name: "train-ner"
|
|
help: "Train the standalone NER component (own internal tok2vec) on ParsTwiNER"
|
|
script:
|
|
- "python -m spacy train configs/fa_ner_sm.cfg --output training/ner --paths.train corpus/ner/train.spacy --paths.dev corpus/ner/dev.spacy --gpu-id ${vars.gpu}"
|
|
deps:
|
|
- "corpus/ner/train.spacy"
|
|
- "corpus/ner/dev.spacy"
|
|
- "configs/fa_ner_sm.cfg"
|
|
outputs:
|
|
- "training/ner/model-best"
|
|
|
|
- name: "assemble"
|
|
help: "Source the trained ner into the core pipeline and write full meta.json"
|
|
script:
|
|
- "python scripts/assemble_core.py training/core/model-best training/ner/model-best training/fa_core_news_sm --version ${vars.package_version}"
|
|
deps:
|
|
- "training/core/model-best"
|
|
- "training/ner/model-best"
|
|
- "scripts/assemble_core.py"
|
|
outputs:
|
|
- "training/fa_core_news_sm"
|
|
|
|
- name: "evaluate"
|
|
help: "Score the assembled pipeline on both held-out test sets"
|
|
script:
|
|
- "python -m spacy benchmark accuracy training/fa_core_news_sm corpus/merged/${vars.treebank}-ud-test.spacy --output metrics/ud-test.json --gpu-id ${vars.gpu}"
|
|
- "python -m spacy benchmark accuracy training/fa_core_news_sm corpus/ner/test.spacy --output metrics/ner-test.json --gpu-id ${vars.gpu}"
|
|
deps:
|
|
- "training/fa_core_news_sm"
|
|
- "corpus/merged/${vars.treebank}-ud-test.spacy"
|
|
- "corpus/ner/test.spacy"
|
|
outputs:
|
|
- "metrics/ud-test.json"
|
|
- "metrics/ner-test.json"
|
|
|
|
- name: "finalize-meta"
|
|
help: >
|
|
Re-assemble, this time folding the test scores into meta.json["performance"].
|
|
Separate from `assemble` because the scores can only exist after `evaluate`, and
|
|
`evaluate` needs an assembled pipeline to score. Cheap: it only copies models.
|
|
script:
|
|
- "python scripts/assemble_core.py training/core/model-best training/ner/model-best training/fa_core_news_sm --version ${vars.package_version} --ud-metrics metrics/ud-test.json --ner-metrics metrics/ner-test.json"
|
|
deps:
|
|
- "metrics/ud-test.json"
|
|
- "metrics/ner-test.json"
|
|
- "scripts/assemble_core.py"
|
|
|
|
- name: "package"
|
|
help: "Build the installable wheel + sdist"
|
|
script:
|
|
- "python -m spacy package training/fa_core_news_sm packages --name ${vars.package_name} --version ${vars.package_version} --build sdist,wheel --force"
|
|
deps:
|
|
- "training/fa_core_news_sm"
|
|
outputs:
|
|
- "packages/${vars.lang}_${vars.package_name}-${vars.package_version}"
|
|
|
|
- name: "smoke"
|
|
help: "Load the packaged pipeline and run it over real Persian text"
|
|
script:
|
|
- "python scripts/smoke_test.py training/fa_core_news_sm"
|
|
deps:
|
|
- "training/fa_core_news_sm"
|
|
|
|
- name: "clean"
|
|
help: "Drop corpora, training runs and metrics (keeps downloaded assets)"
|
|
script:
|
|
- "rm -rf corpus/merged corpus/split corpus/ner training metrics packages"
|