304 lines
14 KiB
YAML
304 lines
14 KiB
YAML
title: "fa_core_news_sm / fa_dep_news_sm"
|
|
description: >
|
|
CPU-sized Persian (fa) pipelines for spaCy 3.8, built entirely from UD_Persian-PerDT
|
|
(PerUDT v1.0, CC BY-SA 4.0): tagger (XPOS), morphologizer (UPOS + FEATS), trainable
|
|
lemmatizer, dependency parser and NER.
|
|
|
|
Two shipping packages, same corpus, differing only in whether NER is included:
|
|
`fa_dep_news_sm` (no NER) and `fa_core_news_sm` (with NER). The NER comes from the
|
|
treebank's own `not-to-release/Dadegan with NER tag/` layer, so both packages share one
|
|
corpus, one genre, one tokenization and one licence.
|
|
|
|
PerDT's NER labels are silver, produced by Beheshti-NER with manual corrections, and are
|
|
transferred onto this pipeline's tokenization by difflib at a 99.86% rate. Per-label
|
|
scores are published in meta.json; MON, TIM and PCT are thin.
|
|
|
|
See docs/MODELS.md for the source and licence analysis and docs/CONTRIBUTING-GUIDE.md for
|
|
how this gets published.
|
|
|
|
Build both with: `spacy project run all`
|
|
Standalone NER-only package: `spacy project run ent`
|
|
|
|
vars:
|
|
lang: "fa"
|
|
dep_package_name: "dep_news_sm"
|
|
core_package_name: "core_news_sm"
|
|
ent_package_name: "ent_news_sm"
|
|
package_version: "3.8.0"
|
|
treebank: "fa_perdt"
|
|
# -1 = CPU. A GTX 940MX (2 GB) is not worth the transfer overhead for an sm pipeline.
|
|
gpu: -1
|
|
n_sents: 10
|
|
|
|
directories:
|
|
- "assets"
|
|
- "corpus"
|
|
- "configs"
|
|
- "scripts"
|
|
- "training"
|
|
- "metrics"
|
|
- "packages"
|
|
|
|
assets:
|
|
- dest: "assets/ud/fa_perdt-ud-train.conllu"
|
|
url: "https://raw.githubusercontent.com/UniversalDependencies/UD_Persian-PerDT/master/fa_perdt-ud-train.conllu"
|
|
checksum: "f5a8ba901a776b4fd1941ecadcc6d506"
|
|
description: "UD_Persian-PerDT train split (CC BY-SA 4.0)"
|
|
- dest: "assets/ud/fa_perdt-ud-dev.conllu"
|
|
url: "https://raw.githubusercontent.com/UniversalDependencies/UD_Persian-PerDT/master/fa_perdt-ud-dev.conllu"
|
|
checksum: "f103020da7c1e917aafb8a8321f4cb84"
|
|
description: "UD_Persian-PerDT dev split (CC BY-SA 4.0)"
|
|
- dest: "assets/ud/fa_perdt-ud-test.conllu"
|
|
url: "https://raw.githubusercontent.com/UniversalDependencies/UD_Persian-PerDT/master/fa_perdt-ud-test.conllu"
|
|
checksum: "b62a66994cef2c50f7e524a1471102d8"
|
|
description: "UD_Persian-PerDT test split (CC BY-SA 4.0)"
|
|
- dest: "assets/ud-ner/train_with_NER_tag.txt"
|
|
url: "https://raw.githubusercontent.com/UniversalDependencies/UD_Persian-PerDT/master/not-to-release/Dadegan%20with%20NER%20tag/train_with_NER_tag.txt"
|
|
checksum: "ecb96cf99b38bc485cac21d22914e413"
|
|
description: "PerDT NER layer, train split, IOB2 (CC BY-SA 4.0)"
|
|
- dest: "assets/ud-ner/dev_with_NER_tag.txt"
|
|
url: "https://raw.githubusercontent.com/UniversalDependencies/UD_Persian-PerDT/master/not-to-release/Dadegan%20with%20NER%20tag/dev_with_NER_tag.txt"
|
|
checksum: "2a56ef7eb2e3732e221317af457d1c09"
|
|
description: "PerDT NER layer, dev split, IOB2 (CC BY-SA 4.0)"
|
|
- dest: "assets/ud-ner/test_with_NER_tag.txt"
|
|
url: "https://raw.githubusercontent.com/UniversalDependencies/UD_Persian-PerDT/master/not-to-release/Dadegan%20with%20NER%20tag/test_with_NER_tag.txt"
|
|
checksum: "6d80dd783527562c2ea5189f218a12b5"
|
|
description: "PerDT NER layer, test split, IOB2 (CC BY-SA 4.0)"
|
|
|
|
workflows:
|
|
# Both shipping artifacts: fa_dep_news_sm and fa_core_news_sm.
|
|
all:
|
|
- inspect
|
|
- convert-ud
|
|
- transfer-ner
|
|
- convert-ner
|
|
- debug-data
|
|
- debug-data-ner
|
|
- train-dep
|
|
- train-ner
|
|
- finalize-dep
|
|
- evaluate-dep
|
|
- assemble-core
|
|
- evaluate-core
|
|
- finalize-meta
|
|
- package
|
|
- smoke
|
|
# Optional third artifact: the NER alone, for users who only want entities.
|
|
ent:
|
|
- finalize-ent
|
|
- evaluate-ent
|
|
- package-ent
|
|
|
|
commands:
|
|
- name: "inspect"
|
|
help: "Report annotation coverage of the downloaded treebank(s)"
|
|
script:
|
|
- "python scripts/inspect_treebanks.py assets/ud"
|
|
deps:
|
|
- "assets/ud/fa_perdt-ud-train.conllu"
|
|
- "scripts/inspect_treebanks.py"
|
|
|
|
- name: "convert-ud"
|
|
help: >
|
|
CoNLL-U -> DocBin. --merge-subtokens fuses multiword-token clitics into single
|
|
tokens; docs/MODELS.md §5 has the measurement that justifies it.
|
|
script:
|
|
- "python -m spacy convert assets/ud/${vars.treebank}-ud-train.conllu corpus/merged --converter conllu --n-sents ${vars.n_sents} --merge-subtokens"
|
|
- "python -m spacy convert assets/ud/${vars.treebank}-ud-dev.conllu corpus/merged --converter conllu --n-sents ${vars.n_sents} --merge-subtokens"
|
|
- "python -m spacy convert assets/ud/${vars.treebank}-ud-test.conllu corpus/merged --converter conllu --n-sents ${vars.n_sents} --merge-subtokens"
|
|
# Reproduce the measurement in docs/MODELS.md §5: convert the dev split WITHOUT
|
|
# merging and compare both against the tokenizer we actually ship.
|
|
- "python -m spacy convert assets/ud/${vars.treebank}-ud-dev.conllu corpus/split --converter conllu --n-sents ${vars.n_sents}"
|
|
- "python scripts/tokenization_report.py corpus/merged/${vars.treebank}-ud-dev.spacy corpus/split/${vars.treebank}-ud-dev.spacy"
|
|
deps:
|
|
- "assets/ud/${vars.treebank}-ud-train.conllu"
|
|
- "assets/ud/${vars.treebank}-ud-dev.conllu"
|
|
- "assets/ud/${vars.treebank}-ud-test.conllu"
|
|
outputs:
|
|
- "corpus/merged/${vars.treebank}-ud-train.spacy"
|
|
- "corpus/merged/${vars.treebank}-ud-dev.spacy"
|
|
- "corpus/merged/${vars.treebank}-ud-test.spacy"
|
|
|
|
- name: "transfer-ner"
|
|
help: >
|
|
Align PerDT's NER layer onto the --merge-subtokens tokenization. The NER files use the
|
|
original Dadegan tokenization, which matches the released UD tokenization in only 57 to
|
|
62% of sentences, so spans are transferred by difflib. Measured rate 99.86%; spans that
|
|
cannot be aligned exactly are dropped rather than guessed.
|
|
script:
|
|
- "python scripts/transfer_perdt_ner.py --conllu-dir assets/ud --ner-dir assets/ud-ner --out corpus/perdt-ner-iob"
|
|
deps:
|
|
- "assets/ud/${vars.treebank}-ud-train.conllu"
|
|
- "assets/ud-ner/train_with_NER_tag.txt"
|
|
- "scripts/transfer_perdt_ner.py"
|
|
outputs:
|
|
- "corpus/perdt-ner-iob/train.txt"
|
|
- "corpus/perdt-ner-iob/dev.txt"
|
|
- "corpus/perdt-ner-iob/test.txt"
|
|
|
|
- name: "convert-ner"
|
|
help: "Transferred IOB2 -> DocBin"
|
|
script:
|
|
- "python -m spacy convert corpus/perdt-ner-iob/train.txt corpus/perdt-ner --converter ner --n-sents ${vars.n_sents} --lang ${vars.lang}"
|
|
- "python -m spacy convert corpus/perdt-ner-iob/dev.txt corpus/perdt-ner --converter ner --n-sents ${vars.n_sents} --lang ${vars.lang}"
|
|
- "python -m spacy convert corpus/perdt-ner-iob/test.txt corpus/perdt-ner --converter ner --n-sents ${vars.n_sents} --lang ${vars.lang}"
|
|
deps:
|
|
- "corpus/perdt-ner-iob/train.txt"
|
|
outputs:
|
|
- "corpus/perdt-ner/train.spacy"
|
|
- "corpus/perdt-ner/dev.spacy"
|
|
- "corpus/perdt-ner/test.spacy"
|
|
|
|
- name: "debug-data"
|
|
help: "Validate the treebank against the config before burning CPU on training"
|
|
script:
|
|
- "python -m spacy debug data configs/fa_dep_news_sm.cfg --paths.train corpus/merged/${vars.treebank}-ud-train.spacy --paths.dev corpus/merged/${vars.treebank}-ud-dev.spacy"
|
|
deps:
|
|
- "corpus/merged/${vars.treebank}-ud-train.spacy"
|
|
- "configs/fa_dep_news_sm.cfg"
|
|
|
|
- name: "debug-data-ner"
|
|
help: "Validate the transferred PerDT NER corpus against the NER config"
|
|
script:
|
|
- "python -m spacy debug data configs/fa_ner_sm.cfg --paths.train corpus/perdt-ner/train.spacy --paths.dev corpus/perdt-ner/dev.spacy"
|
|
deps:
|
|
- "corpus/perdt-ner/train.spacy"
|
|
- "configs/fa_ner_sm.cfg"
|
|
|
|
- name: "train-dep"
|
|
help: "Train tok2vec + tagger + morphologizer + trainable_lemmatizer + parser on PerDT"
|
|
script:
|
|
- "python -m spacy train configs/fa_dep_news_sm.cfg --output training/dep --paths.train corpus/merged/${vars.treebank}-ud-train.spacy --paths.dev corpus/merged/${vars.treebank}-ud-dev.spacy --gpu-id ${vars.gpu}"
|
|
deps:
|
|
- "corpus/merged/${vars.treebank}-ud-train.spacy"
|
|
- "corpus/merged/${vars.treebank}-ud-dev.spacy"
|
|
- "configs/fa_dep_news_sm.cfg"
|
|
outputs:
|
|
- "training/dep/model-best"
|
|
|
|
- name: "train-ner"
|
|
help: "Train the NER component (own embedded tok2vec) on the transferred PerDT layer"
|
|
script:
|
|
- "python -m spacy train configs/fa_ner_sm.cfg --output training/perdt-ner --paths.train corpus/perdt-ner/train.spacy --paths.dev corpus/perdt-ner/dev.spacy --gpu-id ${vars.gpu}"
|
|
deps:
|
|
- "corpus/perdt-ner/train.spacy"
|
|
- "corpus/perdt-ner/dev.spacy"
|
|
- "configs/fa_ner_sm.cfg"
|
|
outputs:
|
|
- "training/perdt-ner/model-best"
|
|
|
|
- name: "finalize-dep"
|
|
help: "Write fa_dep_news_sm metadata (sources, licence, notes) onto the trained model"
|
|
script:
|
|
- "python scripts/finalize_pipeline.py training/dep/model-best training/fa_dep_news_sm --variant dep --version ${vars.package_version}"
|
|
deps:
|
|
- "training/dep/model-best"
|
|
- "scripts/finalize_pipeline.py"
|
|
outputs:
|
|
- "training/fa_dep_news_sm"
|
|
|
|
- name: "evaluate-dep"
|
|
help: "Score fa_dep_news_sm on the held-out UD test split"
|
|
script:
|
|
- "python -m spacy benchmark accuracy training/fa_dep_news_sm corpus/merged/${vars.treebank}-ud-test.spacy --output metrics/ud-test.json --gpu-id ${vars.gpu}"
|
|
deps:
|
|
- "training/fa_dep_news_sm"
|
|
- "corpus/merged/${vars.treebank}-ud-test.spacy"
|
|
outputs:
|
|
- "metrics/ud-test.json"
|
|
|
|
- name: "assemble-core"
|
|
help: >
|
|
Source the trained ner into the dep pipeline to produce fa_core_news_sm. Possible
|
|
because configs/fa_ner_sm.cfg embeds its own tok2vec instead of a Tok2VecListener.
|
|
script:
|
|
- "python scripts/finalize_pipeline.py training/dep/model-best training/fa_core_news_sm --variant core --version ${vars.package_version} --add-ner training/perdt-ner/model-best"
|
|
deps:
|
|
- "training/dep/model-best"
|
|
- "training/perdt-ner/model-best"
|
|
- "scripts/finalize_pipeline.py"
|
|
outputs:
|
|
- "training/fa_core_news_sm"
|
|
|
|
- name: "evaluate-core"
|
|
help: "Score fa_core_news_sm on both held-out test splits"
|
|
script:
|
|
- "python -m spacy benchmark accuracy training/fa_core_news_sm corpus/merged/${vars.treebank}-ud-test.spacy --output metrics/core-ud-test.json --gpu-id ${vars.gpu}"
|
|
- "python -m spacy benchmark accuracy training/fa_core_news_sm corpus/perdt-ner/test.spacy --output metrics/perdt-ner-test.json --gpu-id ${vars.gpu}"
|
|
deps:
|
|
- "training/fa_core_news_sm"
|
|
- "corpus/merged/${vars.treebank}-ud-test.spacy"
|
|
- "corpus/perdt-ner/test.spacy"
|
|
outputs:
|
|
- "metrics/core-ud-test.json"
|
|
- "metrics/perdt-ner-test.json"
|
|
|
|
- name: "finalize-meta"
|
|
help: >
|
|
Re-run finalize on both packages, folding test scores into meta.json["performance"].
|
|
Separate because the scores only exist after evaluation, and evaluation needs a
|
|
finalized pipeline to score. Cheap: it only copies models.
|
|
script:
|
|
- "python scripts/finalize_pipeline.py training/dep/model-best training/fa_dep_news_sm --variant dep --version ${vars.package_version} --ud-metrics metrics/ud-test.json"
|
|
- "python scripts/finalize_pipeline.py training/dep/model-best training/fa_core_news_sm --variant core --version ${vars.package_version} --add-ner training/perdt-ner/model-best --ud-metrics metrics/core-ud-test.json --ner-metrics metrics/perdt-ner-test.json"
|
|
deps:
|
|
- "metrics/ud-test.json"
|
|
- "metrics/perdt-ner-test.json"
|
|
- "scripts/finalize_pipeline.py"
|
|
|
|
- name: "package"
|
|
help: "Build installable wheels + sdists for both shipping packages"
|
|
script:
|
|
- "python -m spacy package training/fa_dep_news_sm packages --name ${vars.dep_package_name} --version ${vars.package_version} --build sdist,wheel --force"
|
|
- "python -m spacy package training/fa_core_news_sm packages --name ${vars.core_package_name} --version ${vars.package_version} --build sdist,wheel --force"
|
|
deps:
|
|
- "training/fa_dep_news_sm"
|
|
- "training/fa_core_news_sm"
|
|
outputs:
|
|
- "packages/${vars.lang}_${vars.dep_package_name}-${vars.package_version}"
|
|
- "packages/${vars.lang}_${vars.core_package_name}-${vars.package_version}"
|
|
|
|
- name: "smoke"
|
|
help: "Load both pipelines and run them over real Persian text"
|
|
script:
|
|
- "python scripts/smoke_test.py training/fa_dep_news_sm"
|
|
- "python scripts/smoke_test.py training/fa_core_news_sm"
|
|
deps:
|
|
- "training/fa_dep_news_sm"
|
|
- "training/fa_core_news_sm"
|
|
|
|
- name: "finalize-ent"
|
|
help: "Write fa_ent_news_sm metadata onto the trained NER model"
|
|
script:
|
|
- "python scripts/finalize_pipeline.py training/perdt-ner/model-best training/fa_ent_news_sm --variant ent --version ${vars.package_version}"
|
|
deps:
|
|
- "training/perdt-ner/model-best"
|
|
- "scripts/finalize_pipeline.py"
|
|
outputs:
|
|
- "training/fa_ent_news_sm"
|
|
|
|
- name: "evaluate-ent"
|
|
help: "Score fa_ent_news_sm on the held-out PerDT NER test split"
|
|
script:
|
|
- "python -m spacy benchmark accuracy training/fa_ent_news_sm corpus/perdt-ner/test.spacy --output metrics/ent-test.json --gpu-id ${vars.gpu}"
|
|
- "python scripts/finalize_pipeline.py training/perdt-ner/model-best training/fa_ent_news_sm --variant ent --version ${vars.package_version} --ner-metrics metrics/ent-test.json"
|
|
deps:
|
|
- "training/fa_ent_news_sm"
|
|
- "corpus/perdt-ner/test.spacy"
|
|
outputs:
|
|
- "metrics/ent-test.json"
|
|
|
|
- name: "package-ent"
|
|
help: "Build the installable fa_ent_news_sm wheel + sdist"
|
|
script:
|
|
- "python -m spacy package training/fa_ent_news_sm packages --name ${vars.ent_package_name} --version ${vars.package_version} --build sdist,wheel --force"
|
|
deps:
|
|
- "training/fa_ent_news_sm"
|
|
outputs:
|
|
- "packages/${vars.lang}_${vars.ent_package_name}-${vars.package_version}"
|
|
|
|
- name: "clean"
|
|
help: "Drop corpora, training runs and metrics (keeps downloaded assets)"
|
|
script:
|
|
- "rm -rf corpus training metrics packages"
|