fazel
/
spacy-fa-pipeline
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
spacy-fa-pipeline/project.yml

713 lines
32 KiB
YAML
Raw Blame History

title: "fa_core_news_sm / fa_dep_news_sm"
description: >
CPU-sized Persian (fa) pipelines for spaCy 3.8, built entirely from UD_Persian-PerDT
(PerUDT v1.0, CC BY-SA 4.0): tagger (XPOS), morphologizer (UPOS + FEATS), trainable
lemmatizer, dependency parser and NER.
Two shipping packages, same corpus, differing only in whether NER is included:
`fa_dep_news_sm` (no NER) and `fa_core_news_sm` (with NER). The NER comes from the
treebank's own `not-to-release/Dadegan with NER tag/` layer, so both packages share one
corpus, one genre, one tokenization and one licence.
PerDT's NER labels are silver, produced by Beheshti-NER with manual corrections, and are
transferred onto this pipeline's tokenization by difflib at a 99.86% rate. Per-label
scores are published in meta.json; MON, TIM and PCT are thin.
See docs/MODELS.md for the source and licence analysis and docs/CONTRIBUTING-GUIDE.md for
how this gets published.
Build both with: `spacy project run all`
Standalone NER-only package: `spacy project run ent`
vars:
lang: "fa"
dep_package_name: "dep_news_sm"
core_package_name: "core_news_sm"
ent_package_name: "ent_news_sm"
package_version: "3.8.0"
treebank: "fa_perdt"
# -1 = CPU. A GTX 940MX (2 GB) is not worth the transfer overhead for an sm pipeline.
gpu: -1
n_sents: 10
# md tier. Same architecture as sm plus the fa_floret static vector table.
dep_md_package_name: "dep_news_md"
core_md_package_name: "core_news_md"
floret_wheel: "fa_floret-0.1.0-py3-none-any-400k-documents.whl"
vectors_dir: "assets/vectors/fa_floret_400k"
# lg tier: same architecture as sm/md, larger floret table (200k rows x 300d, trained on
# the full Persian Wikipedia dump for 5 epochs, vs md's 50k rows / 400k documents).
ent_lg_package_name: "ent_news_lg"
dep_lg_package_name: "dep_news_lg"
core_lg_package_name: "core_news_lg"
floret_lg_wheel: "fa_floret-0.1.0-py3-none-any-full-wiki-200k-5epoch.whl"
vectors_lg_dir: "assets/vectors/fa_floret_lg"
# trf tier: one fine-tuned ParsBERT shared by every component. Needs a real GPU; the
# 940MX cannot fine-tune a 162M-parameter encoder, so `gpu_trf` is set for a rented card.
core_trf_package_name: "core_news_trf"
gpu_trf: 0
directories:
- "assets"
- "corpus"
- "configs"
- "scripts"
- "training"
- "metrics"
- "packages"
assets:
- dest: "assets/ud/fa_perdt-ud-train.conllu"
url: "https://raw.githubusercontent.com/UniversalDependencies/UD_Persian-PerDT/master/fa_perdt-ud-train.conllu"
checksum: "f5a8ba901a776b4fd1941ecadcc6d506"
description: "UD_Persian-PerDT train split (CC BY-SA 4.0)"
- dest: "assets/ud/fa_perdt-ud-dev.conllu"
url: "https://raw.githubusercontent.com/UniversalDependencies/UD_Persian-PerDT/master/fa_perdt-ud-dev.conllu"
checksum: "f103020da7c1e917aafb8a8321f4cb84"
description: "UD_Persian-PerDT dev split (CC BY-SA 4.0)"
- dest: "assets/ud/fa_perdt-ud-test.conllu"
url: "https://raw.githubusercontent.com/UniversalDependencies/UD_Persian-PerDT/master/fa_perdt-ud-test.conllu"
checksum: "b62a66994cef2c50f7e524a1471102d8"
description: "UD_Persian-PerDT test split (CC BY-SA 4.0)"
- dest: "assets/ud-ner/train_with_NER_tag.txt"
url: "https://raw.githubusercontent.com/UniversalDependencies/UD_Persian-PerDT/master/not-to-release/Dadegan%20with%20NER%20tag/train_with_NER_tag.txt"
checksum: "ecb96cf99b38bc485cac21d22914e413"
description: "PerDT NER layer, train split, IOB2 (CC BY-SA 4.0)"
- dest: "assets/ud-ner/dev_with_NER_tag.txt"
url: "https://raw.githubusercontent.com/UniversalDependencies/UD_Persian-PerDT/master/not-to-release/Dadegan%20with%20NER%20tag/dev_with_NER_tag.txt"
checksum: "2a56ef7eb2e3732e221317af457d1c09"
description: "PerDT NER layer, dev split, IOB2 (CC BY-SA 4.0)"
- dest: "assets/ud-ner/test_with_NER_tag.txt"
url: "https://raw.githubusercontent.com/UniversalDependencies/UD_Persian-PerDT/master/not-to-release/Dadegan%20with%20NER%20tag/test_with_NER_tag.txt"
checksum: "6d80dd783527562c2ea5189f218a12b5"
description: "PerDT NER layer, test split, IOB2 (CC BY-SA 4.0)"
workflows:
# Both shipping artifacts: fa_dep_news_sm and fa_core_news_sm.
all:
- inspect
- convert-ud
- transfer-ner
- convert-ner
- debug-data
- debug-data-ner
- train-dep
- train-ner
- finalize-dep
- evaluate-dep
- assemble-core
- evaluate-core
- finalize-meta
- package
- smoke
# Optional third artifact: the NER alone, for users who only want entities.
ent:
- finalize-ent
- evaluate-ent
- package-ent
# The lg tier: same corpus and architecture as sm/md, with a bigger floret table (200k
# rows, full Persian Wikipedia, 5 epochs) than md's (50k rows, 400k documents).
lg:
- vectors-lg
- train-dep-lg
- train-ner-lg
- finalize-dep-lg
- assemble-core-lg
- evaluate-lg
- finalize-meta-lg
- compare-lg
- package-lg
- smoke-lg
# The md tier: same corpus and architecture, plus the fa_floret static vectors.
md:
- vectors-md
- train-dep-md
- train-ner-md
- finalize-dep-md
- assemble-core-md
- evaluate-md
- finalize-meta-md
- compare-md
- package-md
- smoke-md
# The trf tier: one fine-tuned ParsBERT shared by every component, including ner, so it
# trains against a single joint corpus instead of the sm/md/lg dep+ner split. GPU only.
trf:
- merge-joint
- debug-data-trf
- train-trf
- finalize-trf
- evaluate-trf
- finalize-meta-trf
- package-trf
- smoke-trf
commands:
- name: "inspect"
help: "Report annotation coverage of the downloaded treebank(s)"
script:
- "python scripts/inspect_treebanks.py assets/ud"
deps:
- "assets/ud/fa_perdt-ud-train.conllu"
- "scripts/inspect_treebanks.py"
- name: "convert-ud"
help: >
CoNLL-U -> DocBin. --merge-subtokens fuses multiword-token clitics into single
tokens; docs/MODELS.md §5 has the measurement that justifies it.
script:
- "python -m spacy convert assets/ud/${vars.treebank}-ud-train.conllu corpus/merged --converter conllu --n-sents ${vars.n_sents} --merge-subtokens"
- "python -m spacy convert assets/ud/${vars.treebank}-ud-dev.conllu corpus/merged --converter conllu --n-sents ${vars.n_sents} --merge-subtokens"
- "python -m spacy convert assets/ud/${vars.treebank}-ud-test.conllu corpus/merged --converter conllu --n-sents ${vars.n_sents} --merge-subtokens"
# Reproduce the measurement in docs/MODELS.md §5: convert the dev split WITHOUT
# merging and compare both against the tokenizer we actually ship.
- "python -m spacy convert assets/ud/${vars.treebank}-ud-dev.conllu corpus/split --converter conllu --n-sents ${vars.n_sents}"
- "python scripts/tokenization_report.py corpus/merged/${vars.treebank}-ud-dev.spacy corpus/split/${vars.treebank}-ud-dev.spacy"
deps:
- "assets/ud/${vars.treebank}-ud-train.conllu"
- "assets/ud/${vars.treebank}-ud-dev.conllu"
- "assets/ud/${vars.treebank}-ud-test.conllu"
outputs:
- "corpus/merged/${vars.treebank}-ud-train.spacy"
- "corpus/merged/${vars.treebank}-ud-dev.spacy"
- "corpus/merged/${vars.treebank}-ud-test.spacy"
- name: "transfer-ner"
help: >
Align PerDT's NER layer onto the --merge-subtokens tokenization. The NER files use the
original Dadegan tokenization, which matches the released UD tokenization in only 57 to
62% of sentences, so spans are transferred by difflib. Measured rate 99.86%; spans that
cannot be aligned exactly are dropped rather than guessed.
script:
- "python scripts/transfer_perdt_ner.py --conllu-dir assets/ud --ner-dir assets/ud-ner --out corpus/perdt-ner-iob"
deps:
- "assets/ud/${vars.treebank}-ud-train.conllu"
- "assets/ud-ner/train_with_NER_tag.txt"
- "scripts/transfer_perdt_ner.py"
outputs:
- "corpus/perdt-ner-iob/train.txt"
- "corpus/perdt-ner-iob/dev.txt"
- "corpus/perdt-ner-iob/test.txt"
- name: "convert-ner"
help: "Transferred IOB2 -> DocBin"
script:
- "python -m spacy convert corpus/perdt-ner-iob/train.txt corpus/perdt-ner --converter ner --n-sents ${vars.n_sents} --lang ${vars.lang}"
- "python -m spacy convert corpus/perdt-ner-iob/dev.txt corpus/perdt-ner --converter ner --n-sents ${vars.n_sents} --lang ${vars.lang}"
- "python -m spacy convert corpus/perdt-ner-iob/test.txt corpus/perdt-ner --converter ner --n-sents ${vars.n_sents} --lang ${vars.lang}"
deps:
- "corpus/perdt-ner-iob/train.txt"
outputs:
- "corpus/perdt-ner/train.spacy"
- "corpus/perdt-ner/dev.spacy"
- "corpus/perdt-ner/test.spacy"
- name: "debug-data"
help: "Validate the treebank against the config before burning CPU on training"
script:
- "python -m spacy debug data configs/fa_dep_news_sm.cfg --paths.train corpus/merged/${vars.treebank}-ud-train.spacy --paths.dev corpus/merged/${vars.treebank}-ud-dev.spacy"
deps:
- "corpus/merged/${vars.treebank}-ud-train.spacy"
- "configs/fa_dep_news_sm.cfg"
- name: "debug-data-ner"
help: "Validate the transferred PerDT NER corpus against the NER config"
script:
- "python -m spacy debug data configs/fa_ner_sm.cfg --paths.train corpus/perdt-ner/train.spacy --paths.dev corpus/perdt-ner/dev.spacy"
deps:
- "corpus/perdt-ner/train.spacy"
- "configs/fa_ner_sm.cfg"
- name: "train-dep"
help: "Train tok2vec + tagger + morphologizer + trainable_lemmatizer + parser on PerDT"
script:
- "python -m spacy train configs/fa_dep_news_sm.cfg --output training/dep --paths.train corpus/merged/${vars.treebank}-ud-train.spacy --paths.dev corpus/merged/${vars.treebank}-ud-dev.spacy --gpu-id ${vars.gpu}"
deps:
- "corpus/merged/${vars.treebank}-ud-train.spacy"
- "corpus/merged/${vars.treebank}-ud-dev.spacy"
- "configs/fa_dep_news_sm.cfg"
outputs:
- "training/dep/model-best"
- name: "train-ner"
help: "Train the NER component (own embedded tok2vec) on the transferred PerDT layer"
script:
- "python -m spacy train configs/fa_ner_sm.cfg --output training/perdt-ner --paths.train corpus/perdt-ner/train.spacy --paths.dev corpus/perdt-ner/dev.spacy --gpu-id ${vars.gpu}"
deps:
- "corpus/perdt-ner/train.spacy"
- "corpus/perdt-ner/dev.spacy"
- "configs/fa_ner_sm.cfg"
outputs:
- "training/perdt-ner/model-best"
- name: "finalize-dep"
help: "Write fa_dep_news_sm metadata (sources, licence, notes) onto the trained model"
script:
- "python scripts/finalize_pipeline.py training/dep/model-best training/fa_dep_news_sm --variant dep --version ${vars.package_version}"
deps:
- "training/dep/model-best"
- "scripts/finalize_pipeline.py"
outputs:
- "training/fa_dep_news_sm"
- name: "evaluate-dep"
help: "Score fa_dep_news_sm on the held-out UD test split"
script:
- "python -m spacy benchmark accuracy training/fa_dep_news_sm corpus/merged/${vars.treebank}-ud-test.spacy --output metrics/ud-test.json --gpu-id ${vars.gpu}"
deps:
- "training/fa_dep_news_sm"
- "corpus/merged/${vars.treebank}-ud-test.spacy"
outputs:
- "metrics/ud-test.json"
- name: "assemble-core"
help: >
Source the trained ner into the dep pipeline to produce fa_core_news_sm. Possible
because configs/fa_ner_sm.cfg embeds its own tok2vec instead of a Tok2VecListener.
script:
- "python scripts/finalize_pipeline.py training/dep/model-best training/fa_core_news_sm --variant core --version ${vars.package_version} --add-ner training/perdt-ner/model-best"
deps:
- "training/dep/model-best"
- "training/perdt-ner/model-best"
- "scripts/finalize_pipeline.py"
outputs:
- "training/fa_core_news_sm"
- name: "evaluate-core"
help: "Score fa_core_news_sm on both held-out test splits"
script:
- "python -m spacy benchmark accuracy training/fa_core_news_sm corpus/merged/${vars.treebank}-ud-test.spacy --output metrics/core-ud-test.json --gpu-id ${vars.gpu}"
- "python -m spacy benchmark accuracy training/fa_core_news_sm corpus/perdt-ner/test.spacy --output metrics/perdt-ner-test.json --gpu-id ${vars.gpu}"
deps:
- "training/fa_core_news_sm"
- "corpus/merged/${vars.treebank}-ud-test.spacy"
- "corpus/perdt-ner/test.spacy"
outputs:
- "metrics/core-ud-test.json"
- "metrics/perdt-ner-test.json"
- name: "finalize-meta"
help: >
Re-run finalize on both packages, folding test scores into meta.json["performance"].
Separate because the scores only exist after evaluation, and evaluation needs a
finalized pipeline to score. Cheap: it only copies models.
script:
- "python scripts/finalize_pipeline.py training/dep/model-best training/fa_dep_news_sm --variant dep --version ${vars.package_version} --ud-metrics metrics/ud-test.json"
- "python scripts/finalize_pipeline.py training/dep/model-best training/fa_core_news_sm --variant core --version ${vars.package_version} --add-ner training/perdt-ner/model-best --ud-metrics metrics/core-ud-test.json --ner-metrics metrics/perdt-ner-test.json"
deps:
- "metrics/ud-test.json"
- "metrics/perdt-ner-test.json"
- "scripts/finalize_pipeline.py"
- name: "package"
help: "Build installable wheels + sdists for both shipping packages"
script:
- "python -m spacy package training/fa_dep_news_sm packages --name ${vars.dep_package_name} --version ${vars.package_version} --build sdist,wheel --force"
- "python -m spacy package training/fa_core_news_sm packages --name ${vars.core_package_name} --version ${vars.package_version} --build sdist,wheel --force"
deps:
- "training/fa_dep_news_sm"
- "training/fa_core_news_sm"
outputs:
- "packages/${vars.lang}_${vars.dep_package_name}-${vars.package_version}"
- "packages/${vars.lang}_${vars.core_package_name}-${vars.package_version}"
- name: "smoke"
help: "Load both pipelines and run them over real Persian text"
script:
- "python scripts/smoke_test.py training/fa_dep_news_sm"
- "python scripts/smoke_test.py training/fa_core_news_sm"
deps:
- "training/fa_dep_news_sm"
- "training/fa_core_news_sm"
- name: "finalize-ent"
help: "Write fa_ent_news_sm metadata onto the trained NER model"
script:
- "python scripts/finalize_pipeline.py training/perdt-ner/model-best training/fa_ent_news_sm --variant ent --version ${vars.package_version}"
deps:
- "training/perdt-ner/model-best"
- "scripts/finalize_pipeline.py"
outputs:
- "training/fa_ent_news_sm"
- name: "evaluate-ent"
help: "Score fa_ent_news_sm on the held-out PerDT NER test split"
script:
- "python -m spacy benchmark accuracy training/fa_ent_news_sm corpus/perdt-ner/test.spacy --output metrics/ent-test.json --gpu-id ${vars.gpu}"
- "python scripts/finalize_pipeline.py training/perdt-ner/model-best training/fa_ent_news_sm --variant ent --version ${vars.package_version} --ner-metrics metrics/ent-test.json"
deps:
- "training/fa_ent_news_sm"
- "corpus/perdt-ner/test.spacy"
outputs:
- "metrics/ent-test.json"
- name: "package-ent"
help: "Build the installable fa_ent_news_sm wheel + sdist"
script:
- "python -m spacy package training/fa_ent_news_sm packages --name ${vars.ent_package_name} --version ${vars.package_version} --build sdist,wheel --force"
deps:
- "training/fa_ent_news_sm"
outputs:
- "packages/${vars.lang}_${vars.ent_package_name}-${vars.package_version}"
# ---------------------------------------------------------------- lg tier (ner only)
- name: "vectors-lg"
help: >
Unpack the lg-tier fa_floret wheel into a plain spaCy model directory. 200k rows x
300d in floret mode, trained on the full Persian Wikipedia dump for 5 epochs, vs
vectors-md's 50k rows / 400k documents.
script:
- "python scripts/unpack_vectors.py ${vars.floret_lg_wheel} ${vars.vectors_lg_dir}"
deps:
- "${vars.floret_lg_wheel}"
- "scripts/unpack_vectors.py"
outputs:
- "${vars.vectors_lg_dir}"
- name: "train-dep-lg"
help: "Train the dep pipeline with the lg-tier static floret vectors"
script:
- "python -m spacy train configs/fa_dep_news_lg.cfg --output training/dep-lg --paths.train corpus/merged/${vars.treebank}-ud-train.spacy --paths.dev corpus/merged/${vars.treebank}-ud-dev.spacy --paths.vectors ${vars.vectors_lg_dir} --gpu-id ${vars.gpu}"
deps:
- "corpus/merged/${vars.treebank}-ud-train.spacy"
- "corpus/merged/${vars.treebank}-ud-dev.spacy"
- "configs/fa_dep_news_lg.cfg"
- "${vars.vectors_lg_dir}"
outputs:
- "training/dep-lg/model-best"
- name: "train-ner-lg"
help: "Train the NER component with the lg-tier static floret vectors"
script:
- "python -m spacy train configs/fa_ner_lg.cfg --output training/perdt-ner-lg --paths.train corpus/perdt-ner/train.spacy --paths.dev corpus/perdt-ner/dev.spacy --paths.vectors ${vars.vectors_lg_dir} --gpu-id ${vars.gpu}"
deps:
- "corpus/perdt-ner/train.spacy"
- "corpus/perdt-ner/dev.spacy"
- "configs/fa_ner_lg.cfg"
- "${vars.vectors_lg_dir}"
outputs:
- "training/perdt-ner-lg/model-best"
- name: "finalize-ent-lg"
help: "Write fa_ent_news_lg metadata onto the trained lg model"
script:
- "python scripts/finalize_pipeline.py training/perdt-ner-lg/model-best training/fa_ent_news_lg --variant ent --size lg --version ${vars.package_version}"
deps:
- "training/perdt-ner-lg/model-best"
- "scripts/finalize_pipeline.py"
outputs:
- "training/fa_ent_news_lg"
- name: "finalize-dep-lg"
help: "Write fa_dep_news_lg metadata onto the trained lg model"
script:
- "python scripts/finalize_pipeline.py training/dep-lg/model-best training/fa_dep_news_lg --variant dep --size lg --version ${vars.package_version}"
deps:
- "training/dep-lg/model-best"
- "scripts/finalize_pipeline.py"
outputs:
- "training/fa_dep_news_lg"
- name: "assemble-core-lg"
help: "Source the lg ner into the lg dep pipeline to produce fa_core_news_lg"
script:
- "python scripts/finalize_pipeline.py training/dep-lg/model-best training/fa_core_news_lg --variant core --size lg --version ${vars.package_version} --add-ner training/perdt-ner-lg/model-best"
deps:
- "training/dep-lg/model-best"
- "training/perdt-ner-lg/model-best"
- "scripts/finalize_pipeline.py"
outputs:
- "training/fa_core_news_lg"
- name: "evaluate-ent-lg"
help: "Score fa_ent_news_lg on the held-out PerDT NER test split"
script:
- "python -m spacy benchmark accuracy training/fa_ent_news_lg corpus/perdt-ner/test.spacy --output metrics/lg-perdt-ner-test.json --gpu-id ${vars.gpu}"
- "python scripts/finalize_pipeline.py training/perdt-ner-lg/model-best training/fa_ent_news_lg --variant ent --size lg --version ${vars.package_version} --ner-metrics metrics/lg-perdt-ner-test.json"
deps:
- "training/fa_ent_news_lg"
- "corpus/perdt-ner/test.spacy"
outputs:
- "metrics/lg-perdt-ner-test.json"
- name: "evaluate-lg"
help: "Score both lg packages (dep, core) on the held-out test splits"
script:
- "python -m spacy benchmark accuracy training/fa_dep_news_lg corpus/merged/${vars.treebank}-ud-test.spacy --output metrics/lg-ud-test.json --gpu-id ${vars.gpu}"
- "python -m spacy benchmark accuracy training/fa_core_news_lg corpus/merged/${vars.treebank}-ud-test.spacy --output metrics/lg-core-ud-test.json --gpu-id ${vars.gpu}"
- "python -m spacy benchmark accuracy training/fa_core_news_lg corpus/perdt-ner/test.spacy --output metrics/lg-core-perdt-ner-test.json --gpu-id ${vars.gpu}"
deps:
- "training/fa_dep_news_lg"
- "training/fa_core_news_lg"
outputs:
- "metrics/lg-ud-test.json"
- "metrics/lg-core-ud-test.json"
- "metrics/lg-core-perdt-ner-test.json"
- name: "finalize-meta-lg"
help: "Fold the lg test scores into both lg meta.json files"
script:
- "python scripts/finalize_pipeline.py training/dep-lg/model-best training/fa_dep_news_lg --variant dep --size lg --version ${vars.package_version} --ud-metrics metrics/lg-ud-test.json"
- "python scripts/finalize_pipeline.py training/dep-lg/model-best training/fa_core_news_lg --variant core --size lg --version ${vars.package_version} --add-ner training/perdt-ner-lg/model-best --ud-metrics metrics/lg-core-ud-test.json --ner-metrics metrics/lg-core-perdt-ner-test.json"
deps:
- "metrics/lg-ud-test.json"
- "metrics/lg-core-perdt-ner-test.json"
- "scripts/finalize_pipeline.py"
- name: "compare-lg"
help: "Table the sm vs md vs lg deltas from the metrics/ JSON reports"
script:
- "python scripts/compare_tiers.py"
deps:
- "metrics/ud-test.json"
- "metrics/md-ud-test.json"
- "metrics/lg-ud-test.json"
- "metrics/perdt-ner-test.json"
- "metrics/md-perdt-ner-test.json"
- "metrics/lg-perdt-ner-test.json"
- "scripts/compare_tiers.py"
- name: "package-ent-lg"
help: "Build the installable fa_ent_news_lg wheel + sdist"
script:
- "python -m spacy package training/fa_ent_news_lg packages --name ${vars.ent_lg_package_name} --version ${vars.package_version} --build sdist,wheel --force"
deps:
- "training/fa_ent_news_lg"
outputs:
- "packages/${vars.lang}_${vars.ent_lg_package_name}-${vars.package_version}"
- name: "package-lg"
help: "Build installable wheels + sdists for both lg packages"
script:
- "python -m spacy package training/fa_dep_news_lg packages --name ${vars.dep_lg_package_name} --version ${vars.package_version} --build sdist,wheel --force"
- "python -m spacy package training/fa_core_news_lg packages --name ${vars.core_lg_package_name} --version ${vars.package_version} --build sdist,wheel --force"
deps:
- "training/fa_dep_news_lg"
- "training/fa_core_news_lg"
outputs:
- "packages/${vars.lang}_${vars.dep_lg_package_name}-${vars.package_version}"
- "packages/${vars.lang}_${vars.core_lg_package_name}-${vars.package_version}"
- name: "smoke-ent-lg"
help: "Load fa_ent_news_lg and run it over real Persian text"
script:
- "python scripts/smoke_test.py training/fa_ent_news_lg"
deps:
- "training/fa_ent_news_lg"
- name: "smoke-lg"
help: "Load both lg pipelines and run them over real Persian text"
script:
- "python scripts/smoke_test.py training/fa_dep_news_lg"
- "python scripts/smoke_test.py training/fa_core_news_lg"
deps:
- "training/fa_dep_news_lg"
- "training/fa_core_news_lg"
# ---------------------------------------------------------------- md tier
- name: "vectors-md"
help: >
Unpack the fa_floret wheel into a plain spaCy model directory that
`--paths.vectors` can point at. The wheel is a vectors-only pipeline
(empty `pipeline: []`), 50k rows x 300d in floret mode, trained on 400k
Persian documents, so no `spacy init vectors` step is needed.
script:
- "python scripts/unpack_vectors.py ${vars.floret_wheel} ${vars.vectors_dir}"
deps:
- "${vars.floret_wheel}"
- "scripts/unpack_vectors.py"
outputs:
- "${vars.vectors_dir}"
- name: "train-dep-md"
help: "Train the dep pipeline with static floret vectors"
script:
- "python -m spacy train configs/fa_dep_news_md.cfg --output training/dep-md --paths.train corpus/merged/${vars.treebank}-ud-train.spacy --paths.dev corpus/merged/${vars.treebank}-ud-dev.spacy --paths.vectors ${vars.vectors_dir} --gpu-id ${vars.gpu}"
deps:
- "corpus/merged/${vars.treebank}-ud-train.spacy"
- "corpus/merged/${vars.treebank}-ud-dev.spacy"
- "configs/fa_dep_news_md.cfg"
- "${vars.vectors_dir}"
outputs:
- "training/dep-md/model-best"
- name: "train-ner-md"
help: "Train the NER component with static floret vectors"
script:
- "python -m spacy train configs/fa_ner_md.cfg --output training/perdt-ner-md --paths.train corpus/perdt-ner/train.spacy --paths.dev corpus/perdt-ner/dev.spacy --paths.vectors ${vars.vectors_dir} --gpu-id ${vars.gpu}"
deps:
- "corpus/perdt-ner/train.spacy"
- "corpus/perdt-ner/dev.spacy"
- "configs/fa_ner_md.cfg"
- "${vars.vectors_dir}"
outputs:
- "training/perdt-ner-md/model-best"
- name: "finalize-dep-md"
help: "Write fa_dep_news_md metadata onto the trained md model"
script:
- "python scripts/finalize_pipeline.py training/dep-md/model-best training/fa_dep_news_md --variant dep --size md --version ${vars.package_version}"
deps:
- "training/dep-md/model-best"
- "scripts/finalize_pipeline.py"
outputs:
- "training/fa_dep_news_md"
- name: "assemble-core-md"
help: "Source the md ner into the md dep pipeline to produce fa_core_news_md"
script:
- "python scripts/finalize_pipeline.py training/dep-md/model-best training/fa_core_news_md --variant core --size md --version ${vars.package_version} --add-ner training/perdt-ner-md/model-best"
deps:
- "training/dep-md/model-best"
- "training/perdt-ner-md/model-best"
- "scripts/finalize_pipeline.py"
outputs:
- "training/fa_core_news_md"
- name: "evaluate-md"
help: "Score both md packages on the held-out test splits"
script:
- "python -m spacy benchmark accuracy training/fa_dep_news_md corpus/merged/${vars.treebank}-ud-test.spacy --output metrics/md-ud-test.json --gpu-id ${vars.gpu}"
- "python -m spacy benchmark accuracy training/fa_core_news_md corpus/merged/${vars.treebank}-ud-test.spacy --output metrics/md-core-ud-test.json --gpu-id ${vars.gpu}"
- "python -m spacy benchmark accuracy training/fa_core_news_md corpus/perdt-ner/test.spacy --output metrics/md-perdt-ner-test.json --gpu-id ${vars.gpu}"
deps:
- "training/fa_dep_news_md"
- "training/fa_core_news_md"
outputs:
- "metrics/md-ud-test.json"
- "metrics/md-core-ud-test.json"
- "metrics/md-perdt-ner-test.json"
- name: "finalize-meta-md"
help: "Fold the md test scores into both md meta.json files"
script:
- "python scripts/finalize_pipeline.py training/dep-md/model-best training/fa_dep_news_md --variant dep --size md --version ${vars.package_version} --ud-metrics metrics/md-ud-test.json"
- "python scripts/finalize_pipeline.py training/dep-md/model-best training/fa_core_news_md --variant core --size md --version ${vars.package_version} --add-ner training/perdt-ner-md/model-best --ud-metrics metrics/md-core-ud-test.json --ner-metrics metrics/md-perdt-ner-test.json"
deps:
- "metrics/md-ud-test.json"
- "metrics/md-perdt-ner-test.json"
- "scripts/finalize_pipeline.py"
- name: "compare-md"
help: "Table the sm vs md deltas from the metrics/ JSON reports"
script:
- "python scripts/compare_tiers.py"
deps:
- "metrics/md-ud-test.json"
- "metrics/md-perdt-ner-test.json"
- "scripts/compare_tiers.py"
- name: "package-md"
help: "Build installable wheels + sdists for both md packages"
script:
- "python -m spacy package training/fa_dep_news_md packages --name ${vars.dep_md_package_name} --version ${vars.package_version} --build sdist,wheel --force"
- "python -m spacy package training/fa_core_news_md packages --name ${vars.core_md_package_name} --version ${vars.package_version} --build sdist,wheel --force"
deps:
- "training/fa_dep_news_md"
- "training/fa_core_news_md"
outputs:
- "packages/${vars.lang}_${vars.dep_md_package_name}-${vars.package_version}"
- "packages/${vars.lang}_${vars.core_md_package_name}-${vars.package_version}"
- name: "smoke-md"
help: "Load both md pipelines and run them over real Persian text"
script:
- "python scripts/smoke_test.py training/fa_dep_news_md"
- "python scripts/smoke_test.py training/fa_core_news_md"
deps:
- "training/fa_dep_news_md"
- "training/fa_core_news_md"
# ---------------------------------------------------------------- trf tier
- name: "merge-joint"
help: >
Fuse the UD layer and the transferred NER layer onto one set of Docs. The trf tier
shares a single transformer across every component, so it needs one corpus carrying
both annotation layers; the two DocBins are token-for-token identical by construction
and the script asserts it.
script:
- "python scripts/merge_joint_corpus.py --ud-dir corpus/merged --ner-dir corpus/perdt-ner --out corpus/joint"
deps:
- "corpus/merged/${vars.treebank}-ud-train.spacy"
- "corpus/perdt-ner/train.spacy"
- "scripts/merge_joint_corpus.py"
outputs:
- "corpus/joint/train.spacy"
- "corpus/joint/dev.spacy"
- "corpus/joint/test.spacy"
- name: "debug-data-trf"
help: "Validate the joint corpus against the trf config before renting GPU time"
script:
- "python -m spacy debug data configs/fa_core_news_trf.cfg --paths.train corpus/joint/train.spacy --paths.dev corpus/joint/dev.spacy"
deps:
- "corpus/joint/train.spacy"
- "configs/fa_core_news_trf.cfg"
- name: "train-trf"
help: "Fine-tune ParsBERT with tagger + morphologizer + lemmatizer + parser + ner listening"
script:
- "python -m spacy train configs/fa_core_news_trf.cfg --output training/core-trf --paths.train corpus/joint/train.spacy --paths.dev corpus/joint/dev.spacy --gpu-id ${vars.gpu_trf}"
deps:
- "corpus/joint/train.spacy"
- "corpus/joint/dev.spacy"
- "configs/fa_core_news_trf.cfg"
outputs:
- "training/core-trf/model-best"
- name: "finalize-trf"
help: "Write fa_core_news_trf metadata onto the trained model"
script:
- "python scripts/finalize_pipeline.py training/core-trf/model-best training/fa_core_news_trf --variant core --size trf --version ${vars.package_version}"
deps:
- "training/core-trf/model-best"
- "scripts/finalize_pipeline.py"
outputs:
- "training/fa_core_news_trf"
- name: "evaluate-trf"
help: "Score fa_core_news_trf on the held-out UD and NER test splits"
script:
- "python -m spacy benchmark accuracy training/fa_core_news_trf corpus/merged/${vars.treebank}-ud-test.spacy --output metrics/trf-core-ud-test.json --gpu-id ${vars.gpu_trf}"
- "python -m spacy benchmark accuracy training/fa_core_news_trf corpus/perdt-ner/test.spacy --output metrics/trf-perdt-ner-test.json --gpu-id ${vars.gpu_trf}"
deps:
- "training/fa_core_news_trf"
- "corpus/merged/${vars.treebank}-ud-test.spacy"
- "corpus/perdt-ner/test.spacy"
outputs:
- "metrics/trf-core-ud-test.json"
- "metrics/trf-perdt-ner-test.json"
- name: "finalize-meta-trf"
help: "Fold the trf test scores into meta.json"
script:
- "python scripts/finalize_pipeline.py training/core-trf/model-best training/fa_core_news_trf --variant core --size trf --version ${vars.package_version} --ud-metrics metrics/trf-core-ud-test.json --ner-metrics metrics/trf-perdt-ner-test.json"
deps:
- "metrics/trf-core-ud-test.json"
- "metrics/trf-perdt-ner-test.json"
- "scripts/finalize_pipeline.py"
- name: "package-trf"
help: "Build the installable fa_core_news_trf wheel + sdist"
script:
- "python -m spacy package training/fa_core_news_trf packages --name ${vars.core_trf_package_name} --version ${vars.package_version} --build sdist,wheel --force"
deps:
- "training/fa_core_news_trf"
outputs:
- "packages/${vars.lang}_${vars.core_trf_package_name}-${vars.package_version}"
- name: "smoke-trf"
help: "Load fa_core_news_trf and run it over real Persian text"
script:
- "python scripts/smoke_test.py training/fa_core_news_trf"
deps:
- "training/fa_core_news_trf"
- name: "clean"
help: "Drop corpora, training runs and metrics (keeps downloaded assets)"
script:
- "rm -rf corpus training metrics packages"
Reference in New Issue View Git Blame Copy Permalink