title: "fa_core_news_sm / fa_dep_news_sm"
description: >
  CPU-sized Persian (fa) pipelines for spaCy 3.8, built entirely from UD_Persian-PerDT
  (PerUDT v1.0, CC BY-SA 4.0): tagger (XPOS), morphologizer (UPOS + FEATS), trainable
  lemmatizer, dependency parser and NER.

  Two shipping packages, same corpus, differing only in whether NER is included:
  `fa_dep_news_sm` (no NER) and `fa_core_news_sm` (with NER). The NER comes from the
  treebank's own `not-to-release/Dadegan with NER tag/` layer, so both packages share one
  corpus, one genre, one tokenization and one licence.

  PerDT's NER labels are silver, produced by Beheshti-NER with manual corrections, and are
  transferred onto this pipeline's tokenization by difflib at a 99.86% rate. Per-label
  scores are published in meta.json; MON, TIM and PCT are thin.

  See docs/MODELS.md for the source and licence analysis and docs/CONTRIBUTING-GUIDE.md for
  how this gets published.

  Build both with: `spacy project run all`
  Standalone NER-only package: `spacy project run ent`

vars:
  lang: "fa"
  dep_package_name: "dep_news_sm"
  core_package_name: "core_news_sm"
  ent_package_name: "ent_news_sm"
  package_version: "3.8.0"
  treebank: "fa_perdt"
  # -1 = CPU. A GTX 940MX (2 GB) is not worth the transfer overhead for an sm pipeline.
  gpu: -1
  n_sents: 10
  # md tier. Same architecture as sm plus the fa_floret static vector table.
  dep_md_package_name: "dep_news_md"
  core_md_package_name: "core_news_md"
  floret_wheel: "fa_floret-0.1.0-py3-none-any-400k-documents.whl"
  vectors_dir: "assets/vectors/fa_floret_400k"

directories:
  - "assets"
  - "corpus"
  - "configs"
  - "scripts"
  - "training"
  - "metrics"
  - "packages"

assets:
  - dest: "assets/ud/fa_perdt-ud-train.conllu"
    url: "https://raw.githubusercontent.com/UniversalDependencies/UD_Persian-PerDT/master/fa_perdt-ud-train.conllu"
    checksum: "f5a8ba901a776b4fd1941ecadcc6d506"
    description: "UD_Persian-PerDT train split (CC BY-SA 4.0)"
  - dest: "assets/ud/fa_perdt-ud-dev.conllu"
    url: "https://raw.githubusercontent.com/UniversalDependencies/UD_Persian-PerDT/master/fa_perdt-ud-dev.conllu"
    checksum: "f103020da7c1e917aafb8a8321f4cb84"
    description: "UD_Persian-PerDT dev split (CC BY-SA 4.0)"
  - dest: "assets/ud/fa_perdt-ud-test.conllu"
    url: "https://raw.githubusercontent.com/UniversalDependencies/UD_Persian-PerDT/master/fa_perdt-ud-test.conllu"
    checksum: "b62a66994cef2c50f7e524a1471102d8"
    description: "UD_Persian-PerDT test split (CC BY-SA 4.0)"
  - dest: "assets/ud-ner/train_with_NER_tag.txt"
    url: "https://raw.githubusercontent.com/UniversalDependencies/UD_Persian-PerDT/master/not-to-release/Dadegan%20with%20NER%20tag/train_with_NER_tag.txt"
    checksum: "ecb96cf99b38bc485cac21d22914e413"
    description: "PerDT NER layer, train split, IOB2 (CC BY-SA 4.0)"
  - dest: "assets/ud-ner/dev_with_NER_tag.txt"
    url: "https://raw.githubusercontent.com/UniversalDependencies/UD_Persian-PerDT/master/not-to-release/Dadegan%20with%20NER%20tag/dev_with_NER_tag.txt"
    checksum: "2a56ef7eb2e3732e221317af457d1c09"
    description: "PerDT NER layer, dev split, IOB2 (CC BY-SA 4.0)"
  - dest: "assets/ud-ner/test_with_NER_tag.txt"
    url: "https://raw.githubusercontent.com/UniversalDependencies/UD_Persian-PerDT/master/not-to-release/Dadegan%20with%20NER%20tag/test_with_NER_tag.txt"
    checksum: "6d80dd783527562c2ea5189f218a12b5"
    description: "PerDT NER layer, test split, IOB2 (CC BY-SA 4.0)"

workflows:
  # Both shipping artifacts: fa_dep_news_sm and fa_core_news_sm.
  all:
    - inspect
    - convert-ud
    - transfer-ner
    - convert-ner
    - debug-data
    - debug-data-ner
    - train-dep
    - train-ner
    - finalize-dep
    - evaluate-dep
    - assemble-core
    - evaluate-core
    - finalize-meta
    - package
    - smoke
  # Optional third artifact: the NER alone, for users who only want entities.
  ent:
    - finalize-ent
    - evaluate-ent
    - package-ent
  # The md tier: same corpus and architecture, plus the fa_floret static vectors.
  md:
    - vectors-md
    - train-dep-md
    - train-ner-md
    - finalize-dep-md
    - assemble-core-md
    - evaluate-md
    - finalize-meta-md
    - compare-md
    - package-md
    - smoke-md

commands:
  - name: "inspect"
    help: "Report annotation coverage of the downloaded treebank(s)"
    script:
      - "python scripts/inspect_treebanks.py assets/ud"
    deps:
      - "assets/ud/fa_perdt-ud-train.conllu"
      - "scripts/inspect_treebanks.py"

  - name: "convert-ud"
    help: >
      CoNLL-U -> DocBin. --merge-subtokens fuses multiword-token clitics into single
      tokens; docs/MODELS.md §5 has the measurement that justifies it.
    script:
      - "python -m spacy convert assets/ud/${vars.treebank}-ud-train.conllu corpus/merged --converter conllu --n-sents ${vars.n_sents} --merge-subtokens"
      - "python -m spacy convert assets/ud/${vars.treebank}-ud-dev.conllu corpus/merged --converter conllu --n-sents ${vars.n_sents} --merge-subtokens"
      - "python -m spacy convert assets/ud/${vars.treebank}-ud-test.conllu corpus/merged --converter conllu --n-sents ${vars.n_sents} --merge-subtokens"
      # Reproduce the measurement in docs/MODELS.md §5: convert the dev split WITHOUT
      # merging and compare both against the tokenizer we actually ship.
      - "python -m spacy convert assets/ud/${vars.treebank}-ud-dev.conllu corpus/split --converter conllu --n-sents ${vars.n_sents}"
      - "python scripts/tokenization_report.py corpus/merged/${vars.treebank}-ud-dev.spacy corpus/split/${vars.treebank}-ud-dev.spacy"
    deps:
      - "assets/ud/${vars.treebank}-ud-train.conllu"
      - "assets/ud/${vars.treebank}-ud-dev.conllu"
      - "assets/ud/${vars.treebank}-ud-test.conllu"
    outputs:
      - "corpus/merged/${vars.treebank}-ud-train.spacy"
      - "corpus/merged/${vars.treebank}-ud-dev.spacy"
      - "corpus/merged/${vars.treebank}-ud-test.spacy"

  - name: "transfer-ner"
    help: >
      Align PerDT's NER layer onto the --merge-subtokens tokenization. The NER files use the
      original Dadegan tokenization, which matches the released UD tokenization in only 57 to
      62% of sentences, so spans are transferred by difflib. Measured rate 99.86%; spans that
      cannot be aligned exactly are dropped rather than guessed.
    script:
      - "python scripts/transfer_perdt_ner.py --conllu-dir assets/ud --ner-dir assets/ud-ner --out corpus/perdt-ner-iob"
    deps:
      - "assets/ud/${vars.treebank}-ud-train.conllu"
      - "assets/ud-ner/train_with_NER_tag.txt"
      - "scripts/transfer_perdt_ner.py"
    outputs:
      - "corpus/perdt-ner-iob/train.txt"
      - "corpus/perdt-ner-iob/dev.txt"
      - "corpus/perdt-ner-iob/test.txt"

  - name: "convert-ner"
    help: "Transferred IOB2 -> DocBin"
    script:
      - "python -m spacy convert corpus/perdt-ner-iob/train.txt corpus/perdt-ner --converter ner --n-sents ${vars.n_sents} --lang ${vars.lang}"
      - "python -m spacy convert corpus/perdt-ner-iob/dev.txt corpus/perdt-ner --converter ner --n-sents ${vars.n_sents} --lang ${vars.lang}"
      - "python -m spacy convert corpus/perdt-ner-iob/test.txt corpus/perdt-ner --converter ner --n-sents ${vars.n_sents} --lang ${vars.lang}"
    deps:
      - "corpus/perdt-ner-iob/train.txt"
    outputs:
      - "corpus/perdt-ner/train.spacy"
      - "corpus/perdt-ner/dev.spacy"
      - "corpus/perdt-ner/test.spacy"

  - name: "debug-data"
    help: "Validate the treebank against the config before burning CPU on training"
    script:
      - "python -m spacy debug data configs/fa_dep_news_sm.cfg --paths.train corpus/merged/${vars.treebank}-ud-train.spacy --paths.dev corpus/merged/${vars.treebank}-ud-dev.spacy"
    deps:
      - "corpus/merged/${vars.treebank}-ud-train.spacy"
      - "configs/fa_dep_news_sm.cfg"

  - name: "debug-data-ner"
    help: "Validate the transferred PerDT NER corpus against the NER config"
    script:
      - "python -m spacy debug data configs/fa_ner_sm.cfg --paths.train corpus/perdt-ner/train.spacy --paths.dev corpus/perdt-ner/dev.spacy"
    deps:
      - "corpus/perdt-ner/train.spacy"
      - "configs/fa_ner_sm.cfg"

  - name: "train-dep"
    help: "Train tok2vec + tagger + morphologizer + trainable_lemmatizer + parser on PerDT"
    script:
      - "python -m spacy train configs/fa_dep_news_sm.cfg --output training/dep --paths.train corpus/merged/${vars.treebank}-ud-train.spacy --paths.dev corpus/merged/${vars.treebank}-ud-dev.spacy --gpu-id ${vars.gpu}"
    deps:
      - "corpus/merged/${vars.treebank}-ud-train.spacy"
      - "corpus/merged/${vars.treebank}-ud-dev.spacy"
      - "configs/fa_dep_news_sm.cfg"
    outputs:
      - "training/dep/model-best"

  - name: "train-ner"
    help: "Train the NER component (own embedded tok2vec) on the transferred PerDT layer"
    script:
      - "python -m spacy train configs/fa_ner_sm.cfg --output training/perdt-ner --paths.train corpus/perdt-ner/train.spacy --paths.dev corpus/perdt-ner/dev.spacy --gpu-id ${vars.gpu}"
    deps:
      - "corpus/perdt-ner/train.spacy"
      - "corpus/perdt-ner/dev.spacy"
      - "configs/fa_ner_sm.cfg"
    outputs:
      - "training/perdt-ner/model-best"

  - name: "finalize-dep"
    help: "Write fa_dep_news_sm metadata (sources, licence, notes) onto the trained model"
    script:
      - "python scripts/finalize_pipeline.py training/dep/model-best training/fa_dep_news_sm --variant dep --version ${vars.package_version}"
    deps:
      - "training/dep/model-best"
      - "scripts/finalize_pipeline.py"
    outputs:
      - "training/fa_dep_news_sm"

  - name: "evaluate-dep"
    help: "Score fa_dep_news_sm on the held-out UD test split"
    script:
      - "python -m spacy benchmark accuracy training/fa_dep_news_sm corpus/merged/${vars.treebank}-ud-test.spacy --output metrics/ud-test.json --gpu-id ${vars.gpu}"
    deps:
      - "training/fa_dep_news_sm"
      - "corpus/merged/${vars.treebank}-ud-test.spacy"
    outputs:
      - "metrics/ud-test.json"

  - name: "assemble-core"
    help: >
      Source the trained ner into the dep pipeline to produce fa_core_news_sm. Possible
      because configs/fa_ner_sm.cfg embeds its own tok2vec instead of a Tok2VecListener.
    script:
      - "python scripts/finalize_pipeline.py training/dep/model-best training/fa_core_news_sm --variant core --version ${vars.package_version} --add-ner training/perdt-ner/model-best"
    deps:
      - "training/dep/model-best"
      - "training/perdt-ner/model-best"
      - "scripts/finalize_pipeline.py"
    outputs:
      - "training/fa_core_news_sm"

  - name: "evaluate-core"
    help: "Score fa_core_news_sm on both held-out test splits"
    script:
      - "python -m spacy benchmark accuracy training/fa_core_news_sm corpus/merged/${vars.treebank}-ud-test.spacy --output metrics/core-ud-test.json --gpu-id ${vars.gpu}"
      - "python -m spacy benchmark accuracy training/fa_core_news_sm corpus/perdt-ner/test.spacy --output metrics/perdt-ner-test.json --gpu-id ${vars.gpu}"
    deps:
      - "training/fa_core_news_sm"
      - "corpus/merged/${vars.treebank}-ud-test.spacy"
      - "corpus/perdt-ner/test.spacy"
    outputs:
      - "metrics/core-ud-test.json"
      - "metrics/perdt-ner-test.json"

  - name: "finalize-meta"
    help: >
      Re-run finalize on both packages, folding test scores into meta.json["performance"].
      Separate because the scores only exist after evaluation, and evaluation needs a
      finalized pipeline to score. Cheap: it only copies models.
    script:
      - "python scripts/finalize_pipeline.py training/dep/model-best training/fa_dep_news_sm --variant dep --version ${vars.package_version} --ud-metrics metrics/ud-test.json"
      - "python scripts/finalize_pipeline.py training/dep/model-best training/fa_core_news_sm --variant core --version ${vars.package_version} --add-ner training/perdt-ner/model-best --ud-metrics metrics/core-ud-test.json --ner-metrics metrics/perdt-ner-test.json"
    deps:
      - "metrics/ud-test.json"
      - "metrics/perdt-ner-test.json"
      - "scripts/finalize_pipeline.py"

  - name: "package"
    help: "Build installable wheels + sdists for both shipping packages"
    script:
      - "python -m spacy package training/fa_dep_news_sm packages --name ${vars.dep_package_name} --version ${vars.package_version} --build sdist,wheel --force"
      - "python -m spacy package training/fa_core_news_sm packages --name ${vars.core_package_name} --version ${vars.package_version} --build sdist,wheel --force"
    deps:
      - "training/fa_dep_news_sm"
      - "training/fa_core_news_sm"
    outputs:
      - "packages/${vars.lang}_${vars.dep_package_name}-${vars.package_version}"
      - "packages/${vars.lang}_${vars.core_package_name}-${vars.package_version}"

  - name: "smoke"
    help: "Load both pipelines and run them over real Persian text"
    script:
      - "python scripts/smoke_test.py training/fa_dep_news_sm"
      - "python scripts/smoke_test.py training/fa_core_news_sm"
    deps:
      - "training/fa_dep_news_sm"
      - "training/fa_core_news_sm"

  - name: "finalize-ent"
    help: "Write fa_ent_news_sm metadata onto the trained NER model"
    script:
      - "python scripts/finalize_pipeline.py training/perdt-ner/model-best training/fa_ent_news_sm --variant ent --version ${vars.package_version}"
    deps:
      - "training/perdt-ner/model-best"
      - "scripts/finalize_pipeline.py"
    outputs:
      - "training/fa_ent_news_sm"

  - name: "evaluate-ent"
    help: "Score fa_ent_news_sm on the held-out PerDT NER test split"
    script:
      - "python -m spacy benchmark accuracy training/fa_ent_news_sm corpus/perdt-ner/test.spacy --output metrics/ent-test.json --gpu-id ${vars.gpu}"
      - "python scripts/finalize_pipeline.py training/perdt-ner/model-best training/fa_ent_news_sm --variant ent --version ${vars.package_version} --ner-metrics metrics/ent-test.json"
    deps:
      - "training/fa_ent_news_sm"
      - "corpus/perdt-ner/test.spacy"
    outputs:
      - "metrics/ent-test.json"

  - name: "package-ent"
    help: "Build the installable fa_ent_news_sm wheel + sdist"
    script:
      - "python -m spacy package training/fa_ent_news_sm packages --name ${vars.ent_package_name} --version ${vars.package_version} --build sdist,wheel --force"
    deps:
      - "training/fa_ent_news_sm"
    outputs:
      - "packages/${vars.lang}_${vars.ent_package_name}-${vars.package_version}"

  # ---------------------------------------------------------------- md tier

  - name: "vectors-md"
    help: >
      Unpack the fa_floret wheel into a plain spaCy model directory that
      `--paths.vectors` can point at. The wheel is a vectors-only pipeline
      (empty `pipeline: []`), 50k rows x 300d in floret mode, trained on 400k
      Persian documents, so no `spacy init vectors` step is needed.
    script:
      - "python scripts/unpack_vectors.py ${vars.floret_wheel} ${vars.vectors_dir}"
    deps:
      - "${vars.floret_wheel}"
      - "scripts/unpack_vectors.py"
    outputs:
      - "${vars.vectors_dir}"

  - name: "train-dep-md"
    help: "Train the dep pipeline with static floret vectors"
    script:
      - "python -m spacy train configs/fa_dep_news_md.cfg --output training/dep-md --paths.train corpus/merged/${vars.treebank}-ud-train.spacy --paths.dev corpus/merged/${vars.treebank}-ud-dev.spacy --paths.vectors ${vars.vectors_dir} --gpu-id ${vars.gpu}"
    deps:
      - "corpus/merged/${vars.treebank}-ud-train.spacy"
      - "corpus/merged/${vars.treebank}-ud-dev.spacy"
      - "configs/fa_dep_news_md.cfg"
      - "${vars.vectors_dir}"
    outputs:
      - "training/dep-md/model-best"

  - name: "train-ner-md"
    help: "Train the NER component with static floret vectors"
    script:
      - "python -m spacy train configs/fa_ner_md.cfg --output training/perdt-ner-md --paths.train corpus/perdt-ner/train.spacy --paths.dev corpus/perdt-ner/dev.spacy --paths.vectors ${vars.vectors_dir} --gpu-id ${vars.gpu}"
    deps:
      - "corpus/perdt-ner/train.spacy"
      - "corpus/perdt-ner/dev.spacy"
      - "configs/fa_ner_md.cfg"
      - "${vars.vectors_dir}"
    outputs:
      - "training/perdt-ner-md/model-best"

  - name: "finalize-dep-md"
    help: "Write fa_dep_news_md metadata onto the trained md model"
    script:
      - "python scripts/finalize_pipeline.py training/dep-md/model-best training/fa_dep_news_md --variant dep --size md --version ${vars.package_version}"
    deps:
      - "training/dep-md/model-best"
      - "scripts/finalize_pipeline.py"
    outputs:
      - "training/fa_dep_news_md"

  - name: "assemble-core-md"
    help: "Source the md ner into the md dep pipeline to produce fa_core_news_md"
    script:
      - "python scripts/finalize_pipeline.py training/dep-md/model-best training/fa_core_news_md --variant core --size md --version ${vars.package_version} --add-ner training/perdt-ner-md/model-best"
    deps:
      - "training/dep-md/model-best"
      - "training/perdt-ner-md/model-best"
      - "scripts/finalize_pipeline.py"
    outputs:
      - "training/fa_core_news_md"

  - name: "evaluate-md"
    help: "Score both md packages on the held-out test splits"
    script:
      - "python -m spacy benchmark accuracy training/fa_dep_news_md corpus/merged/${vars.treebank}-ud-test.spacy --output metrics/md-ud-test.json --gpu-id ${vars.gpu}"
      - "python -m spacy benchmark accuracy training/fa_core_news_md corpus/merged/${vars.treebank}-ud-test.spacy --output metrics/md-core-ud-test.json --gpu-id ${vars.gpu}"
      - "python -m spacy benchmark accuracy training/fa_core_news_md corpus/perdt-ner/test.spacy --output metrics/md-perdt-ner-test.json --gpu-id ${vars.gpu}"
    deps:
      - "training/fa_dep_news_md"
      - "training/fa_core_news_md"
    outputs:
      - "metrics/md-ud-test.json"
      - "metrics/md-core-ud-test.json"
      - "metrics/md-perdt-ner-test.json"

  - name: "finalize-meta-md"
    help: "Fold the md test scores into both md meta.json files"
    script:
      - "python scripts/finalize_pipeline.py training/dep-md/model-best training/fa_dep_news_md --variant dep --size md --version ${vars.package_version} --ud-metrics metrics/md-ud-test.json"
      - "python scripts/finalize_pipeline.py training/dep-md/model-best training/fa_core_news_md --variant core --size md --version ${vars.package_version} --add-ner training/perdt-ner-md/model-best --ud-metrics metrics/md-core-ud-test.json --ner-metrics metrics/md-perdt-ner-test.json"
    deps:
      - "metrics/md-ud-test.json"
      - "metrics/md-perdt-ner-test.json"
      - "scripts/finalize_pipeline.py"

  - name: "compare-md"
    help: "Table the sm vs md deltas from the metrics/ JSON reports"
    script:
      - "python scripts/compare_tiers.py"
    deps:
      - "metrics/md-ud-test.json"
      - "metrics/md-perdt-ner-test.json"
      - "scripts/compare_tiers.py"

  - name: "package-md"
    help: "Build installable wheels + sdists for both md packages"
    script:
      - "python -m spacy package training/fa_dep_news_md packages --name ${vars.dep_md_package_name} --version ${vars.package_version} --build sdist,wheel --force"
      - "python -m spacy package training/fa_core_news_md packages --name ${vars.core_md_package_name} --version ${vars.package_version} --build sdist,wheel --force"
    deps:
      - "training/fa_dep_news_md"
      - "training/fa_core_news_md"
    outputs:
      - "packages/${vars.lang}_${vars.dep_md_package_name}-${vars.package_version}"
      - "packages/${vars.lang}_${vars.core_md_package_name}-${vars.package_version}"

  - name: "smoke-md"
    help: "Load both md pipelines and run them over real Persian text"
    script:
      - "python scripts/smoke_test.py training/fa_dep_news_md"
      - "python scripts/smoke_test.py training/fa_core_news_md"
    deps:
      - "training/fa_dep_news_md"
      - "training/fa_core_news_md"


  - name: "clean"
    help: "Drop corpora, training runs and metrics (keeps downloaded assets)"
    script:
      - "rm -rf corpus training metrics packages"