title: "fa_dep_news_sm"
description: >
  A CPU-sized Persian (fa) dependency pipeline for spaCy 3.8: tagger (XPOS),
  morphologizer (UPOS + FEATS), trainable lemmatizer and dependency parser. Trained on
  UD_Persian-PerDT (PerUDT v1.0, CC BY-SA 4.0).

  This repo deliberately ships `dep`, not `core`. In spaCy's naming scheme `core` means
  "tagger + parser + lemmatizer + NER" in one package, and the only redistributably-licensed
  Persian NER corpus (ParsTwiNER, MIT) is a Twitter corpus — it scores 67.22 F against
  85-98 for the UD components, and folding it into a single `core` artifact would hide that
  behind one package name. So NER ships separately and optionally as `fa_ent_news_sm`
  (workflow: `ner`), and `fa_core_news_sm` is reserved for when ../ner_dataset delivers
  prose-genre NER data that beats ParsTwiNER on a human-annotated test set.

  See docs/MODELS.md for the source/licence analysis and docs/CONTRIBUTING-GUIDE.md for
  how this gets published.

  Run the shipping pipeline with: `spacy project run all`
  Optionally build the standalone NER with: `spacy project run ner`

vars:
  lang: "fa"
  package_name: "dep_news_sm"
  ner_package_name: "ent_news_sm"
  package_version: "3.8.0"
  treebank: "fa_perdt"
  # -1 = CPU. A GTX 940MX (2 GB) is not worth the transfer overhead for an sm pipeline.
  gpu: -1
  n_sents: 10

directories:
  - "assets"
  - "corpus"
  - "configs"
  - "scripts"
  - "training"
  - "metrics"
  - "packages"

assets:
  - dest: "assets/ud/fa_perdt-ud-train.conllu"
    url: "https://raw.githubusercontent.com/UniversalDependencies/UD_Persian-PerDT/master/fa_perdt-ud-train.conllu"
    checksum: "f5a8ba901a776b4fd1941ecadcc6d506"
    description: "UD_Persian-PerDT train split (CC BY-SA 4.0)"
  - dest: "assets/ud/fa_perdt-ud-dev.conllu"
    url: "https://raw.githubusercontent.com/UniversalDependencies/UD_Persian-PerDT/master/fa_perdt-ud-dev.conllu"
    checksum: "f103020da7c1e917aafb8a8321f4cb84"
    description: "UD_Persian-PerDT dev split (CC BY-SA 4.0)"
  - dest: "assets/ud/fa_perdt-ud-test.conllu"
    url: "https://raw.githubusercontent.com/UniversalDependencies/UD_Persian-PerDT/master/fa_perdt-ud-test.conllu"
    checksum: "b62a66994cef2c50f7e524a1471102d8"
    description: "UD_Persian-PerDT test split (CC BY-SA 4.0)"
  - dest: "assets/ner/ParsTwiNER_corpus_v1.0.0.zip"
    url: "https://github.com/overfit-ir/parstwiner/releases/download/v1.0.0/ParsTwiNER_corpus_v1.0.0.zip"
    checksum: "54615340d76c4d51b8f54751b07a278d"
    description: "ParsTwiNER Persian Twitter NER corpus, IOB2 (MIT)"

workflows:
  # The shipping artifact: fa_dep_news_sm.
  all:
    - inspect
    - convert-ud
    - debug-data
    - train-core
    - finalize
    - evaluate
    - finalize-meta
    - package
    - smoke
  # Optional, separate artifact: fa_ent_news_sm. Not part of `all` — see the note above.
  ner:
    - convert-ner
    - debug-data-ner
    - train-ner
    - finalize-ner
    - evaluate-ner
    - package-ner

commands:
  - name: "inspect"
    help: "Report annotation coverage of the downloaded treebank(s)"
    script:
      - "python scripts/inspect_treebanks.py assets/ud"
    deps:
      - "assets/ud/fa_perdt-ud-train.conllu"
      - "scripts/inspect_treebanks.py"

  - name: "convert-ud"
    help: >
      CoNLL-U -> DocBin. --merge-subtokens fuses multiword-token clitics into single
      tokens; docs/MODELS.md §5 has the measurement that justifies it.
    script:
      - "python -m spacy convert assets/ud/${vars.treebank}-ud-train.conllu corpus/merged --converter conllu --n-sents ${vars.n_sents} --merge-subtokens"
      - "python -m spacy convert assets/ud/${vars.treebank}-ud-dev.conllu corpus/merged --converter conllu --n-sents ${vars.n_sents} --merge-subtokens"
      - "python -m spacy convert assets/ud/${vars.treebank}-ud-test.conllu corpus/merged --converter conllu --n-sents ${vars.n_sents} --merge-subtokens"
      # Reproduce the measurement in docs/MODELS.md §5: convert the dev split WITHOUT
      # merging and compare both against the tokenizer we actually ship.
      - "python -m spacy convert assets/ud/${vars.treebank}-ud-dev.conllu corpus/split --converter conllu --n-sents ${vars.n_sents}"
      - "python scripts/tokenization_report.py corpus/merged/${vars.treebank}-ud-dev.spacy corpus/split/${vars.treebank}-ud-dev.spacy"
    deps:
      - "assets/ud/${vars.treebank}-ud-train.conllu"
      - "assets/ud/${vars.treebank}-ud-dev.conllu"
      - "assets/ud/${vars.treebank}-ud-test.conllu"
    outputs:
      - "corpus/merged/${vars.treebank}-ud-train.spacy"
      - "corpus/merged/${vars.treebank}-ud-dev.spacy"
      - "corpus/merged/${vars.treebank}-ud-test.spacy"

  - name: "convert-ner"
    help: "Unpack ParsTwiNER and convert its IOB2 files to DocBin"
    script:
      - "python scripts/extract_parstwiner.py assets/ner/ParsTwiNER_corpus_v1.0.0.zip assets/ner"
      - "python -m spacy convert assets/ner/train.txt corpus/ner --converter ner --n-sents ${vars.n_sents} --lang ${vars.lang}"
      - "python -m spacy convert assets/ner/dev.txt corpus/ner --converter ner --n-sents ${vars.n_sents} --lang ${vars.lang}"
      - "python -m spacy convert assets/ner/test.txt corpus/ner --converter ner --n-sents ${vars.n_sents} --lang ${vars.lang}"
    deps:
      - "assets/ner/ParsTwiNER_corpus_v1.0.0.zip"
      - "scripts/extract_parstwiner.py"
    outputs:
      - "corpus/ner/train.spacy"
      - "corpus/ner/dev.spacy"
      - "corpus/ner/test.spacy"

  - name: "debug-data"
    help: "Validate the treebank against the config before burning CPU on training"
    script:
      - "python -m spacy debug data configs/fa_dep_news_sm.cfg --paths.train corpus/merged/${vars.treebank}-ud-train.spacy --paths.dev corpus/merged/${vars.treebank}-ud-dev.spacy"
    deps:
      - "corpus/merged/${vars.treebank}-ud-train.spacy"
      - "configs/fa_dep_news_sm.cfg"

  - name: "debug-data-ner"
    help: "Validate the ParsTwiNER corpus against the NER config"
    script:
      - "python -m spacy debug data configs/fa_ner_sm.cfg --paths.train corpus/ner/train.spacy --paths.dev corpus/ner/dev.spacy"
    deps:
      - "corpus/ner/train.spacy"
      - "configs/fa_ner_sm.cfg"

  - name: "train-core"
    help: "Train tok2vec + tagger + morphologizer + trainable_lemmatizer + parser on PerDT"
    script:
      - "python -m spacy train configs/fa_dep_news_sm.cfg --output training/core --paths.train corpus/merged/${vars.treebank}-ud-train.spacy --paths.dev corpus/merged/${vars.treebank}-ud-dev.spacy --gpu-id ${vars.gpu}"
    deps:
      - "corpus/merged/${vars.treebank}-ud-train.spacy"
      - "corpus/merged/${vars.treebank}-ud-dev.spacy"
      - "configs/fa_dep_news_sm.cfg"
    outputs:
      - "training/core/model-best"

  - name: "train-ner"
    help: "Train the standalone NER component (own internal tok2vec) on ParsTwiNER"
    script:
      - "python -m spacy train configs/fa_ner_sm.cfg --output training/ner --paths.train corpus/ner/train.spacy --paths.dev corpus/ner/dev.spacy --gpu-id ${vars.gpu}"
    deps:
      - "corpus/ner/train.spacy"
      - "corpus/ner/dev.spacy"
      - "configs/fa_ner_sm.cfg"
    outputs:
      - "training/ner/model-best"

  - name: "finalize"
    help: "Write fa_dep_news_sm metadata (sources, licence, notes) onto the trained model"
    script:
      - "python scripts/finalize_pipeline.py training/core/model-best training/fa_dep_news_sm --variant dep --version ${vars.package_version}"
    deps:
      - "training/core/model-best"
      - "scripts/finalize_pipeline.py"
    outputs:
      - "training/fa_dep_news_sm"

  - name: "evaluate"
    help: "Score fa_dep_news_sm on the held-out UD test split"
    script:
      - "python -m spacy benchmark accuracy training/fa_dep_news_sm corpus/merged/${vars.treebank}-ud-test.spacy --output metrics/ud-test.json --gpu-id ${vars.gpu}"
    deps:
      - "training/fa_dep_news_sm"
      - "corpus/merged/${vars.treebank}-ud-test.spacy"
    outputs:
      - "metrics/ud-test.json"

  - name: "finalize-meta"
    help: >
      Re-run finalize, this time folding the test scores into meta.json["performance"].
      Separate from `finalize` because the scores can only exist after `evaluate`, and
      `evaluate` needs a finalized pipeline to score. Cheap: it only copies models.
    script:
      - "python scripts/finalize_pipeline.py training/core/model-best training/fa_dep_news_sm --variant dep --version ${vars.package_version} --metrics metrics/ud-test.json"
    deps:
      - "metrics/ud-test.json"
      - "scripts/finalize_pipeline.py"

  - name: "package"
    help: "Build the installable fa_dep_news_sm wheel + sdist"
    script:
      - "python -m spacy package training/fa_dep_news_sm packages --name ${vars.package_name} --version ${vars.package_version} --build sdist,wheel --force"
    deps:
      - "training/fa_dep_news_sm"
    outputs:
      - "packages/${vars.lang}_${vars.package_name}-${vars.package_version}"

  - name: "smoke"
    help: "Load the pipeline and run it over real Persian text"
    script:
      - "python scripts/smoke_test.py training/fa_dep_news_sm"
    deps:
      - "training/fa_dep_news_sm"

  - name: "finalize-ner"
    help: "Write fa_ent_news_sm metadata onto the trained NER model"
    script:
      - "python scripts/finalize_pipeline.py training/ner/model-best training/fa_ent_news_sm --variant ent --version ${vars.package_version} --metrics metrics/ner-test.json"
    deps:
      - "training/ner/model-best"
      - "scripts/finalize_pipeline.py"
    outputs:
      - "training/fa_ent_news_sm"

  - name: "evaluate-ner"
    help: "Score fa_ent_news_sm on the held-out ParsTwiNER test split"
    script:
      - "python -m spacy benchmark accuracy training/fa_ent_news_sm corpus/ner/test.spacy --output metrics/ner-test.json --gpu-id ${vars.gpu}"
      - "python scripts/finalize_pipeline.py training/ner/model-best training/fa_ent_news_sm --variant ent --version ${vars.package_version} --metrics metrics/ner-test.json"
    deps:
      - "training/fa_ent_news_sm"
      - "corpus/ner/test.spacy"
    outputs:
      - "metrics/ner-test.json"

  - name: "package-ner"
    help: "Build the installable fa_ent_news_sm wheel + sdist"
    script:
      - "python -m spacy package training/fa_ent_news_sm packages --name ${vars.ner_package_name} --version ${vars.package_version} --build sdist,wheel --force"
    deps:
      - "training/fa_ent_news_sm"
    outputs:
      - "packages/${vars.lang}_${vars.ner_package_name}-${vars.package_version}"

  - name: "clean"
    help: "Drop corpora, training runs and metrics (keeps downloaded assets)"
    script:
      - "rm -rf corpus/merged corpus/split corpus/ner training metrics packages"