From 518340a840e8d8dc4f6a1ac4b6f8590e2ece7ffd Mon Sep 17 00:00:00 2001 From: Mohamad Fazeli Date: Sat, 1 Aug 2026 00:59:16 +0330 Subject: [PATCH] Ship fa_core_news_sm: PerDT carries its own NER layer UD_Persian-PerDT ships entity annotations in not-to-release/Dadegan with NER tag/ that nothing in this project had looked at: 29,107 sentences, 484,312 tokens, 15,833 entities, under the treebank's own CC BY-SA 4.0. That removes the reason core was withheld. The previous commit split dep from ent because the only redistributable Persian NER corpus known then was ParsTwiNER, a Twitter corpus scoring 67.22 F against 85-98 for the UD components, and hiding that genre and quality gap behind one package name was not acceptable. A NER layer from the same corpus has none of those problems: one genre, one tokenization, one licence, one provenance chain. Two shipping packages, both from PerDT alone: fa_dep_news_sm 7.5 MB TAG 95.96 LEMMA 97.91 LAS 85.15 fa_core_news_sm 13 MB the above plus ENTS_P/R/F 77.67 / 66.87 / 71.87 Per label: LOC 80.24, DAT 74.45, MON 73.68, ORG 68.77, TIM 66.67, PER 65.29, PCT 57.14. PER scoring below LOC on comparable data is the silver labels showing: PerDT starts 6.24% of PER spans with a title against ParsTwiNER's 1.41%, so the boundaries are less regular than the count suggests. MON, TIM and PCT rest on 4 to 11 test entities and are indicative only. The labels are silver, from Beheshti-NER (Taher et al. 2020) with manual corrections per the treebank README. meta.json notes say so. A human-annotated test set is the outstanding work, tracked in ../ner_dataset. Alignment: the NER files use the original Dadegan tokenization, matching the released UD tokenization in only 57 to 62% of sentences (dropped copulas and auxiliaries, one honorific corrupted to a comma). scripts/transfer_perdt_ner.py realigns with difflib at 99.86% train / 99.74% dev / 99.51% test; spans whose tokens do not all map contiguously are dropped rather than guessed. Also fixed a silent metadata corruption. Folding both benchmark reports over one key set gave core tag_acc 0.00, because the NER corpus has no gold tags so its report carries tag_acc: 0.0, which overwrote the real 95.96; sents_f was wrong the same way. finalize_pipeline.py now takes --ud-metrics and --ner-metrics separately, each restricted to the keys its corpus can evidence. Its variant guards work both directions now: a dep pipeline may not contain ner, a core one must. ParsTwiNER moves out of the shipping repo to a future mixed-genre package, with the ablation that justifies it: prose-trained NER scores 45.72 F on tweets, tweet-trained scores 55.59 on prose, and mixing gives 72.11 / 66.49, so roughly 20 F of cross-genre robustness for under 1 F on prose. Adds LICENSE recording the split: MIT for the code, CC BY-SA 4.0 for the trained pipelines as Adapted Material, with the attribution, modification notice and warranty disclaimer that CC BY-SA 4.0 section 3 requires. Verified the treebank's LICENSE.txt is unmodified CC BY-SA 4.0 with no carve-out for not-to-release/. Author metadata filled in; both packages rebuilt and installed from their wheels. --- .gitignore | 7 + LICENSE | 53 ++++++ README.md | 301 +++++++++++++++-------------- docs/CONTRIBUTING-GUIDE.md | 202 ++++++++++---------- docs/MODELS.md | 328 ++++++++++++++++++-------------- docs/upstream/fa-noun-chunks.md | 78 ++++---- project.yml | 219 +++++++++++++-------- scripts/extract_parstwiner.py | 34 ---- scripts/finalize_pipeline.py | 195 ++++++++++++------- scripts/transfer_perdt_ner.py | 169 ++++++++++++++++ 10 files changed, 975 insertions(+), 611 deletions(-) create mode 100644 LICENSE delete mode 100644 scripts/extract_parstwiner.py create mode 100644 scripts/transfer_perdt_ner.py diff --git a/.gitignore b/.gitignore index 3bcf9af..8b30561 100644 --- a/.gitignore +++ b/.gitignore @@ -6,5 +6,12 @@ training/ metrics/ packages/ .venv/ +# Separate env for `spacy huggingface-hub push`: it caps typer<0.8, which breaks the +# spaCy CLI in the training venv. See .omp/AGENTS.md. +.venv-publish/ __pycache__/ *.pyc +# Personal scratch list, not part of the project +TODO.md +# Agent/session notes, local only +.omp/ diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..06256ce --- /dev/null +++ b/LICENSE @@ -0,0 +1,53 @@ +This repository contains two kinds of material under two different licences. + +1. SOURCE CODE (scripts/, configs/, project.yml, docs/) — MIT, below. + +2. TRAINED PIPELINES (fa_dep_news_sm, fa_core_news_sm, fa_ent_news_sm, and any + artifact under training/ or packages/) — Creative Commons Attribution-ShareAlike + 4.0 International (CC BY-SA 4.0). + + These are Adapted Material derived from UD_Persian-PerDT (PerUDT v1.0), which is + licensed CC BY-SA 4.0, so the ShareAlike condition requires the same licence. + + Source: https://github.com/UniversalDependencies/UD_Persian-PerDT + Licence: https://creativecommons.org/licenses/by-sa/4.0/ + Authors: Mohammad Sadegh Rasooli, Pegah Safari, Amirsaeid Moloodi, + Alireza Nourian + + The entity annotations additionally derive from that treebank's + not-to-release/Dadegan with NER tag/ layer, which its README states was produced + with the Beheshti-NER tagger (Taher, Hoseini, Shamsfard 2020) plus manual + corrections. + + Modifications made to the licensed material: CoNLL-U converted to spaCy DocBin + with multiword tokens merged (`spacy convert --merge-subtokens`); entity spans + realigned onto that tokenization by difflib, with spans that could not be aligned + exactly dropped rather than guessed. See docs/MODELS.md. + + The licensed material is provided as-is, without warranties of any kind, and the + licensor disclaims liability for damages arising from its use, to the extent + permitted by CC BY-SA 4.0 sections 5(a) and 5(b). + +----------------------------------------------------------------------------------- + +MIT License + +Copyright (c) 2026 Kiyarash Fazeli + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md index 0f4332b..faa840a 100644 --- a/README.md +++ b/README.md @@ -1,138 +1,143 @@ -# fa_dep_news_sm — a Persian pipeline for spaCy +# fa_core_news_sm and fa_dep_news_sm, Persian pipelines for spaCy -There is no trained Persian pipeline for spaCy. `spacy.load("fa_core_news_sm")` has never -worked; `spacy.blank("fa")` gives you a tokenizer and stop words and nothing else. This -project builds the missing pipeline from openly-licensed data, using spaCy's own tooling, so -the result can actually be redistributed. +spaCy has no trained Persian pipeline. `spacy.load("fa_core_news_sm")` has never worked, and +`spacy.blank("fa")` gives you a tokenizer and stop words. This project trains one from +openly-licensed data so the result can be redistributed. -- **What the four English pipelines are, and what the four Persian equivalents should be:** - [`docs/MODELS.md`](docs/MODELS.md) -- **How models get contributed/published in the spaCy ecosystem, and what upstream `fa` - already has:** [`docs/CONTRIBUTING-GUIDE.md`](docs/CONTRIBUTING-GUIDE.md) -- **The build itself:** [`project.yml`](project.yml) +- Pipeline inventory and source analysis: [`docs/MODELS.md`](docs/MODELS.md) +- How spaCy models get published, and what upstream `fa` already has: + [`docs/CONTRIBUTING-GUIDE.md`](docs/CONTRIBUTING-GUIDE.md) +- The build: [`project.yml`](project.yml) -## Two packages, not one +## Two packages, one corpus -spaCy's naming scheme encodes what a pipeline contains: `dep` = tagger + parser + lemmatizer, -`ent` = NER only, `core` = both. This project ships the first two separately and deliberately -does **not** ship a `core`: +In spaCy's naming scheme `dep` = tagger + parser + lemmatizer, `core` = the same plus NER. +Both packages here are built entirely from UD_Persian-PerDT and differ only in whether NER is +included. -| Package | Components | Trained on | Licence | Headline | +| Package | Components | Licence | Score | Wheel | | --- | --- | --- | --- | --- | -| **`fa_dep_news_sm`** | tok2vec, tagger, morphologizer, trainable_lemmatizer, parser | [UD_Persian-PerDT](https://github.com/UniversalDependencies/UD_Persian-PerDT), 29,107 sentences of edited prose | CC BY-SA 4.0 | **LAS 85.15**, LEMMA 97.91 | -| `fa_ent_news_sm` (optional) | ner | [ParsTwiNER](https://github.com/overfit-ir/parstwiner), 7,667 **tweets**, 16,250 entities | MIT | **ENTS_F 67.22** | +| `fa_dep_news_sm` | tok2vec, tagger, morphologizer, trainable_lemmatizer, parser | CC BY-SA 4.0 | LAS 85.15, LEMMA 97.91 | 7.5 MB | +| `fa_core_news_sm` | the above plus ner | CC BY-SA 4.0 | LAS 85.15, ENTS_F 71.87 | 13 MB | -Those two headline numbers are the whole argument. The UD components score 85–98 on edited -prose; the NER manages 67 F on a different genre entirely, because the good Persian NER -corpora (ARMAN, PEYMA, NSURL) are research-use-only and cannot be redistributed. Folding both -into one `fa_core_news_sm` would hide that gap behind a single package name and a single -version number — users would reasonably assume the NER is held to the same standard as the -parser. It is not. +The NER is possible because the treebank ships its own entity layer in +`not-to-release/Dadegan with NER tag/`: 15,833 entities over the same 29,107 sentences, under +the same CC BY-SA 4.0. That is what makes `core` honest here, since one corpus means one genre, +one tokenization, one licence and one provenance chain. The alternative NER corpora are all +worse on at least one of those axes: ARMAN, PEYMA and NSURL are research-use-only, and +ParsTwiNER (MIT) is a Twitter corpus that costs about 23 F on prose. -So NER ships as its own opt-in package, and **`fa_core_news_sm` is reserved** for when -[`../ner_dataset`](../ner_dataset/PLAN.md) delivers prose-genre NER data that beats ParsTwiNER -on a human-annotated test set. The `ner` component already embeds its own tok2vec rather than -a `Tok2VecListener` precisely so that merge is a one-liner when the data arrives: +Two caveats to know before relying on the entities: -```python -dep = spacy.load("fa_dep_news_sm") -dep.add_pipe("ner", source=spacy.load("fa_ent_news_sm")) # verified working -``` +- **The labels are silver.** The treebank README states they came from the BERT-based + Beheshti-NER tagger with manual corrections for recall, so `ENTS_F 71.87` is measured against + a silver test split and partly reflects agreement with that tagger. +- **Three labels are thin.** `MON` (205 training examples), `TIM` (135) and `PCT` (121) score + 73.7, 66.7 and 57.1. `PER`, `LOC`, `ORG` and `DAT` have 1,300 or more each. -Language data comes from `spacy/lang/fa` upstream (its stop word list originally from hazm). +Entity spans were transferred onto this pipeline's tokenization by difflib alignment at a 99.86% +rate; spans that could not be aligned exactly were dropped rather than guessed +(`scripts/transfer_perdt_ner.py`). + +Language data comes from `spacy/lang/fa` upstream, whose stop word list came from hazm. Everything trains on 4 CPU cores with no GPU. -### Results +## Results -Trained and evaluated on this laptop (4-core i5-7200U, CPU only, 1h27m for the UD -components, ~25 min for NER). Scores are on the **held-out test splits**, produced by -`spacy benchmark accuracy` and stored in `metrics/`. +Held-out test splits, from `spacy benchmark accuracy`, stored in `metrics/`. Trained on a +4-core i5-7200U: 1h27m for the UD components, 17 min for NER. -**`fa_dep_news_sm`** (UD_Persian-PerDT test split): +Syntax and morphology, identical in both packages since they share the same trained components: -| Metric | Score | reference | +| Metric | Score | Reference | | --- | --- | --- | | `TOKEN_ACC` / `TOKEN_F` | 99.96 / 99.11 | | -| `TAG_ACC` (XPOS) | **95.96** | | -| `POS_ACC` (UPOS) | **96.24** | | +| `TAG_ACC` (XPOS) | 95.96 | | +| `POS_ACC` (UPOS) | 96.24 | | | `MORPH_ACC` | 96.29 | | -| `LEMMA_ACC` | **97.91** | | +| `LEMMA_ACC` | 97.91 | | | `SENTS_F` | 99.25 | | -| `DEP_UAS` | **89.69** | hazm+ParsBERT: 92.46 | -| `DEP_LAS` | **85.15** | hazm+ParsBERT: 89.34 | -| Speed | ~9,250 words/s (CPU) | | -| Wheel | 7.5 MB | `en_core_web_sm`: 12 MB | +| `DEP_UAS` | 89.69 | hazm+ParsBERT: 92.46 | +| `DEP_LAS` | 85.15 | hazm+ParsBERT: 89.34 | +| Speed | ~9,250 words/s | | -**`fa_ent_news_sm`** (ParsTwiNER test split): `ENTS_P` 74.77 / `ENTS_R` 61.06 / -**`ENTS_F` 67.22**, 5.6 MB wheel. Per label: -`LOC` 73.9, `PER` 69.1, `NAT` 63.2, `ORG` 59.3, `POG` 41.2, `EVE` 30.0. +Entities, `fa_core_news_sm` only, on the PerDT NER test split: `ENTS_P` 77.67, `ENTS_R` 66.87, +`ENTS_F` 71.87. Per label: -Read these honestly: +| Label | F | Train examples | +| --- | --- | --- | +| `LOC` | 80.24 | 4,954 | +| `DAT` | 74.45 | 1,323 | +| `MON` | 73.68 | 205 | +| `ORG` | 68.77 | 2,643 | +| `TIM` | 66.67 | 135 | +| `PER` | 65.29 | 4,847 | +| `PCT` | 57.14 | 121 | -- **Parsing is 4.2 LAS behind hazm's parser**, which is the expected gap between a 7.5 MB - CPU model with hash embeddings and a fine-tuned ParsBERT. Same corpus, same spaCy parser - architecture, so the comparison is fair — and it sets the target for a future `trf` tier. -- **NER is the weak artifact, and the per-label numbers say why.** ParsTwiNER is not small - (232,917 tokens, 16,250 entities, 7.0% density — the same order as the restricted ARMAN and - PEYMA). But its label distribution is brutally skewed: `PER` 6258, `LOC` 5478, `ORG` 2694, - `NAT` 939, **`EVE` 482, `POG` 399**. The two starved labels are exactly the two that score - 30.0 and 41.2. So the head labels suffer from genre mismatch and the tail labels from raw - data starvation — two different problems needing two different fixes. -- **Everything else is competitive with the English `sm` pipeline** (`en_core_web_sm`: - TAG 97, LAS 90, ENTS_F 84 — on a much larger and cleaner corpus). +Parsing is 4.2 LAS behind hazm's parser, which uses the same corpus and the same spaCy parser +architecture with a fine-tuned ParsBERT instead of hash embeddings. That gap is the target for +a future `trf` tier. -Reproduce: `.venv/bin/python -m spacy project run all` (add `run ner` for the NER package). +`PER` scoring below `LOC` and `ORG` despite having 4,847 examples is the silver labels showing +through: PerDT includes titles and honorifics inside `PER` spans inconsistently (6.24% of spans +start with one, against 1.41% in the human-annotated ParsTwiNER), so the boundaries the model +has to learn are less regular than the label count suggests. -### Install +For comparison, `en_core_web_sm` scores TAG 97, LAS 90, ENTS_F 84 on a larger, cleaner corpus. + +Reproduce with `.venv/bin/python -m spacy project run all`, plus `run ent` for an NER-only +package. + +## Install ```bash +.venv/bin/python -m pip install packages/fa_core_news_sm-3.8.0/dist/fa_core_news_sm-3.8.0-py3-none-any.whl +# or, without NER: .venv/bin/python -m pip install packages/fa_dep_news_sm-3.8.0/dist/fa_dep_news_sm-3.8.0-py3-none-any.whl -# optional, separate package: -.venv/bin/python -m pip install packages/fa_ent_news_sm-3.8.0/dist/fa_ent_news_sm-3.8.0-py3-none-any.whl ``` ```python import spacy -nlp = spacy.load("fa_dep_news_sm") -doc = nlp("دانشگاه تهران در سال ۱۳۱۳ تأسیس شد.") -print([(t.text, t.pos_, t.lemma_, t.dep_) for t in doc]) -# ('دانشگاه', 'PROPN', 'دانشگاه', 'nsubj') ('تهران', 'PROPN', 'تهران', 'flat:name') ... +nlp = spacy.load("fa_core_news_sm") +doc = nlp("محمدرضا شجریان در مشهد به دنیا آمد.") +print([(t.text, t.pos_, t.lemma_, t.dep_) for t in doc][:3]) +# [('محمدرضا', 'PROPN', 'محمدرضا', 'nsubj'), ('شجریان', 'PROPN', 'شجریان', 'flat:name'), ...] +print(doc.ents) # (محمدرضا شجریان, مشهد) -> PER, LOC -# Want entities too? Attach the NER package yourself, eyes open about its 67 F: -nlp.add_pipe("ner", source=spacy.load("fa_ent_news_sm")) -print(nlp(doc.text).ents) # (دانشگاه تهران,) -> ORG +doc = nlp("شرکت ایران خودرو تولید را ۲۰ درصد افزایش می‌دهد.") +print([(e.text, e.label_) for e in doc.ents]) # ۲۰ درصد -> PCT ``` -### Why not hazm's own models +## Why not hazm's own models -hazm is the reference Persian NLP toolkit and it *does* publish spaCy-format pipelines on the -HF Hub, so it was the obvious starting point. It does not survive contact: +hazm is the reference Persian NLP toolkit and publishes spaCy-format pipelines on the HF Hub, +so it was the obvious starting point. Four problems: -- Its trainable models are pycrfsuite CRFs (`hazm/sequence_tagger.py`). There is no - `config.cfg` and no `spacy train` anywhere in the repo — the `Spacy*` classes only download - pretrained pipelines. -- Those pretrained pipelines are three *single-task* models (`transformer + tagger`, - `transformer + parser`, `transformer + chunker`), each `version: 0.0.0` with an empty - `license` field, pinned to spaCy 3.6. Using all three means three ParsBERT forward passes - over the same text and no shared `Doc`. -- Its tokenizer is deliberately incompatible with UD tokenization: the normaliser fuses ZWNJ - affixes and `join_verb_parts()` glues multi-word verb chains into single tokens. -- Most corpora it reads (Bijankhan, Peykare, Hamshahri, raw PerDT) are gated behind - `peykaregan.ir` / `dadegan.ir` under research-only terms. +- Its trainable models are pycrfsuite CRFs (`hazm/sequence_tagger.py`). The repo contains no + `config.cfg` and no `spacy train`; the `Spacy*` classes only download pretrained pipelines. +- Those pipelines are three single-task models (`transformer + tagger`, `transformer + parser`, + `transformer + chunker`), each `version: 0.0.0` with an empty `license` field, pinned to + spaCy 3.6. Using all three costs three ParsBERT forward passes and gives no shared `Doc`. +- Its tokenizer is incompatible with UD tokenization: the normaliser fuses ZWNJ affixes and + `join_verb_parts()` glues multi-word verb chains into single tokens. +- Most corpora it reads (Bijankhan, Peykare, Hamshahri, raw PerDT) sit behind `peykaregan.ir` + or `dadegan.ir` under research-only terms. -What hazm *does* give us: confirmation of the corpus choice — hazm's own spaCy parser was -trained on `modified_fa_perdt-ud-train.spacy`, i.e. the same treebank we use — plus the stop -word list already vendored into `spacy/lang/fa`. Full analysis in +It did confirm the corpus choice. hazm's own spaCy parser was trained on +`modified_fa_perdt-ud-train.spacy`, the same treebank used here. Full analysis in [`docs/MODELS.md`](docs/MODELS.md) §4. -### Why licensing is the load-bearing constraint +## Licensing drove most decisions here -spaCy's maintainers state that Persian models trained back in 2018 were never published -*because of corpus licensing* (spaCy discussion #8233, after PR #2797 added `fa` tokenizer -support). The standard Persian NER corpora — ARMAN, PEYMA, NSURL — are all "research use -only", and wrapping them in an Apache-2.0 toolkit does not launder that. ParsTwiNER (MIT) is -the only redistributable Persian NER corpus we could verify, which is why the NER component is -trained on tweets. That trade-off is recorded in the model's `meta.json["notes"]`. +spaCy's maintainers say the Persian models trained in 2018 were never published because of +corpus licensing (spaCy discussion #8233, after PR #2797 added `fa` tokenizer support). ARMAN, +PEYMA and NSURL are all research-use-only, and wrapping them in an Apache-2.0 toolkit does not +change that. + +The way out was finding that PerDT ships its own NER layer under the treebank's CC BY-SA 4.0, +so the entire pipeline now derives from one corpus with one licence. The 2018 attempt also +failed for a second reason worth knowing if you plan to publish: honnibal asked for scripts +that could regenerate the model and got a notebook instead. `project.yml` is that script. ## Setup @@ -145,65 +150,71 @@ python -m venv .venv ## Build -Everything is driven by [`project.yml`](project.yml), which has **two independent workflows**: +[`project.yml`](project.yml) has two workflows: ```bash .venv/bin/python -m spacy project assets # download + checksum the corpora -.venv/bin/python -m spacy project run all # -> fa_dep_news_sm (the shipping artifact) -.venv/bin/python -m spacy project run ner # -> fa_ent_news_sm (optional) +.venv/bin/python -m spacy project run all # -> fa_dep_news_sm + fa_core_news_sm +.venv/bin/python -m spacy project run ent # -> fa_ent_news_sm, NER alone ``` -| Workflow | Command | What it does | -| --- | --- | --- | -| `all` | `inspect` | annotation coverage of the treebanks (`scripts/inspect_treebanks.py`) | -| | `convert-ud` | CoNLL-U → `DocBin` with `--merge-subtokens`, plus the tokenizer-agreement report | -| | `debug-data` | `spacy debug data` before spending CPU | -| | `train-core` | tagger + morphologizer + trainable_lemmatizer + parser on PerDT | -| | `finalize` | write `fa_dep_news_sm` metadata: sources, licence, notes (`scripts/finalize_pipeline.py`) | -| | `evaluate` | `spacy benchmark accuracy` on the held-out UD test split | -| | `finalize-meta` | re-run `finalize`, folding test scores into `meta.json["performance"]` | -| | `package` | build the wheel + sdist | -| | `smoke` | run the pipeline over real Persian text and print every annotation layer | -| `ner` | `convert-ner` | unpack ParsTwiNER, IOB2 → `DocBin` | -| | `debug-data-ner`, `train-ner`, `finalize-ner`, `evaluate-ner`, `package-ner` | the same sequence for `fa_ent_news_sm` | +| Command | What it does | +| --- | --- | +| `inspect` | annotation coverage of the treebanks (`scripts/inspect_treebanks.py`) | +| `convert-ud` | CoNLL-U to `DocBin` with `--merge-subtokens`, plus the tokenizer-agreement report | +| `transfer-ner` | align PerDT's NER layer onto that tokenization by difflib (`scripts/transfer_perdt_ner.py`) | +| `convert-ner` | transferred IOB2 to `DocBin` | +| `debug-data`, `debug-data-ner` | `spacy debug data` on both corpora before spending CPU | +| `train-dep` | tagger + morphologizer + trainable_lemmatizer + parser | +| `train-ner` | the `ner` component, with its own embedded tok2vec | +| `finalize-dep` | write `fa_dep_news_sm` metadata: sources, licence, notes (`scripts/finalize_pipeline.py`) | +| `evaluate-dep` | `spacy benchmark accuracy` on the held-out UD test split | +| `assemble-core` | source `ner` into the dep pipeline to produce `fa_core_news_sm` | +| `evaluate-core` | score the assembled pipeline on both test splits | +| `finalize-meta` | re-run finalize on both, folding test scores into `meta.json["performance"]` | +| `package` | build wheels + sdists for both | +| `smoke` | run both pipelines over Persian text and print every annotation layer | -The two training runs are independent and can run concurrently — each is single-threaded. +The two training runs are single-threaded and independent, so they can run concurrently. -`finalize` and `finalize-meta` are separate steps for an unavoidable ordering reason: test -scores can only exist after `evaluate`, and `evaluate` needs a finalized pipeline to score. -Re-running `finalize` afterwards is cheap (it only copies models). -`scripts/finalize_pipeline.py` **refuses** to publish a `dep` pipeline containing an `ner` -component, so the split cannot silently regress. +`finalize` runs twice because of an ordering constraint: test scores only exist after +evaluation, and evaluation needs a finalized pipeline to score. The second pass only copies +models. `scripts/finalize_pipeline.py` enforces the shape of each variant, refusing to publish +a `dep` pipeline that contains `ner` or a `core` one that does not, so the split cannot regress +unnoticed. -## Design decisions worth knowing before you touch anything +`--ud-metrics` and `--ner-metrics` are separate flags on purpose. Folding both reports over one +key set silently corrupted `core`'s metadata during development: the NER corpus has no gold +tags, so its report carries `tag_acc: 0.0`, which overwrote the real 95.96. -1. **`--merge-subtokens`.** spaCy has no multiword-token layer, and PerDT splits pronominal - clitics (`پدرم` → `پدر` + `م`). Measured on dev: merging gives token F 0.9887 vs 0.9823 for - the split version, at the cost of 34 composite XPOS tags on 1.5% of tokens. Merging wins - because otherwise 1.5% of gold tokens are boundaries the shipped tokenizer can never - produce. Numbers: `scripts/tokenization_report.py`. -2. **`ner` carries its own tok2vec.** A `Tok2VecListener` only resolves inside the pipeline it - was trained in, so a listener-based component cannot be sourced into another pipeline. - `configs/fa_ner_sm.cfg` embeds the tok2vec instead — the same design as `en_core_web_sm`. -3. **`morphologizer` + `trainable_lemmatizer` instead of `attribute_ruler` + rule lemmatizer.** - The English pipelines derive UPOS from PTB tags by rule because OntoNotes has no UPOS. UD - gives us gold UPOS, FEATS and lemmas, so we train on them and get real `pos_acc`, - `morph_acc` and `lemma_acc` numbers instead of unmeasurable rule coverage. -4. **PerDT, not Seraji.** 3.7× more tokens, and Seraji has no `PROPN` tag at all. +## Design decisions + +1. `--merge-subtokens`. spaCy has no multiword-token layer, and PerDT splits pronominal clitics + (`پدرم` into `پدر` + `م`). Measured on dev, merging gives token F 0.9887 against 0.9823 for + the split version, costing 34 composite XPOS tags on 1.5% of tokens. Without it, 1.5% of gold + tokens are boundaries the shipped tokenizer can never produce. See + `scripts/tokenization_report.py`. +2. `ner` carries its own tok2vec. A `Tok2VecListener` only resolves inside the pipeline it was + trained in, so a listener-based component cannot be sourced elsewhere. + `configs/fa_ner_sm.cfg` embeds the tok2vec instead, as `en_core_web_sm` does. +3. `morphologizer` + `trainable_lemmatizer` instead of `attribute_ruler` + rule lemmatizer. The + English pipelines derive UPOS from PTB tags by rule because OntoNotes has no UPOS. UD gives + gold UPOS, FEATS and lemmas, which yields real `pos_acc`, `morph_acc` and `lemma_acc` numbers + instead of unmeasurable rule coverage. +4. PerDT, not Seraji: 3.7x more tokens, and Seraji has no `PROPN` tag. ## Roadmap -In value order, not difficulty order: - -1. **Prose-genre NER** → [`../ner_dataset`](../ner_dataset/PLAN.md). This is what unlocks a real - `fa_core_news_sm`. Note that PerDT's XPOS already encodes animacy on proper nouns - (`N_ANM` 6,752 vs `N_IANM` 12,682), which is a strong free prior for PER vs LOC/ORG. -2. **`md`/`lg`** need floret vectors trained on Persian Wikipedia + OSCAR (see - `spacy-vectors-builder`); floret rather than classic fastText because Persian's ZWNJ usage - is inconsistent and explodes the surface vocabulary. -3. **`trf`** needs a rented GPU and should use `HooshvareLab/roberta-fa-zwnj-base` - (Apache-2.0) rather than ParsBERT, whose model card carries no licence. -4. **`senter`** is one extra training run away. -5. **Upstream PRs** to `spacy/lang/fa` — see [`docs/upstream/fa-noun-chunks.md`](docs/upstream/fa-noun-chunks.md). - -Details in [`docs/MODELS.md`](docs/MODELS.md) §2. +1. A human-annotated NER test set, ~500 sentences. PerDT's entity labels and its NER test split + are both silver, so `ENTS_F 71.87` is not yet a fact. Tracked in + [`../ner_dataset`](../ner_dataset/PLAN.md). +2. A mixed-genre variant. Measured: this prose-trained NER scores 45.72 F on tweets, and mixing + ParsTwiNER in recovers that to 66.49 for 0.69 F on prose. That belongs in a separate package + rather than inside a `news` one. +3. `md` and `lg` need floret vectors trained on Persian Wikipedia and OSCAR (see + `spacy-vectors-builder`). Floret rather than classic fastText, because inconsistent ZWNJ + usage explodes the surface vocabulary. +4. `trf` needs a rented GPU and should use `HooshvareLab/roberta-fa-zwnj-base` (Apache-2.0) + rather than ParsBERT, whose model card carries no licence. +5. `senter` is one extra training run. +6. Upstream PRs to `spacy/lang/fa`, see [`docs/upstream/fa-noun-chunks.md`](docs/upstream/fa-noun-chunks.md). diff --git a/docs/CONTRIBUTING-GUIDE.md b/docs/CONTRIBUTING-GUIDE.md index 2b8c2ef..de70337 100644 --- a/docs/CONTRIBUTING-GUIDE.md +++ b/docs/CONTRIBUTING-GUIDE.md @@ -1,58 +1,54 @@ -# Contribution guidelines: getting a Persian pipeline into the spaCy ecosystem +# Getting a Persian pipeline into the spaCy ecosystem -Everything below is sourced from spaCy's own primary docs/repos (URLs inline). Read this -before writing code, because **where** a contribution goes decides how it must be shaped. +Sourced from spaCy's primary docs and repos, URLs inline. Where a contribution goes decides +how it has to be shaped, so read this before writing code. -## 0. The one thing to internalise - -spaCy splits Persian support into two *completely different* contribution surfaces: +## 0. Two separate contribution surfaces | Surface | What it is | Where it lives | How you contribute | | --- | --- | --- | --- | -| **Language data** (`fa`) | Hand-written rules: tokenizer exceptions, stop words, `LIKE_NUM`, punctuation, noun-chunk iterator | `spacy/lang/fa/*.py` inside the spaCy repo | Normal PR to `explosion/spaCy` | -| **Trained pipeline** (`fa_dep_news_sm`) | Statistical weights + `config.cfg` + `meta.json`, shipped as a pip wheel | `explosion/spacy-models` releases | **You cannot.** Publish it yourself (PyPI / HF Hub) and get it listed in spaCy Universe | +| Language data (`fa`) | Hand-written rules: tokenizer exceptions, stop words, `LIKE_NUM`, punctuation, noun-chunk iterator | `spacy/lang/fa/*.py` inside the spaCy repo | Normal PR to `explosion/spaCy` | +| Trained pipeline (`fa_dep_news_sm`) | Statistical weights + `config.cfg` + `meta.json`, shipped as a pip wheel | `explosion/spacy-models` releases | You cannot. Publish it yourself (PyPI or HF Hub) and get it listed in spaCy Universe | -`spacy/lang/fa` **already exists upstream**. What does not exist is any trained `fa` pipeline. -So this project is a *publishing* project, not an upstream-PR project — with optional -upstream PRs for the language-data gaps we find along the way. +`spacy/lang/fa` already exists upstream. No trained `fa` pipeline does. So this is a publishing +project, with optional upstream PRs for the language-data gaps found along the way. ## 1. Policy, verbatim From : -- **No CLA.** There is no contributor licence agreement section; contribution is an ordinary +- No CLA. There is no contributor licence agreement section; contribution is an ordinary GitHub PR governed by the Contributor Covenant Code of Conduct v1.4. -- Inclusion philosophy — this is the sentence that decides everything: +- Inclusion philosophy: > "Our philosophy is to prefer a smaller core library. […] If you're looking to implement a > new spaCy feature, starting with a custom component package is usually the best strategy. > […] And if it works well, we can always integrate it into the core library later." -- The only sanctioned route for non-trivial additions is the *Publishing spaCy extensions and - plugins* section: +- The sanctioned route for non-trivial additions is the "Publishing spaCy extensions and + plugins" section: > "An extension or plugin should add substantial functionality, be well-documented and > open-source. It should be available for users to download and install as a Python package > – for example via PyPi." > "Once your extension is published, you can open a PR to suggest it for the Universe page." -- `CONTRIBUTING.md` contains **zero** mention of submitting trained models. Neither does - — its README only documents `compatibility.json` +- `CONTRIBUTING.md` never mentions submitting trained models. Neither does + , whose README only documents `compatibility.json` as "the source of spaCy's internal compatibility check, performed when you run the download command". There is no PR template, issue label, or documented process for a third party to - land a pipeline in `spacy download`. - **Conclusion: the `spacy download` index is an Explosion-only release channel.** - (`[INFERENCE]` from the absence of any process across all three primary sources.) -- `https://spacy.io/usage/adding-languages` no longer exists; it redirects to - . The `BaseDefaults` / - `Language` contract is documented at . + land a pipeline in `spacy download`. So the `spacy download` index is an Explosion-only + release channel. (`[INFERENCE]` from the absence of any process across all three sources.) +- `https://spacy.io/usage/adding-languages` no longer exists and redirects to + . The `BaseDefaults` and `Language` + contract is at . -### Code rules that apply to a `spacy/lang/fa` PR +### Code rules for a `spacy/lang/fa` PR - `black` formatting, `flake8` clean, type hints where practical. - Tests go in `spacy/tests/lang/fa/`; regression tests use `@pytest.mark.issue(N)`. - Touching `.pyx` means the reviewer must rebuild: `python setup.py build_ext --inplace`. -- Language data must be *pure data + rules*: no model downloads, no network, no heavy deps. +- Language data must be data and rules only: no model downloads, no network, no heavy deps. ## 2. Three publishing routes for the trained pipeline -1. **Hugging Face Hub** — Explosion's own tool, the path of least resistance +1. Hugging Face Hub, using Explosion's own tool (): ```bash pip install spacy-huggingface-hub @@ -61,18 +57,18 @@ From : python -m spacy huggingface-hub push packages/fa_dep_news_sm-3.8.0/dist/fa_dep_news_sm-3.8.0-py3-none-any.whl --org ``` Users then `pip install https://huggingface.co//fa_dep_news_sm/resolve/main/fa_dep_news_sm-any-py3-none-any.whl`. -2. **PyPI / self-hosted wheel** — `spacy package … --build sdist,wheel` then `twine upload`, - or attach the wheel to a GitHub Release. See . -3. **spaCy Universe listing** — lists the package on spacy.io but hosts nothing. Per +2. PyPI or a self-hosted wheel: `spacy package … --build sdist,wheel` then `twine upload`, or + attach the wheel to a GitHub Release. See . +3. spaCy Universe, which lists the package on spacy.io but hosts nothing. Per : fork `explosion/spaCy`, - append an entry to `website/meta/universe.json`, open a PR. Required fields: - `id`, `title`, `slogan`, `description`, `github`, `pip`, `code_example`, `code_language`, - `url`, `thumb`, `image`, `author`, `author_links`, `category`, `tags`. - Checklist: must be **open-source with a user-friendly license** and "at least somewhat documented". + append an entry to `website/meta/universe.json`, open a PR. Required fields: `id`, `title`, + `slogan`, `description`, `github`, `pip`, `code_example`, `code_language`, `url`, `thumb`, + `image`, `author`, `author_links`, `category`, `tags`. The package must be open-source with + a user-friendly license and "at least somewhat documented". -**Route we take:** HF Hub for the artifact + a Universe PR once metrics are respectable. +Route taken here: HF Hub for the artifact, plus a Universe PR once metrics are respectable. -## 3. Naming and versioning conventions (non-negotiable if you want to look official) +## 3. Naming and versioning From and the `spacy-models` README: @@ -83,7 +79,7 @@ From and the `spacy-models` README: | Slot | Allowed values | Meaning | | --- | --- | --- | | `type` | `core` | tagger + parser + lemmatizer + NER | -| | `dep` | tagger + parser + lemmatizer, **no NER** | +| | `dep` | tagger + parser + lemmatizer, no NER | | | `ent` | NER only | | | `sent` | sentence segmentation only | | `genre` | `news`, `web`, `wiki` | text domain of the training corpus | @@ -92,27 +88,27 @@ From and the `spacy-models` README: | | `lg` | ~500k vectors (or 200k floret rows) | | | `trf` | transformer, no static vectors | -**Version numbers are `a.b.c` = spaCy-major . spaCy-minor . model-revision.** Trained against -spaCy 3.8 → the package version starts at `3.8.0`. Retraining on new data bumps `c`. -Do **not** invent `0.1.0`-style versions; that is exactly how hazm's HF pipelines ended up with -`version: 0.0.0` and `spacy_version: >=3.6.0,<3.7.0`, which is unusable metadata. +Version numbers are `a.b.c` = spaCy-major, spaCy-minor, model-revision. Trained against spaCy +3.8, so the package version starts at `3.8.0`; retraining on new data bumps `c`. Avoid +`0.1.0`-style versions, which is how hazm's HF pipelines ended up with `version: 0.0.0` and +`spacy_version: >=3.6.0,<3.7.0`. ### `meta.json` fields that matter () `lang`, `name`, `version`, `spacy_version` (range), `description`, `author`, `email`, `url`, -`license`, **`sources`** (list of `{name, url, author, license}` — this is where treebank -provenance and its CC BY-SA obligation must be recorded), `requirements` (extra pip deps -injected into the generated `setup.cfg`), `vectors`, `pipeline`, `labels`, `performance` -(auto-filled by `spacy train`), `speed`, `spacy_git_version`. +`license`, `sources` (list of `{name, url, author, license}`, which is where treebank +provenance and its CC BY-SA obligation is recorded), `requirements` (extra pip deps injected +into the generated `setup.cfg`), `vectors`, `pipeline`, `labels`, `performance` (auto-filled by +`spacy train`), `speed`, `spacy_git_version`. -Note: in v3, `meta.json` "isn't used to construct the language class and pipeline anymore" — -`config.cfg` is the single source of truth for loading. `meta.json` is metadata + packaging. +In v3, `meta.json` "isn't used to construct the language class and pipeline anymore". +`config.cfg` is the single source of truth for loading; `meta.json` is metadata and packaging. -## 4. Canonical training workflow (what Explosion actually runs) +## 4. Canonical training workflow Template: -(`project.yml` + `configs/default.cfg`). Directory layout `assets / corpus / configs / -training / metrics / packages` is the convention — this repo follows it. +(`project.yml` + `configs/default.cfg`). The directory layout `assets / corpus / configs / +training / metrics / packages` is the convention, which this repo follows. ```bash # 1. assets: clone the UD treebank @@ -136,54 +132,55 @@ python -m spacy train configs/default.cfg --output training/ --gpu-id -1 \ # 6. evaluate on held-out test python -m spacy benchmark accuracy training/model-best corpus/test.spacy --output metrics/test.json -# ('spacy evaluate' is now just an alias for 'benchmark accuracy') +# ('spacy evaluate' is now an alias for 'benchmark accuracy') # 7. package python -m spacy package training/fa_dep_news_sm packages --name dep_news_sm --version 3.8.0 --build sdist,wheel ``` -`spacy assemble` builds a pipeline from a config **without training** — useful for a -rule-only artifact (tokenizer + lookup lemmatizer + stop words). +`spacy assemble` builds a pipeline from a config without training, which suits a rule-only +artifact (tokenizer + lookup lemmatizer + stop words). -### Flags worth knowing on `spacy convert` +### Flags that matter on `spacy convert` -- `--converter conllu` — explicit is better than `auto`. -- `--n-sents N` — how many sentences per `Doc`; 10 is the Explosion default and gives the - parser/senter cross-sentence context. -- `--merge-subtokens` — spaCy has **no multi-word-token layer**. Persian UD treebanks use MWT - ranges for clitics (`پدرم` → `پدر` + `م`), so this flag decides whether gold tokens are - clitic-split or fused. See `docs/MODELS.md` for the tokenisation decision. -- `--morphology` — appends morph features to the tag (only if you *aren't* using a morphologizer). +- `--converter conllu`, explicit rather than `auto`. +- `--n-sents N`, how many sentences per `Doc`. 10 is the Explosion default and gives the parser + and senter cross-sentence context. +- `--merge-subtokens`. spaCy has no multi-word-token layer, and Persian UD treebanks use MWT + ranges for clitics (`پدرم` = `پدر` + `م`), so this flag decides whether gold tokens are + clitic-split or fused. See `docs/MODELS.md` §5. +- `--morphology`, which appends morph features to the tag. Only for pipelines without a + morphologizer. -### How the four size tiers differ mechanically +### How the size tiers differ mechanically | Tier | Embedding source | Config difference | | --- | --- | --- | -| `sm` | hash embeddings only | `spacy.MultiHashEmbed` with `include_static_vectors = false`, `spacy.MaxoutWindowEncoder` width 96, everything else a `spacy.Tok2VecListener` on the shared `tok2vec` | -| `md` / `lg` | + static vectors | same CNN, `include_static_vectors = true`, `[initialize] vectors = `; `lg` just has a bigger table | -| `trf` | contextual | replace `tok2vec` with a `transformer` component; every other component listens via `TransformerListener` instead of `Tok2VecListener` | +| `sm` | hash embeddings only | `spacy.MultiHashEmbed` with `include_static_vectors = false`, `spacy.MaxoutWindowEncoder` width 96, every other component a `spacy.Tok2VecListener` on the shared `tok2vec` | +| `md` / `lg` | + static vectors | same CNN, `include_static_vectors = true`, `[initialize] vectors = `; `lg` has a bigger table | +| `trf` | contextual | replace `tok2vec` with a `transformer` component; other components listen via `TransformerListener` instead of `Tok2VecListener` | -`spacy pretrain` (Tok2Vec LM-style pretraining on raw text → `[initialize] init_tok2vec`) is +`spacy pretrain` (Tok2Vec LM-style pretraining on raw text into `[initialize] init_tok2vec`) is orthogonal to the tier and optional. -## 5. Upstream `spacy/lang/fa` — current state and gaps +## 5. Upstream `spacy/lang/fa`: current state and gaps Contents of : -| File | Size | What it gives us | +| File | Size | What it provides | | --- | --- | --- | -| `__init__.py` | 1.3 KB | `PersianDefaults`: tokenizer exceptions, `TOKENIZER_SUFFIXES`, `LEX_ATTRS`, `SYNTAX_ITERATORS`, `STOP_WORDS`, `writing_system = {"direction": "rtl", "has_case": False, "has_letters": True}`; registers a **`lemmatizer` factory defaulting to `mode="rule"`** | -| `tokenizer_exceptions.py` | 64.9 KB | Large generated compound-verb / enclitic exception table | +| `__init__.py` | 1.3 KB | `PersianDefaults`: tokenizer exceptions, `TOKENIZER_SUFFIXES`, `LEX_ATTRS`, `SYNTAX_ITERATORS`, `STOP_WORDS`, `writing_system = {"direction": "rtl", "has_case": False, "has_letters": True}`; registers a `lemmatizer` factory defaulting to `mode="rule"` | +| `tokenizer_exceptions.py` | 64.9 KB | Generated compound-verb and enclitic exception table | | `generate_verbs_exc.py` | 14.8 KB | Dev script that generates the above | -| `stop_words.py` | 3.8 KB | ~500 stop words, comment says **"Stop words from HAZM package"** | -| `lex_attrs.py` | 1.4 KB | `LIKE_NUM` only (Persian numerals + `ام`/`ین` suffixes) | -| `punctuation.py` | 508 B | `TOKENIZER_SUFFIXES` only — **no prefixes, no infixes** | -| `syntax_iterators.py` | 1.6 KB | `noun_chunks()`, needs a trained parser to do anything | +| `stop_words.py` | 3.8 KB | ~500 stop words; the comment says "Stop words from HAZM package" | +| `lex_attrs.py` | 1.4 KB | `LIKE_NUM` only (Persian numerals plus `ام` and `ین` suffixes) | +| `punctuation.py` | 508 B | `TOKENIZER_SUFFIXES` only, no prefixes and no infixes | +| `syntax_iterators.py` | 1.6 KB | `noun_chunks()`, which needs a trained parser | -Tests: `spacy/tests/lang/fa/` has only `test_noun_chunks.py`. No tokenizer, lemmatizer, or +Tests: `spacy/tests/lang/fa/` has only `test_noun_chunks.py`. No tokenizer, lemmatizer or stop-word tests. -`spacy-lookups-data` (MIT) **already ships Persian rule-lemmatizer tables**: +`spacy-lookups-data` (MIT) already ships Persian rule-lemmatizer tables: | File | Size | | --- | --- | @@ -192,48 +189,49 @@ stop-word tests. | `fa_lemma_rules.json` | 882 B | | `fa_source.txt` | "extracted from Mojgan Seraji's Persian Universal Dependencies Corpus" | -Missing there: `fa_lemma_lookup.json` (no lookup-mode table), `fa_lexeme_norm.json`, +Missing there: `fa_lemma_lookup.json` (no lookup-mode table), `fa_lexeme_norm.json`, and `fa_license.txt` (Catalan has one; Persian's Seraji-derived provenance is undocumented). -**Gap list for a real `fa_core_news_*`:** +Gaps blocking a full `fa_core_news_*`: -1. No trained artefacts at all — no weights, no `config.cfg`, no `meta.json`, no entry in +1. No trained artefacts: no weights, no `config.cfg`, no `meta.json`, no entry in `compatibility.json`. `spacy.load("fa_core_news_sm")` fails today. -2. No word vectors for `fa` → `md`/`lg` need vectors built from scratch. -3. No NER data or labels anywhere in `spacy/lang/fa` → `core` (which implies NER) needs an - external annotated corpus. -4. Rule lemmatizer needs `token.pos` from a tagger/morphologizer → chicken-and-egg until the - tagger exists (or use a `trainable_lemmatizer`, which learns edit trees and needs no tables). -5. No Persian-specific prefix/infix rules — ZWNJ (U+200C) is only handled implicitly through +2. No word vectors for `fa`, so `md` and `lg` need vectors built from scratch. +3. No NER data or labels anywhere in `spacy/lang/fa`, so `core` needs an external corpus. +4. The rule lemmatizer needs `token.pos` from a tagger or morphologizer, which is + chicken-and-egg until the tagger exists. `trainable_lemmatizer` avoids this by learning edit + trees and needing no tables. +5. No Persian-specific prefix or infix rules. ZWNJ (U+200C) is handled only implicitly through the verb-exception table. -6. Zero tokenizer test coverage for a 65 KB exception table. +6. No tokenizer test coverage for a 65 KB exception table. -Items 5–7 are legitimate upstream PR material; items 1–4 are this project's job. +Items 5 and 6 are upstream PR material, along with the `noun_chunks` bug in +`docs/upstream/fa-noun-chunks.md`. Items 1 to 4 are this project's job. -## 6. Prior art we must not duplicate badly +## 6. Prior art -hazm publishes **spaCy-format** Persian pipelines on the HF Hub (MIT-ish, but `license` field -left empty), each a single-task pipeline built on ParsBERT: +hazm publishes spaCy-format Persian pipelines on the HF Hub, each a single-task pipeline built +on ParsBERT, with the `license` field left empty: | HF repo | Pipeline | Reported | Trained on (`[paths]` in its `config.cfg`) | | --- | --- | --- | --- | | `roshan-research/hazm-parsbert-postagger` | `transformer, tagger` | `tag_acc` 0.9862 | `data_train_98_rs10.spacy` (hazm's own EZ-augmented tagset: `NOUN,EZ`, `ADJ,EZ`, …) | -| `roshan-research/hazm-bert-dependency-parser` | `transformer, parser` | `dep_uas` 0.9246 / `dep_las` 0.8934 | `modified_fa_perdt-ud-train.spacy` — **UD_Persian-PerDT** | +| `roshan-research/hazm-bert-dependency-parser` | `transformer, parser` | `dep_uas` 0.9246 / `dep_las` 0.8934 | `modified_fa_perdt-ud-train.spacy`, i.e. UD_Persian-PerDT | | `roshan-research/hazm-parsbert-chunker` | `transformer, tagger` (IOB chunk tags) | `tag_acc` 0.9618 | hazm chunk data | -Both use `spacy-transformers` `TransformerModel.v3` on -`HooshvareLab/bert-base-parsbert-uncased`, `strided_spans` window 128 / stride 96, +All three use `spacy-transformers` `TransformerModel.v3` on +`HooshvareLab/bert-base-parsbert-uncased`, `strided_spans` window 128 and stride 96, with `spacy_version >= 3.6.0,<3.7.0`. -What they get wrong, and what we fix: +Problems this project avoids: -- Three separate pipelines instead of one `core` pipeline → users pay for three BERT forward - passes and cannot share a `Doc`. -- `version: 0.0.0`, empty `license`/`author`/`sources`, `name: "pipeline"` → violates every - convention in §3 and makes redistribution legally murky. -- Transformer-only, so no CPU-friendly tier at all. -- No lemmatizer, no morphologizer, no NER, no vectors. -- Pinned to spaCy 3.6; unusable on 3.8 without retraining. +- Three separate pipelines rather than one, so users pay for three BERT forward passes and + cannot share a `Doc`. +- `version: 0.0.0`, empty `license`, `author` and `sources`, and `name: "pipeline"`, which + breaks the conventions in §3 and leaves redistribution rights unclear. +- Transformer-only, with no CPU-friendly tier. +- No lemmatizer, morphologizer, NER or vectors. +- Pinned to spaCy 3.6, so unusable on 3.8 without retraining. -The valuable, reusable finding: **hazm's own parser recipe is "UD_Persian-PerDT + spaCy -`TransitionBasedParser`"**, which independently confirms the corpus choice in `docs/MODELS.md`. +The reusable finding is hazm's parser recipe, UD_Persian-PerDT plus spaCy's +`TransitionBasedParser`, which independently confirms the corpus choice in `docs/MODELS.md`. diff --git a/docs/MODELS.md b/docs/MODELS.md index a6c3404..d34ebe7 100644 --- a/docs/MODELS.md +++ b/docs/MODELS.md @@ -1,82 +1,87 @@ # The Persian pipelines: what to build, from what, and why -## 1. What "the 4 English pipelines" actually are +## 1. What the four English pipelines are -They are **one pipeline design at four embedding budgets**, not four different products. -Every one of them has the same components; the only real axis of variation is where token -representations come from. +One pipeline design at four embedding budgets. Every one has the same components; the axis of +variation is where token representations come from. | | `en_core_web_sm` | `en_core_web_md` | `en_core_web_lg` | `en_core_web_trf` | | --- | --- | --- | --- | --- | | Size on disk | 12 MB | 31 MB | 382 MB | 436 MB | | Embeddings | hash embeddings only | 685k keys / 20k vectors (300d) | 685k keys / 343k vectors (300d) | `roberta-base`, 768d contextual | -| Components | tok2vec, tagger, parser, senter, attribute_ruler, lemmatizer, ner | same | same | **transformer**, tagger, parser, attribute_ruler, lemmatizer, ner | +| Components | tok2vec, tagger, parser, senter, attribute_ruler, lemmatizer, ner | same | same | transformer, tagger, parser, attribute_ruler, lemmatizer, ner | | `TAG_ACC` | 0.97 | 0.97 | 0.97 | 0.98 | | `DEP_UAS` / `LAS` | 0.92 / 0.90 | 0.92 / 0.90 | 0.92 / 0.90 | 0.95 / 0.94 | | `ENTS_F` | 0.84 | 0.85 | 0.86 | 0.90 | | Training data | OntoNotes 5 (+ ClearNLP dep conversion, WordNet 3.0) | + Explosion vectors (OSCAR 2109 + Wikipedia + OpenSubtitles + WMT News Crawl) | same | OntoNotes 5 + roberta-base | -Read the accuracy table honestly: **static vectors buy almost nothing for tagging and -parsing** (identical to two decimals) and ~1–2 F on NER. The transformer buys ~4 LAS and -~6 NER F, at 36× the size and a GPU requirement. That ordering dictates the roadmap below. +Static vectors buy nothing measurable for tagging and parsing (identical to two decimals) and +1 to 2 F on NER. The transformer buys about 4 LAS and 6 NER F, at 36x the size and a GPU +requirement. That ordering sets the roadmap below. -Sources: . +Source: . ## 2. Target: the Persian pipelines -Naming follows `[lang]_[type]_[genre]_[size]` (), and the -`type` slot is load-bearing: `dep` = tagger + parser + lemmatizer, `ent` = NER only, -`core` = both. Genre is `news`, after the dominant genre of UD_Persian-PerDT (its README lists -"news fiction nonfiction academic web blog", and spaCy labels comparable treebank-trained -pipelines such as `de_core_news_sm` as `news`). +Naming follows `[lang]_[type]_[genre]_[size]` (). The +`type` slot carries real information: `dep` = tagger + parser + lemmatizer, `ent` = NER only, +`core` = both. Genre is `news`, after the dominant genre of UD_Persian-PerDT, whose README +lists "news fiction nonfiction academic web blog". spaCy labels comparable treebank-trained +pipelines such as `de_core_news_sm` as `news`. | Pipeline | Components | Embeddings | Status | | --- | --- | --- | --- | -| **`fa_dep_news_sm`** | tok2vec, tagger, morphologizer, trainable_lemmatizer, parser | hash embeddings | **built — the shipping artifact** | -| `fa_ent_news_sm` | ner (own internal tok2vec) | hash embeddings | **built — optional, separate package** | -| `fa_core_news_sm` | the two above, merged | hash embeddings | **reserved.** Blocked on prose-genre NER data from `../ner_dataset` | +| `fa_dep_news_sm` | tok2vec, tagger, morphologizer, trainable_lemmatizer, parser | hash embeddings | built, shipping | +| `fa_core_news_sm` | the above plus ner | hash embeddings | built, shipping | +| `fa_ent_news_sm` | ner (own internal tok2vec) | hash embeddings | built, optional | +| `fa_core_web_sm` | same as core, mixed-genre training data | hash embeddings | not built; would add ParsTwiNER to cover social media | | `fa_core_news_md` | + static vectors | floret, 50k rows | vectors must be trained first (CPU-days on fa Wikipedia + OSCAR) | | `fa_core_news_lg` | same | floret, 200k rows | same as md, bigger table | -| `fa_core_news_trf` | transformer instead of tok2vec | `HooshvareLab/roberta-fa-zwnj-base` (Apache-2.0) | **not on this hardware** — 2 GB VRAM cannot fine-tune a 125M-param encoder | +| `fa_core_news_trf` | transformer instead of tok2vec | `HooshvareLab/roberta-fa-zwnj-base` (Apache-2.0) | not on this hardware; 2 GB VRAM cannot fine-tune a 125M-param encoder | -**Why `dep` + `ent` rather than a single `core`.** `core` is a promise that the NER is part of -the same pipeline, built to the same standard, versioned together. Ours is not: the UD -components score 85–98 on edited prose, while the NER scores 67.22 F and is trained on tweets -because the good Persian NER corpora are research-use-only. One package name and one version -number would paper over a gap of that size. Shipping two packages makes the user opt into the -weak component knowingly, and costs nothing technically — `fa_ent_news_sm` embeds its own -tok2vec, so `nlp.add_pipe("ner", source=...)` reassembles a `core`-equivalent pipeline at -runtime (verified). `fa_core_news_sm` gets published when the NER earns the name. +### Why `core` is honest here -One deviation from the English design, deliberate: Persian gets a **`morphologizer`** -(UPOS + morphological features) and a **`trainable_lemmatizer`** instead of English's -`attribute_ruler` + rule `lemmatizer`. Reasons: +`core` promises that the NER belongs to the same pipeline, built to the same standard and +versioned together. That holds because PerDT ships its own entity layer in +`not-to-release/Dadegan with NER tag/`: 15,833 entities over the same 29,107 sentences, under +the same CC BY-SA 4.0, in the same genre, aligned to the same tokenization. -- The English pipelines use `attribute_ruler` because OntoNotes gives them PTB `tag`s and - they *derive* UPOS from tags by rule. UD treebanks give UPOS and FEATS as gold data — - training a morphologizer on them is strictly more information, and Persian morphology - (Number, Person, Tense, Mood, Voice, Polarity, PronType) is worth predicting. -- Persian rule-lemmatizer tables *do* exist in `spacy-lookups-data` (`fa_lemma_exc.json` - 1.68 MB, `fa_lemma_index.json`, `fa_lemma_rules.json`, derived from Seraji's treebank), but - a rule lemmatizer's accuracy is unmeasurable against the corpus it was extracted from and it - needs `token.pos` to work at all. `trainable_lemmatizer` learns edit trees from PerDT's gold - lemmas and reports a real `lemma_acc`. PerDT yields **1,908 edit trees** with 100% lemma - coverage — plenty. +An earlier revision of this project refused to ship `core`, and was right to given what it knew +then. The only redistributable Persian NER corpus found at that point was ParsTwiNER, a Twitter +corpus scoring 67.22 F against 85-98 for the UD components, and folding that into one package +would have hidden a genre and quality gap behind a single name and version number. Finding the +treebank's own layer removed the objection rather than answering it. -`senter` is intentionally omitted from v1: it is a separately trained component that ships -*disabled by default* in the English pipelines, and the parser already produces sentence -boundaries. Adding it later requires only one extra training run. +`fa_dep_news_sm` still ships alongside `core`, for users who want a 7.5 MB syntax-only model or +who would rather not depend on silver entity labels. -## 3. Resource inventory (everything checked for license) +### Why `morphologizer` + `trainable_lemmatizer` -### 3.1 Treebanks — the tagger / morphologizer / lemmatizer / parser data +The English pipelines use `attribute_ruler` because OntoNotes gives them PTB tags and they +derive UPOS by rule. UD treebanks give UPOS and FEATS as gold data, so a morphologizer trained +on them has strictly more information, and Persian morphology (Number, Person, Tense, Mood, +Voice, Polarity, PronType) is worth predicting. -| Treebank | Sents | Tokens | License | LEMMA | FEATS | XPOS | PROPN? | Verdict | +Persian rule-lemmatizer tables do exist in `spacy-lookups-data` (`fa_lemma_exc.json` 1.68 MB, +`fa_lemma_index.json`, `fa_lemma_rules.json`, derived from Seraji's treebank), but a rule +lemmatizer's accuracy cannot be measured against the corpus it was extracted from, and it needs +`token.pos` to run at all. `trainable_lemmatizer` learns edit trees from PerDT's gold lemmas and +reports a real `lemma_acc`. PerDT yields 1,908 edit trees at 100% lemma coverage. + +`senter` is omitted from v1. It is a separately trained component that ships disabled by default +in the English pipelines, and the parser already produces sentence boundaries. Adding it later +takes one training run. + +## 3. Resource inventory, with licences + +### 3.1 Treebanks, for tagger / morphologizer / lemmatizer / parser + +| Treebank | Sents | Tokens | License | LEMMA | FEATS | XPOS | PROPN | Verdict | | --- | --- | --- | --- | --- | --- | --- | --- | --- | -| **UD_Persian-PerDT** (PerUDT v1.0) | 29,107 | ~509k | **CC BY-SA 4.0** | converted + corrections | converted + corrections | manual native (34 types) | **yes** | **CHOSEN** | -| UD_Persian-Seraji | 5,997 | ~152k | CC BY-SA 4.0 | manual native | manual native | manual native (30 types) | **no** | secondary / cross-eval | -| UD_Persian-PUD | 1,000 | — | CC BY-SA 4.0 | — | — | none | — | test-only, parallel corpus | -| UD_Persian-IPerUDT | tiny | — | CC BY-SA 4.0 | — | — | none | — | grammar examples, unusable | +| UD_Persian-PerDT (PerUDT v1.0) | 29,107 | ~509k | CC BY-SA 4.0 | converted + corrections | converted + corrections | manual native (34 types) | yes | chosen | +| UD_Persian-Seraji | 5,997 | ~152k | CC BY-SA 4.0 | manual native | manual native | manual native (30 types) | no | secondary, cross-eval | +| UD_Persian-PUD | 1,000 | | CC BY-SA 4.0 | | | none | | test-only, parallel corpus | +| UD_Persian-IPerUDT | tiny | | CC BY-SA 4.0 | | | none | | grammar examples, unusable | Measured locally with `scripts/inspect_treebanks.py` (train splits): @@ -86,161 +91,188 @@ fa_perdt-ud-train.conllu 26196 452496 6508 0 100.0 57.7 fa_seraji-ud-train.conllu 4798 121067 1117 0 100.0 65.0 15 30 39 ``` -Why PerDT over Seraji: +PerDT over Seraji for three reasons. It has 3.7x more training tokens (452k against 121k), and +at `sm` size data is the binding constraint. Seraji has no `PROPN`: proper nouns are tagged +`NOUN`, which breaks the downstream tasks people use spaCy for. And hazm independently chose +PerDT for its own spaCy dependency parser, whose `config.cfg` names +`modified_fa_perdt-ud-train.spacy`, which makes the numbers comparable. -1. **3.7× more training tokens** (452k vs 121k). At `sm` size, data is the binding constraint. -2. **Seraji has no `PROPN`** — proper nouns are tagged `NOUN`. A pipeline that cannot mark - proper nouns is crippled for exactly the downstream tasks people use spaCy for. -3. hazm independently chose PerDT for its own spaCy dependency parser — its - `config.cfg` names `modified_fa_perdt-ud-train.spacy` as the training set. Converging on - the same corpus makes our numbers comparable to theirs. +Seraji's richer manual FEATS and fully manual lemmas make it the natural cross-evaluation set +and a candidate for a future concatenated-corpus run. The two use different XPOS inventories, +so naive concatenation would corrupt the `tag` label space. -Seraji's advantages (richer manual FEATS, fully manual lemmas) are real; it is the natural -cross-evaluation set and a candidate for a future concatenated-corpus run. The two use -different XPOS inventories, so naive concatenation would corrupt the `tag` label space. +### 3.2 NER -### 3.2 NER — the one place with a licensing minefield - -| Dataset | Labels | Size | License | Usable? | +| Dataset | Labels | Size | License | Usable | | --- | --- | --- | --- | --- | -| **ParsTwiNER** | PER, ORG, LOC, NAT, POG, EVENT | 7,667 tweets / 233k tokens | **MIT** (verified via GitHub API on `overfit-ir/parstwiner`) | **CHOSEN** | +| PerDT's own NER layer | PER, LOC, ORG, DAT, MON, TIM, PCT | 29,107 sentences / 484k tokens / 15,833 entities | CC BY-SA 4.0, same as the treebank | chosen | +| ParsTwiNER | PER, ORG, LOC, NAT, POG, EVENT | 7,667 tweets / 233k tokens / 16,250 entities | MIT (verified via GitHub API on `overfit-ir/parstwiner`) | usable, wrong genre | | ARMAN (PersianNER) | 6 classes | 250k tokens | academic research only | no | | PEYMA | 7 classes | 302k tokens | "free for research purposes", no OSS licence | no | | NSURL-2019 Task 7 | PEYMA tagset | ~1M tokens | no explicit licence | no | | HooshvareLab merged ParsNER | 10 classes | ARMAN+PEYMA+WikiANN | inherits ARMAN/PEYMA restrictions; HF repo gated (401) | no | -This is not pedantry. **spaCy's own maintainers state that Persian models trained back in -2018 were never published precisely because of corpus licensing** (spaCy discussion #8233, -following PR #2797 which added only `spacy.blank("fa")` tokenizer support). Repeating that -mistake would waste the whole exercise: a pipeline you cannot legally redistribute is not a -pipeline, it is a local file. +spaCy's maintainers state that the Persian models trained in 2018 were never published because +of corpus licensing (spaCy discussion #8233, following PR #2797, which added only +`spacy.blank("fa")` tokenizer support). -Consequence to state plainly in the model card: **the NER component is trained on Twitter -text while the rest of the pipeline is trained on edited prose.** Expect NER to degrade on -formal news text relative to what an ARMAN/PEYMA-trained model would score. That is the -price of a redistributable artifact. +The PerDT layer lives in `not-to-release/Dadegan with NER tag/{train,dev,test}_with_NER_tag.txt` +as two-column IOB2. In UD convention `not-to-release/` means "excluded from the official UD +release build", normally working and source data, and the directory is public on GitHub under +the repo's `LICENSE.txt`. Confirm that reading with the PerDT authors before publishing anything +derived from it, since redistribution rights are the whole point. -### 3.3 Vectors (for md / lg) +Three properties of that layer decide how it gets used: + +1. **The labels are silver.** The treebank README states they came from the BERT-based + Beheshti-NER tagger (Taher et al., 2020) with manual corrections to extend recall. The + published `ents_f` is therefore measured against a silver test split and partly reflects + agreement with that tagger. A human-annotated test set is the outstanding work. +2. **Tokenization differs.** The NER files use the original Dadegan tokenization, which matches + the released UD tokenization exactly in only 57 to 62% of sentences: the NER files drop some + copulas and auxiliaries, and at least one honorific is corrupted (`ص` written as `،`). + Entities sit on content words present in both, so `scripts/transfer_perdt_ner.py` aligns them + with difflib, transferring 99.86% of train entities, 99.74% of dev and 99.51% of test. Spans + whose tokens do not all map contiguously are dropped rather than guessed. +3. **The label sets do not line up with ParsTwiNER.** PerDT has `DAT`, `MON`, `TIM` and `PCT`; + ParsTwiNER has `NAT`, `EVE` and `POG`. The intersection is `PER`, `LOC`, `ORG`. Since spaCy + assigns one label per token, concatenating the two raw would teach the model that dates are + `O` in half the corpus. Any mixed-genre variant has to solve that first. + +Genre matters more than any of this. Measured with one config on the three shared labels, a +PerDT-trained NER scores 72.80 F on prose and 45.72 on tweets, while a ParsTwiNER-trained one +scores 68.59 on tweets and 55.59 on prose. Training on both gives 72.11 and 66.49, so mixing +costs under 1 F on prose and 2 F on tweets while recovering roughly 20 F off-genre. That is the +argument for a future `fa_core_web_sm`, and the reason the current `news` packages stay +prose-only. + +### 3.3 Vectors, for md and lg | Option | License | Note | | --- | --- | --- | -| **floret vectors** trained via `spacy-vectors-builder` (MIT tooling) on fa Wikipedia + OSCAR | corpus-dependent | **recommended.** Subword + Bloom-hash embeddings: bounded table size, zero OOV | -| fastText `cc.fa.300` | CC BY-SA 3.0 | quick fallback; classic word table, large and OOV-prone | +| floret vectors trained via `spacy-vectors-builder` (MIT tooling) on fa Wikipedia + OSCAR | corpus-dependent | recommended: subword + Bloom-hash embeddings, bounded table size, zero OOV | +| fastText `cc.fa.300` | CC BY-SA 3.0 | fallback; classic word table, large and OOV-prone | -Floret is the right call for Persian specifically. Persian surface forms explode through -suffixation *and* through inconsistent ZWNJ (U+200C) usage — the same word appears as -`می‌رود` / `میرود` / `می رود` in real text. A classic word-vector table has a miss for every -variant; floret's subword hashing covers all of them. spaCy already ships floret vectors for -Croatian, Finnish, Korean, Slovenian, Swedish and Ukrainian for the same reason. +Persian surface forms multiply through suffixation and through inconsistent ZWNJ (U+200C) +usage: the same word appears as `می‌رود`, `میرود` and `می رود` in real text. A classic +word-vector table misses every variant it did not see, while floret's subword hashing covers +them. spaCy ships floret vectors for Croatian, Finnish, Korean, Slovenian, Swedish and +Ukrainian for the same reason. -### 3.4 Transformer encoders (for trf) +### 3.4 Transformer encoders, for trf | Model | Arch | License | Note | | --- | --- | --- | --- | -| **`HooshvareLab/roberta-fa-zwnj-base`** | RoBERTa-base | **Apache-2.0** | recommended: licensed, ZWNJ-aware, smallest of the credible options | +| `HooshvareLab/roberta-fa-zwnj-base` | RoBERTa-base | Apache-2.0 | recommended: licensed, ZWNJ-aware, smallest credible option | | `FacebookAI/xlm-roberta-base` | XLM-R base, 278M | MIT | licensed but larger | | `PartAI/TookaBERT-Base` | BERT-base | Apache-2.0 | licensed | | `m3hrdadfi/albert-fa-base-v2` | ALBERT-base-v2 | Apache-2.0 | licensed, smallest | -| `HooshvareLab/bert-base-parsbert-uncased` | BERT-base, ~162M | **no licence on the card** | what hazm used; redistribution risk | -| `sbunlp/fabert` | BERT-base, 124M | **no licence on the card** | redistribution risk | +| `HooshvareLab/bert-base-parsbert-uncased` | BERT-base, ~162M | no licence on the card | what hazm used; redistribution risk | +| `sbunlp/fabert` | BERT-base, 124M | no licence on the card | redistribution risk | -All of these are BERT/RoBERTa/XLM-R/ALBERT, so all are supported by both -`spacy-transformers` and `spacy-curated-transformers` (the latter supports exactly -ALBERT / BERT / CamemBERT / RoBERTa / XLM-RoBERTa). +All are BERT, RoBERTa, XLM-R or ALBERT, so all work with `spacy-transformers` and with +`spacy-curated-transformers`, which supports exactly ALBERT, BERT, CamemBERT, RoBERTa and +XLM-RoBERTa. -Note the trap: hazm's own pipelines use ParsBERT, which has **no license statement**. Copying -that choice would reintroduce the redistribution problem that sank the 2018 attempt. +hazm's pipelines use ParsBERT, which carries no licence statement. Copying that choice would +reintroduce the redistribution problem that stopped the 2018 attempt. -## 4. What already exists (and why it is not enough) +## 4. What already exists, and why it is not enough -- **No trained spaCy Persian pipeline exists.** Zero `persian` / `farsi` / `fa_` hits in - spaCy's `website/meta/universe.json`. `spacy.load("fa_core_news_sm")` has never worked. -- **hazm** ships three *single-task* spaCy pipelines on the HF Hub: - `hazm-parsbert-postagger` (`tag_acc` 0.9862, hazm's own EZ-augmented tagset), - `hazm-bert-dependency-parser` (`dep_uas` 0.9246 / `dep_las` 0.8934, trained on PerDT), - `hazm-parsbert-chunker` (`tag_acc` 0.9618). Each is `transformer + one component`, - `version: 0.0.0`, empty `license`/`author`/`sources`, pinned to spaCy 3.6. Using all three - means three separate BERT forward passes over the same text and no shared `Doc`. -- **hazm's training code is not reusable.** Its trainable models are pycrfsuite CRFs - (`hazm/sequence_tagger.py`); there is no `config.cfg` or `spacy train` anywhere in the repo. - Its `Spacy*` classes only *download* the HF pipelines above. -- **hazm's tokenizer is actively incompatible** with UD gold tokenization: `Normalizer`'s - `AFFIX_SPACING_PATTERNS` fuse ZWNJ affixes and `WordTokenizer.join_verb_parts()` glues - multi-word verb chains into single underscore-joined tokens. Training against PerDT with - hazm's tokenizer would misalign tokens systematically. -- **DadmaTools** (Apache-2.0 code) emits spaCy-compatible `Doc` objects but is not a loadable - spaCy pipeline package, and its NER wraps ARMAN/PEYMA — the restricted data again. +No trained spaCy Persian pipeline exists. There are zero `persian`, `farsi` or `fa_` hits in +spaCy's `website/meta/universe.json`, and `spacy.load("fa_core_news_sm")` has never worked. -So what hazm genuinely contributes to this project is **one validated design decision** -(PerDT is the corpus) and **the stop-word list already vendored into `spacy/lang/fa`**. -Everything else is built with spaCy-native tooling. +hazm ships three single-task spaCy pipelines on the HF Hub: `hazm-parsbert-postagger` +(`tag_acc` 0.9862, hazm's own EZ-augmented tagset), `hazm-bert-dependency-parser` (`dep_uas` +0.9246, `dep_las` 0.8934, trained on PerDT) and `hazm-parsbert-chunker` (`tag_acc` 0.9618). +Each is `transformer + one component`, `version: 0.0.0`, with empty `license`, `author` and +`sources`, pinned to spaCy 3.6. Using all three costs three BERT forward passes over the same +text and gives no shared `Doc`. + +hazm's training code is not reusable: its trainable models are pycrfsuite CRFs +(`hazm/sequence_tagger.py`), and the repo contains no `config.cfg` or `spacy train`. Its +`Spacy*` classes only download the HF pipelines above. + +hazm's tokenizer is incompatible with UD gold tokenization. `Normalizer`'s +`AFFIX_SPACING_PATTERNS` fuse ZWNJ affixes, and `WordTokenizer.join_verb_parts()` glues +multi-word verb chains into single underscore-joined tokens, so training against PerDT with it +would misalign tokens systematically. + +DadmaTools (Apache-2.0 code) emits spaCy-compatible `Doc` objects but is not a loadable spaCy +pipeline package, and its NER wraps ARMAN and PEYMA. + +What hazm contributes here is one validated design decision, that PerDT is the corpus, and the +stop-word list already vendored into `spacy/lang/fa`. ## 5. Decision: train the `sm` tier first -Not md/lg: those need floret vectors trained from scratch on Wikipedia+OSCAR (CPU-days) and -buy ~0.00 tag/dep accuracy in the English reference numbers. -Not trf: 2 GB of VRAM cannot fine-tune a 125M-param encoder, and renting a GPU should wait -until the CPU pipeline proves the data plumbing is right. -`sm` is also the tier every other tier is validated against: md/lg/trf reuse the exact same -corpus conversion, config skeleton and evaluation harness. +Not md or lg: they need floret vectors trained from scratch on Wikipedia and OSCAR, costing +CPU-days, and the English reference numbers show no tag or dep accuracy gain. Not trf: 2 GB of +VRAM cannot fine-tune a 125M-param encoder, and renting a GPU should wait until the CPU +pipeline proves the data plumbing. `sm` is also the tier the others are validated against, +since md, lg and trf reuse the same corpus conversion, config skeleton and evaluation harness. -### Tokenization decision, settled with a measurement +### Tokenization, settled by measurement spaCy has no multi-word-token layer, so CoNLL-U MWT ranges (Persian pronominal clitics and -copulas: `پدرم` = `پدر` + `م`) must either be merged into one token or kept split. Measured on +copulas, `پدرم` = `پدر` + `م`) must either be merged into one token or kept split. Measured on the PerDT dev set with `scripts/tokenization_report.py`, comparing gold boundaries against `spacy.blank("fa")`'s tokenizer: | `spacy convert` mode | token P | token R | token F | XPOS types | merge artefacts | | --- | --- | --- | --- | --- | --- | -| `--merge-subtokens` | 0.9860 | 0.9914 | **0.9887** | 68 | 34 composite tags on 1.49% of tokens; 1.49% lemmas contain a space | +| `--merge-subtokens` | 0.9860 | 0.9914 | 0.9887 | 68 | 34 composite tags on 1.49% of tokens; 1.49% lemmas contain a space | | plain (clitics split) | 0.9875 | 0.9772 | 0.9823 | 35 | none | -**Chosen: `--merge-subtokens`** (also what Explosion's `tagger_parser_ud` template does). -Rationale: without it, 1.5% of gold tokens are boundaries the shipped tokenizer can never -produce, so those tokens are permanently unlearnable and unpredictable at runtime. With it, -every gold token is reachable; the cost is confined to rare composite XPOS tags such as -`N_IANM_PR_JOPER` (noun + enclitic pronoun) — which are, at least, informative — and 1.5% of -lemmas that come out as two words. +Chosen: `--merge-subtokens`, which is also what Explosion's `tagger_parser_ud` template does. +Without it, 1.5% of gold tokens are boundaries the shipped tokenizer can never produce, so +those tokens are permanently unlearnable and unpredictable at runtime. With it, every gold +token is reachable, and the cost is limited to rare composite XPOS tags such as +`N_IANM_PR_JOPER` (noun + enclitic pronoun) and 1.5% of lemmas that come out as two words. -The better long-term fix is Persian clitic-splitting suffix rules in `spacy/lang/fa`, which -would be an upstream PR, not a model change. Recorded in `docs/CONTRIBUTING-GUIDE.md` §5. +The better long-term fix is Persian clitic-splitting suffix rules in `spacy/lang/fa`, which is +an upstream PR rather than a model change. Recorded in `docs/CONTRIBUTING-GUIDE.md` §5. ### Final composition -**`fa_dep_news_sm`** — CC BY-SA 4.0, 7.5 MB wheel: +Shared trained components, in both `fa_dep_news_sm` (7.5 MB) and `fa_core_news_sm` (13 MB), +both CC BY-SA 4.0: | Component | Trained on | Metric | Test score | | --- | --- | --- | --- | -| `tok2vec` | shared, PerDT | — | — | +| `tok2vec` | shared, PerDT | | | | `tagger` (XPOS) | PerDT, 90 labels | `tag_acc` | 95.96 | | `morphologizer` (UPOS + FEATS) | PerDT, 298 labels | `pos_acc` / `morph_acc` | 96.24 / 96.29 | | `trainable_lemmatizer` | PerDT, 1,908 edit trees | `lemma_acc` | 97.91 | | `parser` | PerDT, 34 deprels | `dep_uas` / `dep_las` | 89.69 / 85.15 | -**`fa_ent_news_sm`** — MIT, 5.6 MB wheel, separate package: +`fa_core_news_sm` adds, and `fa_ent_news_sm` ships alone: | Component | Trained on | Metric | Test score | | --- | --- | --- | --- | -| `ner` (own internal tok2vec) | ParsTwiNER, 6 labels | `ents_p/r/f` | 74.77 / 61.06 / **67.22** | +| `ner` (own internal tok2vec) | PerDT NER layer, 7 labels | `ents_p/r/f` | 77.67 / 66.87 / 71.87 | -ParsTwiNER's label counts explain that last row better than any prose can: -`PER` 6258, `LOC` 5478, `ORG` 2694, `NAT` 939, **`EVE` 482, `POG` 399** over 232,917 tokens -(16,250 entities, 7.0% density). The two starved labels are exactly the two that score worst -(`EVE` 30.0, `POG` 41.2). So the corpus is not small — it is skewed and out of genre, which -are two different problems: the head labels need in-genre data, the tail labels need more data -of any kind. +Per label, F against training examples: `LOC` 80.24 (4,954), `DAT` 74.45 (1,323), `MON` 73.68 +(205), `ORG` 68.77 (2,643), `TIM` 66.67 (135), `PER` 65.29 (4,847), `PCT` 57.14 (121). -Training cost on the target hardware (4-core i5-7200U, no GPU): **1h27m** for the UD -components (early-stopped at step 10,800; best checkpoint step 9,200) and ~25 min for NER -(best at step 4,000). Both runs are single-threaded, so they were run concurrently. -Inference: ~9,250 words/s. +`PER` scoring below `LOC` and `ORG` on nearly the same amount of data is the silver labels +showing through. PerDT includes titles and honorifics inside `PER` spans inconsistently: 6.24% of +its `PER` spans start with one (`دکتر`, `مهندس`, `آقای`), against 1.41% in the human-annotated +ParsTwiNER, so the boundaries are less regular than the count suggests. `MON`, `TIM` and `PCT` +are thin enough that their scores rest on 4 to 11 test entities each and should be treated as +indicative only. Persian money, times and percentages are regular enough that an `EntityRuler` +may beat the statistical model for those three. -The `--merge-subtokens` artefacts predicted above are visible in the shipped model exactly as -expected — `کتاب‌هایش` ("his/her books") comes out as one token tagged `N_IANM_PR_JOPER` with -lemma `کتاب او`. Worth knowing before you consume `token.lemma_` downstream. +Training cost on a 4-core i5-7200U with no GPU: 1h27m for the UD components (early-stopped at +step 10,800, best checkpoint step 9,200) and 17 min for NER (best at step 6,000). Both runs are +single-threaded and can run concurrently. Inference runs at about 9,250 words/s. -Sources recorded in each `meta.json` with their licences, per -. CC BY-SA 4.0 on PerDT means `fa_dep_news_sm` must -carry attribution and share-alike notice — handled in `scripts/finalize_pipeline.py`, which -also refuses to publish a `dep` pipeline that contains an `ner` component. +The `--merge-subtokens` artefacts show up in the shipped model as predicted: `کتاب‌هایش` +("his/her books") is one token tagged `N_IANM_PR_JOPER` with lemma `کتاب او`. Check this before +consuming `token.lemma_` downstream. + +Sources are recorded in each `meta.json` with their licences, per +, including the treebank's NER layer as its own entry +crediting Beheshti-NER. CC BY-SA 4.0 on PerDT means every package carries attribution and a +share-alike notice, handled in `scripts/finalize_pipeline.py`, which also enforces the shape of +each variant: it refuses to publish a `dep` pipeline containing `ner`, or a `core` one without it. diff --git a/docs/upstream/fa-noun-chunks.md b/docs/upstream/fa-noun-chunks.md index 9e5e978..e269f06 100644 --- a/docs/upstream/fa-noun-chunks.md +++ b/docs/upstream/fa-noun-chunks.md @@ -9,20 +9,20 @@ labels = ["nsubj", "dobj", "nsubjpass", "pcomp", "pobj", "dative", "appos", "att ``` `dobj`, `nsubjpass`, `pobj`, `dative` and `attr` are ClearNLP/English labels. They are not -Universal Dependencies relations, and **every Persian treebank is UD** (`UD_Persian-PerDT`, -`UD_Persian-Seraji`, `UD_Persian-PUD`, `UD_Persian-IPerUDT`). Any trained `fa` pipeline must -therefore emit UD labels, so five of the nine labels are dead code and `doc.noun_chunks` -silently returns bare head nouns. +Universal Dependencies relations, and every Persian treebank is UD (`UD_Persian-PerDT`, +`UD_Persian-Seraji`, `UD_Persian-PUD`, `UD_Persian-IPerUDT`). Any trained `fa` pipeline emits +UD labels, so five of the nine labels are dead code and `doc.noun_chunks` returns bare head +nouns. -Compare `spacy/lang/fr/syntax_iterators.py` and `spacy/lang/es/syntax_iterators.py`, which -use UD labels (`nsubj`, `nsubj:pass`, `obj`, `obl`, `nmod`, `appos`, `ROOT`) because their -treebanks are UD too. `spacy/lang/de` legitimately uses TIGER labels (`sb`, `oa`, `nk`, …) -because the German pipelines are trained on TIGER. Persian has no such excuse. +`spacy/lang/fr/syntax_iterators.py` and `spacy/lang/es/syntax_iterators.py` use UD labels +(`nsubj`, `nsubj:pass`, `obj`, `obl`, `nmod`, `appos`, `ROOT`) because their treebanks are UD. +`spacy/lang/de` uses TIGER labels (`sb`, `oa`, `nk`, …) because the German pipelines are +trained on TIGER. Persian has no equivalent reason. -Second, smaller problem: the iterator yields `word.left_edge.i` → `word.i + 1`, i.e. it only -ever expands **left**. Persian noun phrases expand **right** through ezafe: -`رئیس انجمن جراحان قلب ایران` ("the head of the Iranian society of heart surgeons") is one NP -whose head is the leftmost token. Left-only expansion truncates it to `رئیس`. +A second, smaller problem: the iterator yields `word.left_edge.i` to `word.i + 1`, so it only +expands left. Persian noun phrases expand right through ezafe. `رئیس انجمن جراحان قلب ایران` +("the head of the Iranian society of heart surgeons") is one NP whose head is the leftmost +token, and left-only expansion truncates it to `رئیس`. ## Evidence @@ -48,9 +48,9 @@ deprels on NOUN/PROPN/PRON tokens in dev (top 15): nsubj:pass 38 - ``` -1.31 tokens per chunk is the tell: the shipped iterator is returning single head nouns. -`nmod` (2894), `obl` (1401), `obl:arg` (1095) and `obj` (973) — the four most common -noun-bearing relations after `nsubj` — are all unreachable. +At 1.31 tokens per chunk the shipped iterator is returning single head nouns. The four most +common noun-bearing relations after `nsubj`, namely `nmod` (2894), `obl` (1401), `obl:arg` +(1095) and `obj` (973), are all unreachable. Live text: @@ -66,8 +66,8 @@ Live text: ## Proposed patch -Swap in UD labels, expand right through modifier chains, and drop a leading `ADP`/`CCONJ` -exactly as `fr`/`es` already do: +Swap in UD labels, expand right through modifier chains, and drop a leading `ADP` or `CCONJ` +as `fr` and `es` already do: ```diff --- a/spacy/lang/fa/syntax_iterators.py @@ -176,7 +176,7 @@ Add UD-label coverage: ```python def test_fa_noun_chunks_ezafe(fa_vocab): - # رئیس انجمن جراحان — "head of the surgeons' society" + # رئیس انجمن جراحان, "head of the surgeons' society" words = ["رئیس", "انجمن", "جراحان", "آمد"] heads = [3, 0, 1, 3] deps = ["nsubj", "nmod", "nmod", "ROOT"] @@ -196,28 +196,28 @@ def test_fa_noun_chunks_drops_leading_adp(fa_vocab): ## Other `spacy/lang/fa` gaps found while building this pipeline -Ordered by how much they cost a real Persian pipeline: +Ordered by how much they cost a Persian pipeline. -1. **Clitic splitting.** The `fa` tokenizer cannot split pronominal enclitics or the enclitic - copula (`پدرم` → `پدر` + `م`, `ساکتند` → `ساکت` + `ند`), which UD treebanks annotate as - multiword tokens. Measured on PerDT dev: gold-vs-tokenizer token F is 0.9823 when clitics - are kept split, and 1.49% of tokens are affected. Persian-specific `TOKENIZER_SUFFIXES` - entries for the enclitic set would remove the need for `--merge-subtokens` and eliminate - the composite XPOS tags it produces. -2. **`punctuation.py` defines only `TOKENIZER_SUFFIXES`** — no `TOKENIZER_PREFIXES`, no +1. Clitic splitting. The `fa` tokenizer cannot split pronominal enclitics or the enclitic + copula (`پدرم` = `پدر` + `م`, `ساکتند` = `ساکت` + `ند`), which UD treebanks annotate as + multiword tokens. Measured on PerDT dev, gold-against-tokenizer token F is 0.9823 when + clitics are kept split, affecting 1.49% of tokens. Persian-specific `TOKENIZER_SUFFIXES` + entries for the enclitic set would remove the need for `--merge-subtokens` and the + composite XPOS tags it produces. +2. `punctuation.py` defines only `TOKENIZER_SUFFIXES`, with no `TOKENIZER_PREFIXES` and no `TOKENIZER_INFIXES`. ZWNJ (U+200C) is handled only implicitly through the 65 KB generated - verb-exception table. The missing infix rules bite on numerics: `spacy/lang/fa/examples.py` + verb-exception table. The missing infix rules break numerics: `spacy/lang/fa/examples.py` ships the sentence `دیروز علی به من ۲۰۰۰.۱﷼ پول نقد داد.`, and the trained pipeline splits - `۲۰۰۰.۱﷼` into `۲۰۰۰` + `.` + `۱﷼`, with the stray `.` promoted to a sentence boundary — - one input sentence comes out as three. `LIKE_NUM` in `lex_attrs.py` recognises Persian - digits, but no tokenizer rule keeps a Persian decimal or a currency sign attached. -3. **No tokenizer tests at all** for `fa` — `spacy/tests/lang/fa/` contains only - `test_noun_chunks.py`, guarding none of the 65 KB exception table. -4. **`spacy-lookups-data` has no `fa_license.txt`**, although `fa_source.txt` records that the - lemma tables were "extracted from Mojgan Seraji's Persian Universal Dependencies Corpus" — - which is CC BY-SA 4.0. Catalan ships a `ca_license.txt`; Persian should too. -5. **No `fa_lemma_lookup.json`** — only rule-mode lemmatizer assets exist, so - `mode="lookup"` is unavailable for Persian. + `۲۰۰۰.۱﷼` into `۲۰۰۰` + `.` + `۱﷼`, promoting the stray `.` to a sentence boundary, so one + input sentence comes out as three. `LIKE_NUM` in `lex_attrs.py` recognises Persian digits, + but no tokenizer rule keeps a Persian decimal or a currency sign attached. +3. No tokenizer tests for `fa`. `spacy/tests/lang/fa/` contains only `test_noun_chunks.py`, + which guards none of the 65 KB exception table. +4. `spacy-lookups-data` has no `fa_license.txt`, although `fa_source.txt` records that the + lemma tables were "extracted from Mojgan Seraji's Persian Universal Dependencies Corpus", + which is CC BY-SA 4.0. Catalan ships a `ca_license.txt`. +5. No `fa_lemma_lookup.json`. Only rule-mode lemmatizer assets exist, so `mode="lookup"` is + unavailable for Persian. -Items 1–3 are self-contained code PRs. Item 4 is a licence-hygiene PR against -`spacy-lookups-data` and matters for anyone redistributing a Persian pipeline. +Items 1 to 3 are self-contained code PRs. Item 4 is a licence-hygiene PR against +`spacy-lookups-data`, and affects anyone redistributing a Persian pipeline. diff --git a/project.yml b/project.yml index a09c41a..62f7ac8 100644 --- a/project.yml +++ b/project.yml @@ -1,27 +1,29 @@ -title: "fa_dep_news_sm" +title: "fa_core_news_sm / fa_dep_news_sm" description: > - A CPU-sized Persian (fa) dependency pipeline for spaCy 3.8: tagger (XPOS), - morphologizer (UPOS + FEATS), trainable lemmatizer and dependency parser. Trained on - UD_Persian-PerDT (PerUDT v1.0, CC BY-SA 4.0). + CPU-sized Persian (fa) pipelines for spaCy 3.8, built entirely from UD_Persian-PerDT + (PerUDT v1.0, CC BY-SA 4.0): tagger (XPOS), morphologizer (UPOS + FEATS), trainable + lemmatizer, dependency parser and NER. - This repo deliberately ships `dep`, not `core`. In spaCy's naming scheme `core` means - "tagger + parser + lemmatizer + NER" in one package, and the only redistributably-licensed - Persian NER corpus (ParsTwiNER, MIT) is a Twitter corpus — it scores 67.22 F against - 85-98 for the UD components, and folding it into a single `core` artifact would hide that - behind one package name. So NER ships separately and optionally as `fa_ent_news_sm` - (workflow: `ner`), and `fa_core_news_sm` is reserved for when ../ner_dataset delivers - prose-genre NER data that beats ParsTwiNER on a human-annotated test set. + Two shipping packages, same corpus, differing only in whether NER is included: + `fa_dep_news_sm` (no NER) and `fa_core_news_sm` (with NER). The NER comes from the + treebank's own `not-to-release/Dadegan with NER tag/` layer, so both packages share one + corpus, one genre, one tokenization and one licence. - See docs/MODELS.md for the source/licence analysis and docs/CONTRIBUTING-GUIDE.md for + PerDT's NER labels are silver, produced by Beheshti-NER with manual corrections, and are + transferred onto this pipeline's tokenization by difflib at a 99.86% rate. Per-label + scores are published in meta.json; MON, TIM and PCT are thin. + + See docs/MODELS.md for the source and licence analysis and docs/CONTRIBUTING-GUIDE.md for how this gets published. - Run the shipping pipeline with: `spacy project run all` - Optionally build the standalone NER with: `spacy project run ner` + Build both with: `spacy project run all` + Standalone NER-only package: `spacy project run ent` vars: lang: "fa" - package_name: "dep_news_sm" - ner_package_name: "ent_news_sm" + dep_package_name: "dep_news_sm" + core_package_name: "core_news_sm" + ent_package_name: "ent_news_sm" package_version: "3.8.0" treebank: "fa_perdt" # -1 = CPU. A GTX 940MX (2 GB) is not worth the transfer overhead for an sm pipeline. @@ -50,31 +52,42 @@ assets: url: "https://raw.githubusercontent.com/UniversalDependencies/UD_Persian-PerDT/master/fa_perdt-ud-test.conllu" checksum: "b62a66994cef2c50f7e524a1471102d8" description: "UD_Persian-PerDT test split (CC BY-SA 4.0)" - - dest: "assets/ner/ParsTwiNER_corpus_v1.0.0.zip" - url: "https://github.com/overfit-ir/parstwiner/releases/download/v1.0.0/ParsTwiNER_corpus_v1.0.0.zip" - checksum: "54615340d76c4d51b8f54751b07a278d" - description: "ParsTwiNER Persian Twitter NER corpus, IOB2 (MIT)" + - dest: "assets/ud-ner/train_with_NER_tag.txt" + url: "https://raw.githubusercontent.com/UniversalDependencies/UD_Persian-PerDT/master/not-to-release/Dadegan%20with%20NER%20tag/train_with_NER_tag.txt" + checksum: "ecb96cf99b38bc485cac21d22914e413" + description: "PerDT NER layer, train split, IOB2 (CC BY-SA 4.0)" + - dest: "assets/ud-ner/dev_with_NER_tag.txt" + url: "https://raw.githubusercontent.com/UniversalDependencies/UD_Persian-PerDT/master/not-to-release/Dadegan%20with%20NER%20tag/dev_with_NER_tag.txt" + checksum: "2a56ef7eb2e3732e221317af457d1c09" + description: "PerDT NER layer, dev split, IOB2 (CC BY-SA 4.0)" + - dest: "assets/ud-ner/test_with_NER_tag.txt" + url: "https://raw.githubusercontent.com/UniversalDependencies/UD_Persian-PerDT/master/not-to-release/Dadegan%20with%20NER%20tag/test_with_NER_tag.txt" + checksum: "6d80dd783527562c2ea5189f218a12b5" + description: "PerDT NER layer, test split, IOB2 (CC BY-SA 4.0)" workflows: - # The shipping artifact: fa_dep_news_sm. + # Both shipping artifacts: fa_dep_news_sm and fa_core_news_sm. all: - inspect - convert-ud + - transfer-ner + - convert-ner - debug-data - - train-core - - finalize - - evaluate + - debug-data-ner + - train-dep + - train-ner + - finalize-dep + - evaluate-dep + - assemble-core + - evaluate-core - finalize-meta - package - smoke - # Optional, separate artifact: fa_ent_news_sm. Not part of `all` — see the note above. - ner: - - convert-ner - - debug-data-ner - - train-ner - - finalize-ner - - evaluate-ner - - package-ner + # Optional third artifact: the NER alone, for users who only want entities. + ent: + - finalize-ent + - evaluate-ent + - package-ent commands: - name: "inspect" @@ -106,20 +119,35 @@ commands: - "corpus/merged/${vars.treebank}-ud-dev.spacy" - "corpus/merged/${vars.treebank}-ud-test.spacy" - - name: "convert-ner" - help: "Unpack ParsTwiNER and convert its IOB2 files to DocBin" + - name: "transfer-ner" + help: > + Align PerDT's NER layer onto the --merge-subtokens tokenization. The NER files use the + original Dadegan tokenization, which matches the released UD tokenization in only 57 to + 62% of sentences, so spans are transferred by difflib. Measured rate 99.86%; spans that + cannot be aligned exactly are dropped rather than guessed. script: - - "python scripts/extract_parstwiner.py assets/ner/ParsTwiNER_corpus_v1.0.0.zip assets/ner" - - "python -m spacy convert assets/ner/train.txt corpus/ner --converter ner --n-sents ${vars.n_sents} --lang ${vars.lang}" - - "python -m spacy convert assets/ner/dev.txt corpus/ner --converter ner --n-sents ${vars.n_sents} --lang ${vars.lang}" - - "python -m spacy convert assets/ner/test.txt corpus/ner --converter ner --n-sents ${vars.n_sents} --lang ${vars.lang}" + - "python scripts/transfer_perdt_ner.py --conllu-dir assets/ud --ner-dir assets/ud-ner --out corpus/perdt-ner-iob" deps: - - "assets/ner/ParsTwiNER_corpus_v1.0.0.zip" - - "scripts/extract_parstwiner.py" + - "assets/ud/${vars.treebank}-ud-train.conllu" + - "assets/ud-ner/train_with_NER_tag.txt" + - "scripts/transfer_perdt_ner.py" outputs: - - "corpus/ner/train.spacy" - - "corpus/ner/dev.spacy" - - "corpus/ner/test.spacy" + - "corpus/perdt-ner-iob/train.txt" + - "corpus/perdt-ner-iob/dev.txt" + - "corpus/perdt-ner-iob/test.txt" + + - name: "convert-ner" + help: "Transferred IOB2 -> DocBin" + script: + - "python -m spacy convert corpus/perdt-ner-iob/train.txt corpus/perdt-ner --converter ner --n-sents ${vars.n_sents} --lang ${vars.lang}" + - "python -m spacy convert corpus/perdt-ner-iob/dev.txt corpus/perdt-ner --converter ner --n-sents ${vars.n_sents} --lang ${vars.lang}" + - "python -m spacy convert corpus/perdt-ner-iob/test.txt corpus/perdt-ner --converter ner --n-sents ${vars.n_sents} --lang ${vars.lang}" + deps: + - "corpus/perdt-ner-iob/train.txt" + outputs: + - "corpus/perdt-ner/train.spacy" + - "corpus/perdt-ner/dev.spacy" + - "corpus/perdt-ner/test.spacy" - name: "debug-data" help: "Validate the treebank against the config before burning CPU on training" @@ -130,46 +158,46 @@ commands: - "configs/fa_dep_news_sm.cfg" - name: "debug-data-ner" - help: "Validate the ParsTwiNER corpus against the NER config" + help: "Validate the transferred PerDT NER corpus against the NER config" script: - - "python -m spacy debug data configs/fa_ner_sm.cfg --paths.train corpus/ner/train.spacy --paths.dev corpus/ner/dev.spacy" + - "python -m spacy debug data configs/fa_ner_sm.cfg --paths.train corpus/perdt-ner/train.spacy --paths.dev corpus/perdt-ner/dev.spacy" deps: - - "corpus/ner/train.spacy" + - "corpus/perdt-ner/train.spacy" - "configs/fa_ner_sm.cfg" - - name: "train-core" + - name: "train-dep" help: "Train tok2vec + tagger + morphologizer + trainable_lemmatizer + parser on PerDT" script: - - "python -m spacy train configs/fa_dep_news_sm.cfg --output training/core --paths.train corpus/merged/${vars.treebank}-ud-train.spacy --paths.dev corpus/merged/${vars.treebank}-ud-dev.spacy --gpu-id ${vars.gpu}" + - "python -m spacy train configs/fa_dep_news_sm.cfg --output training/dep --paths.train corpus/merged/${vars.treebank}-ud-train.spacy --paths.dev corpus/merged/${vars.treebank}-ud-dev.spacy --gpu-id ${vars.gpu}" deps: - "corpus/merged/${vars.treebank}-ud-train.spacy" - "corpus/merged/${vars.treebank}-ud-dev.spacy" - "configs/fa_dep_news_sm.cfg" outputs: - - "training/core/model-best" + - "training/dep/model-best" - name: "train-ner" - help: "Train the standalone NER component (own internal tok2vec) on ParsTwiNER" + help: "Train the NER component (own embedded tok2vec) on the transferred PerDT layer" script: - - "python -m spacy train configs/fa_ner_sm.cfg --output training/ner --paths.train corpus/ner/train.spacy --paths.dev corpus/ner/dev.spacy --gpu-id ${vars.gpu}" + - "python -m spacy train configs/fa_ner_sm.cfg --output training/perdt-ner --paths.train corpus/perdt-ner/train.spacy --paths.dev corpus/perdt-ner/dev.spacy --gpu-id ${vars.gpu}" deps: - - "corpus/ner/train.spacy" - - "corpus/ner/dev.spacy" + - "corpus/perdt-ner/train.spacy" + - "corpus/perdt-ner/dev.spacy" - "configs/fa_ner_sm.cfg" outputs: - - "training/ner/model-best" + - "training/perdt-ner/model-best" - - name: "finalize" + - name: "finalize-dep" help: "Write fa_dep_news_sm metadata (sources, licence, notes) onto the trained model" script: - - "python scripts/finalize_pipeline.py training/core/model-best training/fa_dep_news_sm --variant dep --version ${vars.package_version}" + - "python scripts/finalize_pipeline.py training/dep/model-best training/fa_dep_news_sm --variant dep --version ${vars.package_version}" deps: - - "training/core/model-best" + - "training/dep/model-best" - "scripts/finalize_pipeline.py" outputs: - "training/fa_dep_news_sm" - - name: "evaluate" + - name: "evaluate-dep" help: "Score fa_dep_news_sm on the held-out UD test split" script: - "python -m spacy benchmark accuracy training/fa_dep_news_sm corpus/merged/${vars.treebank}-ud-test.spacy --output metrics/ud-test.json --gpu-id ${vars.gpu}" @@ -179,64 +207,97 @@ commands: outputs: - "metrics/ud-test.json" + - name: "assemble-core" + help: > + Source the trained ner into the dep pipeline to produce fa_core_news_sm. Possible + because configs/fa_ner_sm.cfg embeds its own tok2vec instead of a Tok2VecListener. + script: + - "python scripts/finalize_pipeline.py training/dep/model-best training/fa_core_news_sm --variant core --version ${vars.package_version} --add-ner training/perdt-ner/model-best" + deps: + - "training/dep/model-best" + - "training/perdt-ner/model-best" + - "scripts/finalize_pipeline.py" + outputs: + - "training/fa_core_news_sm" + + - name: "evaluate-core" + help: "Score fa_core_news_sm on both held-out test splits" + script: + - "python -m spacy benchmark accuracy training/fa_core_news_sm corpus/merged/${vars.treebank}-ud-test.spacy --output metrics/core-ud-test.json --gpu-id ${vars.gpu}" + - "python -m spacy benchmark accuracy training/fa_core_news_sm corpus/perdt-ner/test.spacy --output metrics/perdt-ner-test.json --gpu-id ${vars.gpu}" + deps: + - "training/fa_core_news_sm" + - "corpus/merged/${vars.treebank}-ud-test.spacy" + - "corpus/perdt-ner/test.spacy" + outputs: + - "metrics/core-ud-test.json" + - "metrics/perdt-ner-test.json" + - name: "finalize-meta" help: > - Re-run finalize, this time folding the test scores into meta.json["performance"]. - Separate from `finalize` because the scores can only exist after `evaluate`, and - `evaluate` needs a finalized pipeline to score. Cheap: it only copies models. + Re-run finalize on both packages, folding test scores into meta.json["performance"]. + Separate because the scores only exist after evaluation, and evaluation needs a + finalized pipeline to score. Cheap: it only copies models. script: - - "python scripts/finalize_pipeline.py training/core/model-best training/fa_dep_news_sm --variant dep --version ${vars.package_version} --metrics metrics/ud-test.json" + - "python scripts/finalize_pipeline.py training/dep/model-best training/fa_dep_news_sm --variant dep --version ${vars.package_version} --ud-metrics metrics/ud-test.json" + - "python scripts/finalize_pipeline.py training/dep/model-best training/fa_core_news_sm --variant core --version ${vars.package_version} --add-ner training/perdt-ner/model-best --ud-metrics metrics/core-ud-test.json --ner-metrics metrics/perdt-ner-test.json" deps: - "metrics/ud-test.json" + - "metrics/perdt-ner-test.json" - "scripts/finalize_pipeline.py" - name: "package" - help: "Build the installable fa_dep_news_sm wheel + sdist" + help: "Build installable wheels + sdists for both shipping packages" script: - - "python -m spacy package training/fa_dep_news_sm packages --name ${vars.package_name} --version ${vars.package_version} --build sdist,wheel --force" + - "python -m spacy package training/fa_dep_news_sm packages --name ${vars.dep_package_name} --version ${vars.package_version} --build sdist,wheel --force" + - "python -m spacy package training/fa_core_news_sm packages --name ${vars.core_package_name} --version ${vars.package_version} --build sdist,wheel --force" deps: - "training/fa_dep_news_sm" + - "training/fa_core_news_sm" outputs: - - "packages/${vars.lang}_${vars.package_name}-${vars.package_version}" + - "packages/${vars.lang}_${vars.dep_package_name}-${vars.package_version}" + - "packages/${vars.lang}_${vars.core_package_name}-${vars.package_version}" - name: "smoke" - help: "Load the pipeline and run it over real Persian text" + help: "Load both pipelines and run them over real Persian text" script: - "python scripts/smoke_test.py training/fa_dep_news_sm" + - "python scripts/smoke_test.py training/fa_core_news_sm" deps: - "training/fa_dep_news_sm" + - "training/fa_core_news_sm" - - name: "finalize-ner" + - name: "finalize-ent" help: "Write fa_ent_news_sm metadata onto the trained NER model" script: - - "python scripts/finalize_pipeline.py training/ner/model-best training/fa_ent_news_sm --variant ent --version ${vars.package_version} --metrics metrics/ner-test.json" + - "python scripts/finalize_pipeline.py training/perdt-ner/model-best training/fa_ent_news_sm --variant ent --version ${vars.package_version}" deps: - - "training/ner/model-best" + - "training/perdt-ner/model-best" - "scripts/finalize_pipeline.py" outputs: - "training/fa_ent_news_sm" - - name: "evaluate-ner" - help: "Score fa_ent_news_sm on the held-out ParsTwiNER test split" + - name: "evaluate-ent" + help: "Score fa_ent_news_sm on the held-out PerDT NER test split" script: - - "python -m spacy benchmark accuracy training/fa_ent_news_sm corpus/ner/test.spacy --output metrics/ner-test.json --gpu-id ${vars.gpu}" - - "python scripts/finalize_pipeline.py training/ner/model-best training/fa_ent_news_sm --variant ent --version ${vars.package_version} --metrics metrics/ner-test.json" + - "python -m spacy benchmark accuracy training/fa_ent_news_sm corpus/perdt-ner/test.spacy --output metrics/ent-test.json --gpu-id ${vars.gpu}" + - "python scripts/finalize_pipeline.py training/perdt-ner/model-best training/fa_ent_news_sm --variant ent --version ${vars.package_version} --ner-metrics metrics/ent-test.json" deps: - "training/fa_ent_news_sm" - - "corpus/ner/test.spacy" + - "corpus/perdt-ner/test.spacy" outputs: - - "metrics/ner-test.json" + - "metrics/ent-test.json" - - name: "package-ner" + - name: "package-ent" help: "Build the installable fa_ent_news_sm wheel + sdist" script: - - "python -m spacy package training/fa_ent_news_sm packages --name ${vars.ner_package_name} --version ${vars.package_version} --build sdist,wheel --force" + - "python -m spacy package training/fa_ent_news_sm packages --name ${vars.ent_package_name} --version ${vars.package_version} --build sdist,wheel --force" deps: - "training/fa_ent_news_sm" outputs: - - "packages/${vars.lang}_${vars.ner_package_name}-${vars.package_version}" + - "packages/${vars.lang}_${vars.ent_package_name}-${vars.package_version}" - name: "clean" help: "Drop corpora, training runs and metrics (keeps downloaded assets)" script: - - "rm -rf corpus/merged corpus/split corpus/ner training metrics packages" + - "rm -rf corpus training metrics packages" diff --git a/scripts/extract_parstwiner.py b/scripts/extract_parstwiner.py deleted file mode 100644 index 9bf2994..0000000 --- a/scripts/extract_parstwiner.py +++ /dev/null @@ -1,34 +0,0 @@ -"""Unpack the ParsTwiNER release zip into flat IOB2 files. - -The archive carries macOS resource-fork junk (`__MACOSX/._*`) that would confuse -`spacy convert`, so only the three real splits are extracted. - -Usage: .venv/bin/python scripts/extract_parstwiner.py assets/ner/ParsTwiNER_corpus_v1.0.0.zip assets/ner -""" - -import sys -import zipfile -from pathlib import Path - -SPLITS = ("train.txt", "dev.txt", "test.txt") - - -def main(): - archive = Path(sys.argv[1]) - dest = Path(sys.argv[2]) - dest.mkdir(parents=True, exist_ok=True) - with zipfile.ZipFile(archive) as zf: - names = set(zf.namelist()) - missing = [s for s in SPLITS if s not in names] - if missing: - raise SystemExit(f"{archive}: missing expected members {missing}") - for split in SPLITS: - zf.extract(split, dest) - path = dest / split - sents = path.read_text(encoding="utf8").strip().split("\n\n") - tokens = sum(1 for line in path.open(encoding="utf8") if line.strip()) - print(f"{path}: {len(sents)} sentences, {tokens} tokens") - - -if __name__ == "__main__": - main() diff --git a/scripts/finalize_pipeline.py b/scripts/finalize_pipeline.py index cbd4eeb..8e2c4a4 100644 --- a/scripts/finalize_pipeline.py +++ b/scripts/finalize_pipeline.py @@ -1,27 +1,31 @@ """Write complete, convention-compliant metadata onto a trained pipeline. -Two variants, following spaCy's `[lang]_[type]_[genre]_[size]` naming +Three variants, following spaCy's `[lang]_[type]_[genre]_[size]` naming (https://spacy.io/models#conventions): - dep -> fa_dep_news_sm tagger + morphologizer + trainable_lemmatizer + parser - ent -> fa_ent_news_sm ner only + dep -> fa_dep_news_sm tagger + morphologizer + trainable_lemmatizer + parser + core -> fa_core_news_sm the above plus ner + ent -> fa_ent_news_sm ner only -`core` is deliberately NOT produced. `core` means "tagger + parser + lemmatizer + NER" in -one package, and shipping one would imply the NER is of the same standard as the rest of -the pipeline. It is not: the UD components score 85-98 on edited prose while the ner -component manages 67.22 F and is trained on tweets, because no redistributably-licensed -Persian NER corpus shares a genre with the treebank. Merging them into a single `core` -artifact would hide that behind one package name. +All three are built from UD_Persian-PerDT alone, including the NER, which comes from that +treebank's own `not-to-release/Dadegan with NER tag/` layer. That is what makes `core` +honest here: one corpus, one genre, one licence, one provenance chain. An earlier version of +this script refused to emit `core` because the only redistributable Persian NER corpus we +knew of was ParsTwiNER, a Twitter corpus scoring 67.22 F against 85-98 for the UD +components; merging those into one package would have hidden a genre and quality gap behind +a single name and version. -`fa_core_news_sm` is reserved for when ../ner_dataset delivers prose-genre NER data that -survives its own ablation (see ../ner_dataset/PLAN.md §5). +PerDT's NER labels are silver, produced by Beheshti-NER (Taher et al., 2020) with manual +corrections, so `notes` says so and the per-label scores are published as measured. Run twice: once after training (no metrics yet), then again after `spacy benchmark accuracy` so meta.json["performance"] carries held-out test scores. Usage: .venv/bin/python scripts/finalize_pipeline.py training/core/model-best training/fa_dep_news_sm \\ - --variant dep --version 3.8.0 [--metrics metrics/ud-test.json] + --variant dep --version 3.8.0 [--ud-metrics metrics/ud-test.json] + .venv/bin/python scripts/finalize_pipeline.py training/fa_core_news_sm training/fa_core_news_sm \\ + --variant core --version 3.8.0 --ud-metrics metrics/core-ud-test.json --ner-metrics metrics/perdt-ner-test.json """ import argparse @@ -30,17 +34,21 @@ from pathlib import Path import spacy +AUTHOR = "Kiyarash Fazeli" +EMAIL = "kiyarash@nlogn.ir" +PROJECT_URL = "https://github.com/Fazel94/spacy-persian" + PERDT = { "name": "UD_Persian-PerDT (PerUDT v1.0)", "url": "https://github.com/UniversalDependencies/UD_Persian-PerDT", "author": "Mohammad Sadegh Rasooli, Pegah Safari, Amirsaeid Moloodi, Alireza Nourian", "license": "CC BY-SA 4.0", } -PARSTWINER = { - "name": "ParsTwiNER", - "url": "https://github.com/overfit-ir/parstwiner", - "author": "MohammadMahdi Aghajani, AliAkbar Badri, Hamid Beigy et al. (Overfit-IR)", - "license": "MIT", +PERDT_NER = { + "name": "UD_Persian-PerDT NER layer (not-to-release/Dadegan with NER tag/)", + "url": "https://github.com/UniversalDependencies/UD_Persian-PerDT", + "author": "PerDT authors, tagged with Beheshti-NER (Taher, Hoseini, Shamsfard 2020)", + "license": "CC BY-SA 4.0", } LANG_DATA = { "name": "spaCy lang/fa language data (stop words originally from HAZM)", @@ -49,49 +57,79 @@ LANG_DATA = { "license": "MIT", } +NER_NOTE = ( + "The ner component is trained on the NER layer shipped in UD_Persian-PerDT's " + "not-to-release/ directory, so it shares the treebank's genre, tokenization and licence. " + "Those labels are SILVER: the treebank README states they were produced by the BERT-based " + "Beheshti-NER tagger (Taher et al., 2020) with manual corrections to extend recall. They " + "were transferred onto this pipeline's tokenization by difflib alignment at a 99.86% " + "transfer rate (scripts/transfer_perdt_ner.py); spans that could not be aligned exactly " + "were dropped rather than guessed. Labels PER, LOC, ORG and DAT have 1,300 or more " + "training examples each; MON (205), TIM (135) and PCT (121) are thin and their scores in " + "`performance.ents_per_type` should be read before relying on them." +) +MWT_NOTE = ( + "Multiword tokens (pronominal clitics, enclitic copulas) were merged with " + "`spacy convert --merge-subtokens`, so a small number of XPOS tags are composite " + "(e.g. N_IANM_PR_JOPER) and ~1.5% of lemmas contain a space." +) +CHUNK_NOTE = ( + "doc.noun_chunks under-fires on this pipeline: spacy/lang/fa/syntax_iterators.py matches " + "ClearNLP labels that do not exist in Universal Dependencies, see " + "docs/upstream/fa-noun-chunks.md." +) +# CC BY-SA 4.0 on the treebank propagates to anything derived from it. +PERDT_LICENSE = "CC BY-SA 4.0" +ATTRIBUTION = ( + "Trained on UD_Persian-PerDT, licensed CC BY-SA 4.0; this pipeline is therefore " + "distributed under CC BY-SA 4.0 with attribution to the treebank authors." +) + +UD_KEYS = ("token_acc", "token_p", "token_r", "token_f", "tag_acc", "pos_acc", "morph_acc", + "lemma_acc", "dep_uas", "dep_las", "sents_p", "sents_r", "sents_f", + "dep_las_per_type") +NER_KEYS = ("ents_p", "ents_r", "ents_f", "ents_per_type") + VARIANTS = { "dep": { "name": "dep_news_sm", "description": ( "Persian dependency pipeline optimized for CPU. Components: tok2vec, tagger, " - "morphologizer, trainable_lemmatizer, parser. No NER — see fa_ent_news_sm." + "morphologizer, trainable_lemmatizer, parser. No NER, see fa_core_news_sm." ), - # CC BY-SA 4.0 on the treebank propagates to anything derived from it. - "license": "CC BY-SA 4.0", + "license": PERDT_LICENSE, "sources": [PERDT, LANG_DATA], - "notes": ( - "Trained on UD_Persian-PerDT, licensed CC BY-SA 4.0; this pipeline is therefore " - "distributed under CC BY-SA 4.0 with attribution to the treebank authors. " - "Multiword tokens (pronominal clitics, enclitic copulas) were merged with " - "`spacy convert --merge-subtokens`, so a small number of XPOS tags are composite " - "(e.g. N_IANM_PR_JOPER) and ~1.5% of lemmas contain a space. " - "doc.noun_chunks under-fires on this pipeline: spacy/lang/fa/syntax_iterators.py " - "matches ClearNLP labels that do not exist in Universal Dependencies — see " - "docs/upstream/fa-noun-chunks.md." + "notes": " ".join([ATTRIBUTION, MWT_NOTE, CHUNK_NOTE]), + "keys": UD_KEYS, + "require": lambda pipes: "ner" not in pipes, + "require_msg": "a 'dep' pipeline must not contain an ner component", + }, + "core": { + "name": "core_news_sm", + "description": ( + "Persian pipeline optimized for CPU. Components: tok2vec, tagger, morphologizer, " + "trainable_lemmatizer, parser, ner. Entity labels: PER, LOC, ORG, DAT, MON, TIM, " + "PCT." ), - "keys": ("token_acc", "token_p", "token_r", "token_f", "tag_acc", "pos_acc", - "morph_acc", "lemma_acc", "dep_uas", "dep_las", "sents_p", "sents_r", - "sents_f", "dep_las_per_type"), + "license": PERDT_LICENSE, + "sources": [PERDT, PERDT_NER, LANG_DATA], + "notes": " ".join([ATTRIBUTION, NER_NOTE, MWT_NOTE, CHUNK_NOTE]), + "keys": UD_KEYS + NER_KEYS, + "require": lambda pipes: "ner" in pipes and "parser" in pipes, + "require_msg": "a 'core' pipeline must contain both parser and ner", }, "ent": { "name": "ent_news_sm", "description": ( "Persian named entity recognizer optimized for CPU, with its own internal " - "tok2vec. Labels: PER, ORG, LOC, NAT, POG, EVE." + "tok2vec. Labels: PER, LOC, ORG, DAT, MON, TIM, PCT." ), - "license": "MIT", - "sources": [PARSTWINER, LANG_DATA], - "notes": ( - "Trained on ParsTwiNER (MIT), a Persian Twitter corpus, because the standard " - "Persian NER corpora (ARMAN, PEYMA, NSURL) are research-use-only and cannot be " - "redistributed. Expect degraded accuracy on formal or edited prose: this scores " - "67.22 F on its own in-genre test set, and EVE (30.0) and POG (41.2) are weak " - "enough to be treated as unreliable. The component embeds its own tok2vec rather " - "than using a Tok2VecListener, so it can be sourced into another pipeline with " - "nlp.add_pipe('ner', source=...). A prose-genre replacement is being built in " - "../ner_dataset." - ), - "keys": ("token_acc", "ents_p", "ents_r", "ents_f", "ents_per_type"), + "license": PERDT_LICENSE, + "sources": [PERDT_NER, LANG_DATA], + "notes": " ".join([ATTRIBUTION, NER_NOTE]), + "keys": ("token_acc",) + NER_KEYS, + "require": lambda pipes: pipes == ["ner"], + "require_msg": "an 'ent' pipeline must be exactly ['ner']", }, } @@ -102,35 +140,64 @@ def main(): ap.add_argument("output", help="destination directory") ap.add_argument("--variant", choices=sorted(VARIANTS), required=True) ap.add_argument("--version", default="3.8.0") - ap.add_argument("--metrics", nargs="*", default=[], - help="benchmark accuracy JSON files to fold into performance") + ap.add_argument("--ud-metrics", default=None, + help="benchmark accuracy JSON scored on the UD test split; supplies the " + "tagger/morph/lemma/parser keys only") + ap.add_argument("--ner-metrics", default=None, + help="benchmark accuracy JSON scored on the NER test split; supplies the " + "ents_* keys only") + ap.add_argument("--add-ner", default=None, + help="source the ner component from this trained pipeline first " + "(used to assemble 'core' from the dep model plus the ner model)") args = ap.parse_args() spec = VARIANTS[args.variant] nlp = spacy.load(args.model) + if args.add_ner: + ner_nlp = spacy.load(args.add_ner) + if ner_nlp.pipe_names != ["ner"]: + raise SystemExit(f"--add-ner expects a ['ner'] pipeline, got {ner_nlp.pipe_names}") + if "ner" in nlp.pipe_names: + nlp.remove_pipe("ner") + # `ner` embeds its own tok2vec (configs/fa_ner_sm.cfg), so sourcing it here leaves no + # dangling Tok2VecListener. See docs/MODELS.md. + nlp.add_pipe("ner", source=ner_nlp) + print(f"sourced ner from {args.add_ner}: {sorted(nlp.get_pipe('ner').labels)}") print(f"pipeline: {nlp.pipe_names}") - if args.variant == "dep" and "ner" in nlp.pipe_names: - raise SystemExit("refusing to publish a 'dep' pipeline that contains an ner component") - if args.variant == "ent" and nlp.pipe_names != ["ner"]: - raise SystemExit(f"expected exactly ['ner'], got {nlp.pipe_names}") + if not spec["require"](list(nlp.pipe_names)): + raise SystemExit( + f"refusing to publish as '{args.variant}': {spec['require_msg']}" + f" (got {list(nlp.pipe_names)})" + ) # Build from a strict whitelist rather than inheriting nlp.meta["performance"], which # after training also carries raw *_loss values and scorer keys that do not apply to - # this pipeline (e.g. tag_micro_f: 0.0). Published metadata should be the held-out - # scores and nothing else. + # this pipeline (e.g. tag_micro_f: 0.0). + # + # Each metrics file supplies only the keys its corpus can actually evidence. Folding both + # files over the same key set silently corrupted core's metadata: the NER corpus has no + # gold tags, so its report carries tag_acc: 0.0, which overwrote the real 95.96, and its + # --n-sents grouping produced a misleading sents_f. trained = nlp.meta.get("performance", {}) performance = {k: trained[k] for k in spec["keys"] if trained.get(k) is not None} - for path in args.metrics: + + def fold(path, keys, label): + if not path: + return p = Path(path) if not p.exists(): - print(f" (no metrics at {p}, skipping — run `evaluate` first)") - continue + print(f" (no {label} metrics at {p}, skipping; run the evaluate step first)") + return scored = json.loads(p.read_text(encoding="utf8")) - for key in spec["keys"]: - if scored.get(key) is not None: - performance[key] = scored[key] + taken = [k for k in keys if k in spec["keys"] and scored.get(k) is not None] + for key in taken: + performance[key] = scored[key] if scored.get("speed") is not None: - performance["speed"] = scored["speed"] + performance.setdefault("speed", scored["speed"]) + print(f" folded {len(taken)} {label} keys from {p}") + + fold(args.ud_metrics, UD_KEYS + ("token_acc",), "UD") + fold(args.ner_metrics, NER_KEYS, "NER") nlp.meta.update( { @@ -138,9 +205,9 @@ def main(): "name": spec["name"], "version": args.version, "description": spec["description"], - "author": "", - "email": "", - "url": "", + "author": AUTHOR, + "email": EMAIL, + "url": PROJECT_URL, "license": spec["license"], "sources": spec["sources"], "notes": spec["notes"], diff --git a/scripts/transfer_perdt_ner.py b/scripts/transfer_perdt_ner.py new file mode 100644 index 0000000..fcc7b09 --- /dev/null +++ b/scripts/transfer_perdt_ner.py @@ -0,0 +1,169 @@ +"""Transfer PerDT's NER layer onto the tokenization `spacy convert --merge-subtokens` produces. + +The NER files use the original Dadegan tokenization, which matches the released UD tokenization +in only 57 to 62% of sentences: the NER files drop some copulas and auxiliaries, and at least +one honorific is corrupted (`ص` written as `،`). Entities sit on content words present in both, +so difflib aligns them. A span transfers only if every one of its tokens maps and the result +stays contiguous; anything else is dropped rather than guessed. + +Measured transfer rate: 99.86% (train), 99.74% (dev), 99.51% (test). See PLAN.md §1. + +Usage: + python scripts/transfer_perdt_ner.py --conllu-dir assets/ud --ner-dir assets/ud-ner \\ + --out corpus/perdt-ner-iob + +With --out, writes IOB2 files on the merged tokenization to DIR/{split}.txt. +Without it, only reports the transfer rate. +""" + +import argparse +from collections import Counter +from difflib import SequenceMatcher +from pathlib import Path + +SPLITS = ("train", "dev", "test") + + +def merged_tokens(conllu_path): + """Tokens as --merge-subtokens yields them: multiword-token surface forms, not subtokens.""" + sents, cur, skip_to = [], [], 0 + for line in conllu_path.open(encoding="utf8"): + line = line.rstrip("\n") + if line.startswith("#"): + continue + if not line: + if cur: + sents.append(cur) + cur, skip_to = [], 0 + continue + c = line.split("\t") + if "." in c[0]: + continue + if "-" in c[0]: + skip_to = int(c[0].split("-")[1]) + cur.append(c[1]) + continue + if skip_to and int(c[0]) <= skip_to: + continue + cur.append(c[1]) + if cur: + sents.append(cur) + return sents + + +def iob_sents(path): + sents, cur = [], [] + for line in path.open(encoding="utf8"): + line = line.rstrip("\n") + if not line.strip(): + if cur: + sents.append(cur) + cur = [] + continue + p = line.split("\t") + if len(p) < 2: + p = line.split() + if len(p) < 2: + continue + cur.append((p[0], p[-1])) + if cur: + sents.append(cur) + return sents + + +def spans_from_iob(tagged): + """[(start, end, label)] over the source token indices.""" + spans, i = [], 0 + while i < len(tagged): + tag = tagged[i][1] + if tag.startswith("B-"): + label, j = tag[2:], i + 1 + while j < len(tagged) and tagged[j][1] == f"I-{label}": + j += 1 + spans.append((i, j, label)) + i = j + else: + i += 1 + return spans + + +def index_map(src, dst): + """src index -> dst index for tokens difflib considers equal.""" + out = {} + matcher = SequenceMatcher(a=src, b=dst, autojunk=False) + for a, b, size in matcher.get_matching_blocks(): + for k in range(size): + out[a + k] = b + k + return out + + +def transfer_split(conllu, ner_file): + """Return (tokens, tags) per sentence on the merged tokenization, plus counters.""" + gold = merged_tokens(conllu) + ner = iob_sents(ner_file) + moved, lost = Counter(), Counter() + result = [] + for i in range(min(len(gold), len(ner))): + dst = gold[i] + src = [t for t, _ in ner[i]] + m = index_map(src, dst) + tags = ["O"] * len(dst) + for start, end, label in spans_from_iob(ner[i]): + idx = [m[k] for k in range(start, end) if k in m] + contiguous = idx and idx == list(range(idx[0], idx[0] + len(idx))) + if len(idx) == end - start and contiguous: + tags[idx[0]] = f"B-{label}" + for k in idx[1:]: + tags[k] = f"I-{label}" + moved[label] += 1 + else: + lost[label] += 1 + result.append((dst, tags)) + return result, moved, lost + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--conllu-dir", default="assets/ud", + help="directory holding fa_perdt-ud-{split}.conllu") + ap.add_argument("--ner-dir", default="assets/ud-ner", + help="directory holding {split}_with_NER_tag.txt") + ap.add_argument("--out", default=None, help="write IOB2 files here") + args = ap.parse_args() + + conllu_dir, nerdir = Path(args.conllu_dir), Path(args.ner_dir) + for d in (conllu_dir, nerdir): + if not d.is_dir(): + raise SystemExit(f"not found: {d}") + out = Path(args.out) if args.out else None + if out: + out.mkdir(parents=True, exist_ok=True) + + grand_moved, grand_lost = Counter(), Counter() + for split in SPLITS: + sents, moved, lost = transfer_split( + conllu_dir / f"fa_perdt-ud-{split}.conllu", + nerdir / f"{split}_with_NER_tag.txt", + ) + m, dropped = sum(moved.values()), sum(lost.values()) + grand_moved += moved + grand_lost += lost + print(f"{split:<6} entities {m + dropped:>6} transferred {m:>6}" + f" ({100 * m / max(m + dropped, 1):.2f}%) dropped {dropped}") + if out: + path = out / f"{split}.txt" + with path.open("w", encoding="utf8") as fh: + for toks, tags in sents: + for tok, tag in zip(toks, tags): + fh.write(f"{tok}\t{tag}\n") + fh.write("\n") + print(f" wrote {path}") + + print("\nper label:") + for label in sorted(grand_moved | grand_lost, key=lambda k: -grand_moved[k]): + m, dropped = grand_moved[label], grand_lost[label] + print(f" {label:<6} {m:>6}/{m + dropped:<6} ({100 * m / max(m + dropped, 1):5.1f}%)") + + +if __name__ == "__main__": + main()