diff --git a/.gitignore b/.gitignore index 8b30561..021bc8c 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,10 @@ packages/ # Separate env for `spacy huggingface-hub push`: it caps typer<0.8, which breaks the # spaCy CLI in the training venv. See .omp/AGENTS.md. .venv-publish/ +# Local envs for verifying and benchmarking the trf wheel: CPU-only torch, and a cu126 build +# for the 940MX. Kept out of .venv so a CUDA-lib downgrade cannot reach the training env. +.venv-trf/ +.venv-trf-gpu/ __pycache__/ *.pyc # Personal scratch list, not part of the project diff --git a/README.fa.md b/README.fa.md index a2417f1..5108ae3 100644 --- a/README.fa.md +++ b/README.fa.md @@ -26,18 +26,26 @@ print(doc.ents) # (محمدرضا شجریان, مشهد) ارزیابی با `spacy benchmark accuracy` روی بخش آزمون همان پیکره انجام شده است: -| سنجه | امتیاز | مرجع | -| --- | --- | --- | -| `TOKEN_ACC` / `TOKEN_F` | ۹۹٫۹۶ / ۹۹٫۱۱ | | -| `TAG_ACC` (XPOS) | ۹۵٫۹۶ | | -| `POS_ACC` (UPOS) | ۹۶٫۲۴ | | -| `MORPH_ACC` | ۹۶٫۲۹ | | -| `LEMMA_ACC` | ۹۷٫۹۱ | | -| `SENTS_F` | ۹۹٫۲۵ | | -| `DEP_UAS` | ۸۹٫۶۹ | hazm+ParsBERT: ۹۲٫۴۶ | -| `DEP_LAS` | ۸۵٫۱۵ | hazm+ParsBERT: ۸۹٫۳۴ | -| `ENTS_F` | ۷۱٫۸۷ | تنها در `fa_core_news_sm` | -| سرعت | حدود ۹٬۲۵۰ واژه بر ثانیه | | +| سنجه | `sm` | `md` | `lg` | `trf` | مرجع | +| --- | --- | --- | --- | --- | --- | +| `TOKEN_ACC` / `TOKEN_F` | ۹۹٫۹۶ / ۹۹٫۱۱ | ۹۹٫۹۶ / ۹۹٫۱۱ | ۹۹٫۹۶ / ۹۹٫۱۱ | ۹۹٫۹۶ / ۹۹٫۱۱ | | +| `TAG_ACC` (XPOS) | ۹۵٫۹۶ | ۹۶٫۲۵ | ۹۶٫۵۵ | **۹۷٫۶۲** | | +| `POS_ACC` (UPOS) | ۹۶٫۲۴ | ۹۶٫۶۴ | ۹۶٫۶۸ | **۹۷٫۶۳** | | +| `MORPH_ACC` | ۹۶٫۲۹ | ۹۶٫۶۴ | ۹۶٫۷۰ | **۹۷٫۸۲** | | +| `LEMMA_ACC` | ۹۷٫۹۱ | ۹۷٫۹۶ | **۹۸٫۰۸** | ۹۷٫۳۱ | | +| `SENTS_F` | ۹۹٫۲۵ | **۹۹٫۲۸** | ۹۹٫۱۸ | ۹۷٫۳۵ | | +| `DEP_UAS` | ۸۹٫۶۹ | ۹۰٫۵۲ | ۹۰٫۹۶ | **۹۳٫۸۷** | hazm+ParsBERT: ۹۲٫۴۶ | +| `DEP_LAS` | ۸۵٫۱۵ | ۸۶٫۳۴ | ۸۶٫۶۰ | **۹۰٫۷۹** | hazm+ParsBERT: ۸۹٫۳۴ | +| `ENTS_P` | ۷۷٫۶۷ | ۷۶٫۵۶ | ۸۱٫۵۱ | **۸۴٫۰۶** | | +| `ENTS_R` | ۶۶٫۸۷ | ۷۲٫۹۵ | ۷۱٫۰۹ | **۸۱٫۷۶** | | +| `ENTS_F` | ۷۱٫۸۷ | ۷۴٫۷۱ | ۷۵٫۹۴ | **۸۲٫۸۹** | | +| سرعت (940MX، دستهٔ ۳۲) | ۱۰٬۲۳۵ | ۹٬۰۵۸ | ۹٬۲۱۵ | بخش توان عملیاتی | | +| حجم بستهٔ نصب | ۱۳٫۵ مگابایت | ۶۸٫۵ مگابایت | ۲۳۵ مگابایت | ۶۰۸ مگابایت | | + +ردهٔ `trf` که ParsBERT را ریزتنظیم می‌کند در همه‌جا جلو است مگر در واژه‌یابی و مرزبندی جمله، که +`lg` با واژه‌یاب درخت‌ویرایش روی زیرواژه‌های floret همچنان بهتر عمل می‌کند. تنها ردهٔ‌ای است که از +مرجع `DEP_LAS` برابر ۸۹٫۳۴ عبور می‌کند. این رده به کارت گرافیک نیاز دارد و مدل پایهٔ آن پروانهٔ +مشخصی ندارد، پس قابل بازانتشار نیست؛ هر دو نکته در `docs/MODELS.md` بخش ۸ آمده است. برچسب‌های موجودیت «نقره‌ای» هستند: از لایه‌ای در خود پیکره می‌آیند که با برچسب‌زن Beheshti-NER تولید و سپس دستی اصلاح شده است. بنابراین `ENTS_F` تا اندازه‌ای هم‌خوانی با آن برچسب‌زن را @@ -46,6 +54,29 @@ print(doc.ents) # (محمدرضا شجریان, مشهد) آموزش روی یک پردازندهٔ چهارهسته‌ای i5-7200U و بدون کارت گرافیک انجام شده است: ۱ ساعت و ۲۷ دقیقه برای اجزای نحوی و ۱۷ دقیقه برای NER. این دو اجرا مستقل‌اند و می‌توانند هم‌زمان انجام شوند. +## توان عملیاتی + +میانهٔ چند اجرای پیاپی `nlp.pipe` روی ۱۴۶ سند بخش آزمون PerDT (۲۳٬۸۲۵ توکن). تنها زمان خودِ +`pipe` اندازه‌گیری شده و اجرای گرم‌کردن کنار گذاشته می‌شود. برای بازتولید: +`python scripts/benchmark_throughput.py --gpu-id `؛ دادهٔ خام در +`metrics/throughput-*.json` است. + +| رده | پردازنده i5-7200U | کارت 940MX | کارت Tesla T4 | +| --- | ---: | ---: | ---: | +| `sm` | ۵٬۴۸۴ | ۱۰٬۲۳۵ | | +| `md` | ۵٬۴۰۸ | ۹٬۰۵۸ | | +| `lg` | ۴٬۷۱۵ | ۹٬۲۱۵ | | +| `trf` | ۱۸۷ | | ۸٬۳۲۰ | + +ردهٔ `trf` جنس دیگری دارد: روی همان پردازندهٔ لپ‌تاپ ۱۸۷ واژه بر ثانیه است، یعنی حدود ۲۹ برابر +کندتر از `sm` با ۵٬۴۸۴. روی T4 به ۸٬۳۲۰ می‌رسد و روی پردازندهٔ همان ماشین ۳۳۶، یعنی شتاب ۲۵ +برابری. پس کارت گرافیک برای این رده یک نیاز است نه یک بهینه‌سازی. کارت 940MX اصلاً `trf` را +اجرا نمی‌کند، چون نسخه‌های امروزی PyTorch پشتیبانی از معماری sm_50 را کنار گذاشته‌اند. + +فاصلهٔ `sm` و `md` و `lg` روی پردازنده کمتر از ۱۵ درصد است، یعنی کمتر از آنچه تفاوت اندازهٔ +جدول بردارها نشان می‌دهد: گلوگاه tok2vec نیست، تجزیه‌گر و واژه‌یاب است. پراکندگی اجراها روی +لپ‌تاپ بسته به دمای دستگاه حدود ۱۰± درصد است، پس تفاوت‌های کمتر از آن نویز شمرده می‌شوند. + گام‌های تبدیل پیکره، آموزش، ارزیابی و بسته‌بندی در [`project.yml`](project.yml) تعریف شده‌اند. توضیح بیشتر دربارهٔ گزینش پیکره و پروانه‌ها در [`docs/MODELS.md`](docs/MODELS.md) و شرح انگلیسی پروژه در [`README.md`](README.md) آمده است. diff --git a/README.md b/README.md index 7c3fc36..89c3132 100644 --- a/README.md +++ b/README.md @@ -53,20 +53,27 @@ The `md` tier adds a 50k x 300d floret vector table trained on 400k Persian docu config differs from `sm` by exactly one line (`include_static_vectors`), so the columns below isolate what the vectors buy. Full breakdown in `docs/MODELS.md` §6. -| Metric | `sm` | `md` | Reference | -| --- | --- | --- | --- | -| `TOKEN_ACC` / `TOKEN_F` | 99.96 / 99.11 | 99.96 / 99.11 | | -| `TAG_ACC` (XPOS) | 95.96 | 96.25 | | -| `POS_ACC` (UPOS) | 96.24 | 96.64 | | -| `MORPH_ACC` | 96.29 | 96.64 | | -| `LEMMA_ACC` | 97.91 | 97.96 | | -| `SENTS_F` | 99.25 | 99.28 | | -| `DEP_UAS` | 89.69 | 90.52 | hazm+ParsBERT: 92.46 | -| `DEP_LAS` | 85.15 | 86.34 | hazm+ParsBERT: 89.34 | -| `ENTS_P` | 77.67 | 76.56 | | -| `ENTS_R` | 66.87 | 72.95 | | -| `ENTS_F` | 71.87 | 74.71 | | -| Speed | ~9,250 words/s | ~7,700 words/s | | +| Metric | `sm` | `md` | `lg` | `trf` | Reference | +| --- | --- | --- | --- | --- | --- | +| `TOKEN_ACC` / `TOKEN_F` | 99.96 / 99.11 | 99.96 / 99.11 | 99.96 / 99.11 | 99.96 / 99.11 | | +| `TAG_ACC` (XPOS) | 95.96 | 96.25 | 96.55 | **97.62** | | +| `POS_ACC` (UPOS) | 96.24 | 96.64 | 96.68 | **97.63** | | +| `MORPH_ACC` | 96.29 | 96.64 | 96.70 | **97.82** | | +| `LEMMA_ACC` | 97.91 | 97.96 | **98.08** | 97.31 | | +| `SENTS_F` | 99.25 | **99.28** | 99.18 | 97.35 | | +| `DEP_UAS` | 89.69 | 90.52 | 90.96 | **93.87** | hazm+ParsBERT: 92.46 | +| `DEP_LAS` | 85.15 | 86.34 | 86.60 | **90.79** | hazm+ParsBERT: 89.34 | +| `ENTS_P` | 77.67 | 76.56 | 81.51 | **84.06** | | +| `ENTS_R` | 66.87 | 72.95 | 71.09 | **81.76** | | +| `ENTS_F` | 71.87 | 74.71 | 75.94 | **82.89** | | +| Speed (940MX, batch 32) | 10,235 words/s | 9,058 words/s | 9,215 words/s | see §Throughput | | +| Wheel size | 13.5 MB | 68.5 MB | 235 MB | 608 MB | | + +`trf` fine-tunes ParsBERT and wins everywhere except lemmatization and sentence +segmentation, where `lg`'s edit-tree lemmatizer over floret subwords still leads. It is the +only tier to pass the hazm+ParsBERT `DEP_LAS` reference of 89.34. It needs a GPU and its +encoder has no stated licence, so it is not redistributable; `docs/MODELS.md` §8 has both +caveats. Entity scores are `fa_core_news_*` on the PerDT NER test split; per-label breakdown and caveats are in [Named entity recognition](#named-entity-recognition). @@ -76,6 +83,31 @@ For comparison, `en_core_web_sm` scores TAG 97, LAS 90, ENTS_F 84 on a larger, c Trained on a 4-core i5-7200U with no GPU: `sm` 1h27m syntax + 17 min NER, `md` 1h54m syntax + 25 min NER (the two `md` runs overlapped, so wall clock overstates each). +## Throughput + +Median of repeated `nlp.pipe` passes over the 146-document PerDT test split (23,825 tokens), +timing the pipe only, warmup discarded. Reproduce with +`python scripts/benchmark_throughput.py --gpu-id `; raw records are in +`metrics/throughput-*.json`. + +| Tier | CPU, i5-7200U | GPU, GeForce 940MX | GPU, Tesla T4 | +| --- | ---: | ---: | ---: | +| `sm` | 5,484 | 10,235 | | +| `md` | 5,408 | 9,058 | | +| `lg` | 4,715 | 9,215 | | +| `trf` | 187 | | 8,320 | + +The `trf` tier is a different kind of thing: 187 words/s on the same laptop CPU that runs +`sm` at 5,484, so about 29x slower. On a T4 it reaches 8,320, and on that VM's own Xeon it +manages 336, a 25x GPU speedup. Treat GPU as a requirement rather than an optimization. +The 940MX cannot run `trf` at all, since current PyTorch wheels have dropped its sm_50 +compute capability. + +`sm`, `md` and `lg` are within about 15% of each other on CPU, which is smaller than the +gap in vector-table size suggests: the tok2vec is not the bottleneck, the parser and +lemmatizer are. Run-to-run spread on the laptop is roughly +/-10% depending on thermal +state, so treat differences under that as noise. + ## Named entity recognition Seven labels: `LOC`, `PER`, `ORG`, `DAT`, `MON`, `TIM`, `PCT`. They come from PerDT's own diff --git a/configs/fa_core_news_trf.cfg b/configs/fa_core_news_trf.cfg new file mode 100644 index 0000000..9a25242 --- /dev/null +++ b/configs/fa_core_news_trf.cfg @@ -0,0 +1,291 @@ +# fa_core_news_trf: the whole pipeline on one fine-tuned ParsBERT encoder. +# +# Differences from the sm/md/lg tiers, all forced by the transformer: +# +# * One corpus, not two. sm/md/lg train `ner` separately (own embedded tok2vec) and source +# it into the dep model. Fine-tuning a 162M-parameter encoder twice would double GPU cost +# and ship two encoders in one wheel, and the second would collide on the `transformer` +# component name. So every component listens to a single shared transformer and trains +# against corpus/joint/, built by scripts/merge_joint_corpus.py (UD layer + the +# difflib-transferred NER layer on identical tokenization). +# * `use_upper = false` on both transition-based parsers: with a transformer upstream the +# extra maxout layer is redundant, and this matches the upstream *_trf configs. +# * Adam + warmup_linear and accumulate_gradient=3, not the flat 0.001 the CPU tiers use. +# Fine-tuning a pretrained encoder at 1e-3 diverges. +# * gpu_allocator = "pytorch" so thinc and torch share one CUDA memory pool. +# +# Encoder: HooshvareLab/bert-base-parsbert-uncased. NOTE the licence caveat in +# docs/MODELS.md §3.4 - ParsBERT's model card carries no licence statement, so this wheel is +# NOT redistributable on those grounds; HooshvareLab/roberta-fa-zwnj-base (Apache-2.0) is the +# publishable alternative and drops in by changing `name` below. + +[paths] +train = null +dev = null +vectors = null +init_tok2vec = null + +[system] +gpu_allocator = "pytorch" +seed = 0 + +[nlp] +lang = "fa" +pipeline = ["transformer","tagger","morphologizer","trainable_lemmatizer","parser","ner"] +batch_size = 128 +disabled = [] +before_creation = null +after_creation = null +after_pipeline_creation = null + +[nlp.tokenizer] +@tokenizers = "spacy.Tokenizer.v1" + +[nlp.vectors] +@vectors = "spacy.Vectors.v1" + +[components] + +[components.transformer] +factory = "transformer" +max_batch_items = 4096 + +[components.transformer.set_extra_annotations] +@annotation_setters = "spacy-transformers.null_annotation_setter.v1" + +[components.transformer.model] +@architectures = "spacy-transformers.TransformerModel.v3" +name = "HooshvareLab/bert-base-parsbert-uncased" +mixed_precision = false + +[components.transformer.model.get_spans] +@span_getters = "spacy-transformers.strided_spans.v1" +window = 128 +stride = 96 + +[components.transformer.model.tokenizer_config] +use_fast = true + +[components.transformer.model.transformer_config] + +[components.transformer.model.grad_scaler_config] + +[components.tagger] +factory = "tagger" +label_smoothing = 0.05 +overwrite = false +neg_prefix = "!" + +[components.tagger.model] +@architectures = "spacy.Tagger.v2" +nO = null +normalize = false + +[components.tagger.model.tok2vec] +@architectures = "spacy-transformers.TransformerListener.v1" +grad_factor = 1.0 +upstream = "*" + +[components.tagger.model.tok2vec.pooling] +@layers = "reduce_mean.v1" + +[components.tagger.scorer] +@scorers = "spacy.tagger_scorer.v1" + +[components.morphologizer] +factory = "morphologizer" +label_smoothing = 0.05 +overwrite = true +extend = false + +[components.morphologizer.model] +@architectures = "spacy.Tagger.v2" +nO = null +normalize = false + +[components.morphologizer.model.tok2vec] +@architectures = "spacy-transformers.TransformerListener.v1" +grad_factor = 1.0 +upstream = "*" + +[components.morphologizer.model.tok2vec.pooling] +@layers = "reduce_mean.v1" + +[components.morphologizer.scorer] +@scorers = "spacy.morphologizer_scorer.v1" + +[components.trainable_lemmatizer] +factory = "trainable_lemmatizer" +backoff = "orth" +min_tree_freq = 3 +overwrite = false +top_k = 1 + +[components.trainable_lemmatizer.model] +@architectures = "spacy.Tagger.v2" +nO = null +normalize = false + +[components.trainable_lemmatizer.model.tok2vec] +@architectures = "spacy-transformers.TransformerListener.v1" +grad_factor = 1.0 +upstream = "*" + +[components.trainable_lemmatizer.model.tok2vec.pooling] +@layers = "reduce_mean.v1" + +[components.trainable_lemmatizer.scorer] +@scorers = "spacy.lemmatizer_scorer.v1" + +[components.parser] +factory = "parser" +moves = null +update_with_oracle_cut_size = 100 +learn_tokens = false +min_action_freq = 30 + +[components.parser.model] +@architectures = "spacy.TransitionBasedParser.v2" +state_type = "parser" +extra_state_tokens = false +hidden_width = 64 +maxout_pieces = 2 +use_upper = false +nO = null + +[components.parser.model.tok2vec] +@architectures = "spacy-transformers.TransformerListener.v1" +grad_factor = 1.0 +upstream = "*" + +[components.parser.model.tok2vec.pooling] +@layers = "reduce_mean.v1" + +[components.parser.scorer] +@scorers = "spacy.parser_scorer.v1" + +[components.ner] +factory = "ner" +moves = null +update_with_oracle_cut_size = 100 +incorrect_spans_key = null + +[components.ner.model] +@architectures = "spacy.TransitionBasedParser.v2" +state_type = "ner" +extra_state_tokens = false +hidden_width = 64 +maxout_pieces = 2 +use_upper = false +nO = null + +[components.ner.model.tok2vec] +@architectures = "spacy-transformers.TransformerListener.v1" +grad_factor = 1.0 +upstream = "*" + +[components.ner.model.tok2vec.pooling] +@layers = "reduce_mean.v1" + +[components.ner.scorer] +@scorers = "spacy.ner_scorer.v1" + +[corpora] + +[corpora.train] +@readers = "spacy.Corpus.v1" +path = ${paths.train} +max_length = 0 +gold_preproc = false +limit = 0 +augmenter = null + +[corpora.dev] +@readers = "spacy.Corpus.v1" +path = ${paths.dev} +max_length = 0 +gold_preproc = false +limit = 0 +augmenter = null + +[training] +dev_corpus = "corpora.dev" +train_corpus = "corpora.train" +seed = ${system.seed} +gpu_allocator = ${system.gpu_allocator} +dropout = 0.1 +accumulate_gradient = 3 +# 3000 steps is ~40 epochs over this 445k-token corpus, measured at ~29 steps/min on a T4 +# (~1.8h). The CPU tiers' 20000/1600 would be ~270 epochs and ~12h here, and worse than +# wasteful: warmup_linear anneals against `total_steps`, so a run stopped early by patience +# never leaves the peak learning rate. Budget and schedule are kept equal on purpose: +# training.optimizer.learn_rate.total_steps must track any change to max_steps. +patience = 600 +max_epochs = 0 +max_steps = 3000 +eval_frequency = 100 +frozen_components = [] +annotating_components = [] +before_to_disk = null +before_update = null + +[training.optimizer] +@optimizers = "Adam.v1" +beta1 = 0.9 +beta2 = 0.999 +L2_is_weight_decay = true +L2 = 0.01 +grad_clip = 1.0 +use_averages = false +eps = 1e-08 + +[training.optimizer.learn_rate] +@schedules = "warmup_linear.v1" +warmup_steps = 250 +total_steps = 3000 +initial_rate = 5e-5 + +[training.batcher] +@batchers = "spacy.batch_by_padded.v1" +discard_oversize = true +size = 2000 +buffer = 256 +get_length = null + +[training.logger] +@loggers = "spacy.ConsoleLogger.v1" +progress_bar = false + +[training.score_weights] +tag_acc = 0.16 +pos_acc = 0.08 +tag_micro_p = null +tag_micro_r = null +tag_micro_f = null +morph_acc = 0.08 +morph_per_feat = null +lemma_acc = 0.16 +dep_uas = 0.08 +dep_las = 0.16 +dep_las_per_type = null +sents_p = null +sents_r = null +sents_f = 0.0 +ents_f = 0.28 +ents_p = 0.0 +ents_r = 0.0 +ents_per_type = null + +[initialize] +vectors = ${paths.vectors} +init_tok2vec = ${paths.init_tok2vec} +vocab_data = null +lookups = null +before_init = null +after_init = null + +[initialize.tokenizer] + +[initialize.components] + +[pretraining] diff --git a/docs/MODELS.md b/docs/MODELS.md index 0a57481..8a9a33b 100644 --- a/docs/MODELS.md +++ b/docs/MODELS.md @@ -424,10 +424,98 @@ at step 7,200, ~13 min, in line with `sm`/`md`. throughput dropped as expected (9,387 / 6,655 words/s vs `sm`'s 12,505 / 8,834, `md`'s 10,493 / 7,269 words/s; the larger table costs real lookup time), but the standalone `ent_lg` run showed 15,614 words/s, higher than `sm`/`md`'s ent runs despite an identical `ner` -architecture and the same larger table. Treat that one figure as single-run CPU contention -noise on shared hardware, not a real speedup, and re-benchmark before citing it. +architecture and the same larger table. That figure was single-run CPU contention noise on +shared hardware, not a real speedup. Those numbers are superseded by §9, which times +`nlp.pipe` alone instead of reading a scoring-contaminated figure off the benchmark command. For a 4x download over `md` (and up to 39x over `sm`) buying +1.45 DEP_LAS / +1.23 ENTS_F over `md` (+1.45 DEP_LAS / +4.08 ENTS_F over `sm`), `lg` is a server/offline-batch pipeline, not something to ship to a browser or a cold-start function. All three variants (`dep`, `ent`, `core`) are built and evaluated at this tier, same as `md`. + +## 8. The `trf` tier: one fine-tuned ParsBERT + +`configs/fa_core_news_trf.cfg` replaces the static-vector tok2vec with +`HooshvareLab/bert-base-parsbert-uncased`, fine-tuned during training. Trained on a rented +Colab T4 in 1h58m: 3000 steps, no early stop, the full learning-rate anneal. + +### One corpus, because a transformer cannot be trained twice + +The `sm`/`md`/`lg` tiers train `ner` as its own pipeline with its own embedded tok2vec and +then source it into the dep model. That is affordable because a hash-embed tok2vec is cheap. +A 162M-parameter encoder is not: fine-tuning it once per component would double GPU cost and +put two encoders in one wheel, and sourcing the second would collide on the `transformer` +component name. + +So every component listens to a single shared transformer through a `TransformerListener`, +which requires one corpus carrying both the UD and NER annotation layers on the same `Doc`. +`scripts/merge_joint_corpus.py` builds it. The fusion is exact rather than approximate: +`corpus/perdt-ner/` was converted from the same `--merge-subtokens` CoNLL-U as +`corpus/merged/` with the same `--n-sents`, so the two DocBins are token-for-token identical. +The script asserts that per document and copies only `doc.ents` across. Char offsets are not +usable for the copy, because the two converters differ in trailing whitespace, which shifts +`char_span` off the token grid and returns None; the transfer goes by token index. + +### Results against `lg` + +| Metric | `lg` | `trf` | Delta | +| --- | ---: | ---: | ---: | +| `TAG_ACC` | 96.55 | 97.62 | +1.07 | +| `POS_ACC` | 96.68 | 97.63 | +0.95 | +| `MORPH_ACC` | 96.70 | 97.82 | +1.12 | +| `LEMMA_ACC` | 98.08 | 97.31 | -0.77 | +| `DEP_UAS` | 90.96 | 93.87 | +2.91 | +| `DEP_LAS` | 86.60 | 90.79 | +4.19 | +| `SENTS_F` | 99.18 | 97.35 | -1.83 | +| `ENTS_F` | 75.94 | 82.89 | +6.95 | + +The parser gain is the headline: `DEP_LAS` 90.79 passes the hazm+ParsBERT reference of 89.34, +which no CPU tier reached. NER gains 6.95 F, almost all of it recall (71.09 to 81.76) at +higher precision, which is what a pretrained encoder buys on the difflib-transferred layer. + +Two metrics regress. `SENTS_F` drops 1.83, most likely because `strided_spans` at +`window = 128, stride = 96` leaves 32 tokens of overlap, so tokens near a span edge see +truncated right context where the CPU tiers' tok2vec sees the whole doc. `LEMMA_ACC` drops +0.77 and is the one metric where a static-vector tier wins: `trainable_lemmatizer` reads a +single `reduce_mean`-pooled vector per token, while `lg` runs an edit-tree lemmatizer over +floret subwords that model Persian orthography directly. Neither is a training-length +problem; see TODO.md for the evidence that more steps do not help. + +### Cost, and the licence problem + +608 MB wheel, 2.6x `lg` and 45x `sm`. 187 words/s on the laptop CPU against `sm`'s 5,484 +(§9), so this tier needs a GPU in production rather than merely benefiting from one. + +ParsBERT's model card states no licence. §3.4 picked `HooshvareLab/roberta-fa-zwnj-base` +(Apache-2.0) for exactly this reason, and the published wheel therefore embeds weights whose +redistribution terms are unknown. `scripts/finalize_pipeline.py` reads the encoder name out +of the trained config and writes a redistribution warning into `meta.json` when the encoder +has no licence, so the artifact carries the caveat. Retraining on the Apache-2.0 encoder is a +one-line change to `name` in the config. + +## 9. Throughput + +Measured with `scripts/benchmark_throughput.py`, which times `nlp.pipe` and nothing else. +The `words/s` printed by `spacy benchmark accuracy` runs the Scorer's per-token alignment +inside the timed region, which is why the §7 numbers disagree with these and why one of them +was impossible. + +Median of repeated passes over the 146-document PerDT test split (23,825 tokens), batch 32, +warmup discarded. Raw records in `metrics/throughput-*.json`. + +| Tier | CPU, i5-7200U | GPU, GeForce 940MX | CPU, Xeon @ 2.00GHz | GPU, Tesla T4 | +| --- | ---: | ---: | ---: | ---: | +| `sm` | 5,484 | 10,235 | | | +| `md` | 5,408 | 9,058 | | | +| `lg` | 4,715 | 9,215 | | | +| `trf` | 187 | | 336 | 8,320 | + +The CPU tiers sit within about 15% of each other, less than their vector-table sizes suggest, +so the tok2vec lookup is not the bottleneck; the parser and lemmatizer are. Run-to-run spread +on the laptop is roughly 10% either way with thermal state, and a background rsync halved +every number, so treat small differences as noise. + +`trf` is 29x slower than `sm` on the same CPU. The T4 column and the Xeon column come from +the same Colab VM, giving a clean 25x GPU speedup for the transformer. The 940MX column is +empty for `trf` because current PyTorch wheels dropped sm_50, so that GPU cannot run it at +all. diff --git a/project.yml b/project.yml index 09e4da1..fc4576d 100644 --- a/project.yml +++ b/project.yml @@ -41,6 +41,10 @@ vars: core_lg_package_name: "core_news_lg" floret_lg_wheel: "fa_floret-0.1.0-py3-none-any-full-wiki-200k-5epoch.whl" vectors_lg_dir: "assets/vectors/fa_floret_lg" + # trf tier: one fine-tuned ParsBERT shared by every component. Needs a real GPU; the + # 940MX cannot fine-tune a 162M-parameter encoder, so `gpu_trf` is set for a rented card. + core_trf_package_name: "core_news_trf" + gpu_trf: 0 directories: - "assets" @@ -125,6 +129,17 @@ workflows: - compare-md - package-md - smoke-md + # The trf tier: one fine-tuned ParsBERT shared by every component, including ner, so it + # trains against a single joint corpus instead of the sm/md/lg dep+ner split. GPU only. + trf: + - merge-joint + - debug-data-trf + - train-trf + - finalize-trf + - evaluate-trf + - finalize-meta-trf + - package-trf + - smoke-trf commands: - name: "inspect" @@ -604,6 +619,92 @@ commands: - "training/fa_dep_news_md" - "training/fa_core_news_md" + # ---------------------------------------------------------------- trf tier + + - name: "merge-joint" + help: > + Fuse the UD layer and the transferred NER layer onto one set of Docs. The trf tier + shares a single transformer across every component, so it needs one corpus carrying + both annotation layers; the two DocBins are token-for-token identical by construction + and the script asserts it. + script: + - "python scripts/merge_joint_corpus.py --ud-dir corpus/merged --ner-dir corpus/perdt-ner --out corpus/joint" + deps: + - "corpus/merged/${vars.treebank}-ud-train.spacy" + - "corpus/perdt-ner/train.spacy" + - "scripts/merge_joint_corpus.py" + outputs: + - "corpus/joint/train.spacy" + - "corpus/joint/dev.spacy" + - "corpus/joint/test.spacy" + + - name: "debug-data-trf" + help: "Validate the joint corpus against the trf config before renting GPU time" + script: + - "python -m spacy debug data configs/fa_core_news_trf.cfg --paths.train corpus/joint/train.spacy --paths.dev corpus/joint/dev.spacy" + deps: + - "corpus/joint/train.spacy" + - "configs/fa_core_news_trf.cfg" + + - name: "train-trf" + help: "Fine-tune ParsBERT with tagger + morphologizer + lemmatizer + parser + ner listening" + script: + - "python -m spacy train configs/fa_core_news_trf.cfg --output training/core-trf --paths.train corpus/joint/train.spacy --paths.dev corpus/joint/dev.spacy --gpu-id ${vars.gpu_trf}" + deps: + - "corpus/joint/train.spacy" + - "corpus/joint/dev.spacy" + - "configs/fa_core_news_trf.cfg" + outputs: + - "training/core-trf/model-best" + + - name: "finalize-trf" + help: "Write fa_core_news_trf metadata onto the trained model" + script: + - "python scripts/finalize_pipeline.py training/core-trf/model-best training/fa_core_news_trf --variant core --size trf --version ${vars.package_version}" + deps: + - "training/core-trf/model-best" + - "scripts/finalize_pipeline.py" + outputs: + - "training/fa_core_news_trf" + + - name: "evaluate-trf" + help: "Score fa_core_news_trf on the held-out UD and NER test splits" + script: + - "python -m spacy benchmark accuracy training/fa_core_news_trf corpus/merged/${vars.treebank}-ud-test.spacy --output metrics/trf-core-ud-test.json --gpu-id ${vars.gpu_trf}" + - "python -m spacy benchmark accuracy training/fa_core_news_trf corpus/perdt-ner/test.spacy --output metrics/trf-perdt-ner-test.json --gpu-id ${vars.gpu_trf}" + deps: + - "training/fa_core_news_trf" + - "corpus/merged/${vars.treebank}-ud-test.spacy" + - "corpus/perdt-ner/test.spacy" + outputs: + - "metrics/trf-core-ud-test.json" + - "metrics/trf-perdt-ner-test.json" + + - name: "finalize-meta-trf" + help: "Fold the trf test scores into meta.json" + script: + - "python scripts/finalize_pipeline.py training/core-trf/model-best training/fa_core_news_trf --variant core --size trf --version ${vars.package_version} --ud-metrics metrics/trf-core-ud-test.json --ner-metrics metrics/trf-perdt-ner-test.json" + deps: + - "metrics/trf-core-ud-test.json" + - "metrics/trf-perdt-ner-test.json" + - "scripts/finalize_pipeline.py" + + - name: "package-trf" + help: "Build the installable fa_core_news_trf wheel + sdist" + script: + - "python -m spacy package training/fa_core_news_trf packages --name ${vars.core_trf_package_name} --version ${vars.package_version} --build sdist,wheel --force" + deps: + - "training/fa_core_news_trf" + outputs: + - "packages/${vars.lang}_${vars.core_trf_package_name}-${vars.package_version}" + + - name: "smoke-trf" + help: "Load fa_core_news_trf and run it over real Persian text" + script: + - "python scripts/smoke_test.py training/fa_core_news_trf" + deps: + - "training/fa_core_news_trf" + - name: "clean" help: "Drop corpora, training runs and metrics (keeps downloaded assets)" diff --git a/scripts/benchmark_throughput.py b/scripts/benchmark_throughput.py new file mode 100644 index 0000000..ff2354d --- /dev/null +++ b/scripts/benchmark_throughput.py @@ -0,0 +1,112 @@ +"""Measure inference throughput (words/second) for a pipeline, on CPU or GPU. + +`spacy benchmark accuracy` prints a speed number, but it is scoring-contaminated: the +Scorer's per-token alignment and per-type bookkeeping run inside the timed region, which +matters a lot for the cheap CPU tiers and understates them. This times `nlp.pipe` only. + +Reported figure is the median of `--runs` passes over the same texts, after a discarded +warmup pass. Median rather than mean because the first CUDA kernel launches, cuBLAS +autotuning and any page-cache miss produce outliers that a mean would smear into the result. + +Batch size matters far more for the trf tier than the CPU tiers (a transformer amortizes a +GEMM over the batch; a hash-embed tok2vec barely cares), so it is a parameter and gets +recorded in the output rather than being left implicit. +""" + +import argparse +import json +import platform +import statistics +import subprocess +import time +from pathlib import Path + +import spacy +from spacy.tokens import DocBin + + +def cpu_model(): + try: + for line in Path("/proc/cpuinfo").read_text().splitlines(): + if line.startswith("model name"): + return line.split(":", 1)[1].strip() + except OSError: + pass + return platform.processor() or "unknown" + + +def gpu_model(): + try: + out = subprocess.run( + ["nvidia-smi", "--query-gpu=name,memory.total", "--format=csv,noheader"], + capture_output=True, text=True, timeout=30, + ) + if out.returncode == 0: + return out.stdout.strip().splitlines()[0].strip() + except (OSError, subprocess.SubprocessError): + pass + return "unknown" + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("model", help="installed package name or path to a pipeline") + ap.add_argument("--corpus", default="corpus/merged/fa_perdt-ud-test.spacy", + help="DocBin whose raw texts are used as input") + ap.add_argument("--gpu-id", type=int, default=-1, help="-1 for CPU") + ap.add_argument("--batch-size", type=int, default=32) + ap.add_argument("--runs", type=int, default=3) + ap.add_argument("--limit", type=int, default=0, help="cap number of docs (0 = all)") + ap.add_argument("--output", default=None, help="write a JSON record here") + args = ap.parse_args() + + if args.gpu_id >= 0: + # require_gpu, not prefer_gpu: a silent fall back to CPU would be reported as a GPU + # number, which is exactly the measurement error this script exists to avoid. + spacy.require_gpu(args.gpu_id) + device = f"gpu:{args.gpu_id} ({gpu_model()})" + else: + device = f"cpu ({cpu_model()})" + + nlp = spacy.load(args.model) + vocab_docs = list(DocBin().from_disk(args.corpus).get_docs(spacy.blank("fa").vocab)) + if args.limit: + vocab_docs = vocab_docs[:args.limit] + texts = [d.text for d in vocab_docs] + n_words = sum(len(d) for d in vocab_docs) + + # Warmup: first pass pays for lazy CUDA context creation, cuBLAS handles and any + # transformer weight transfer. Timing it would misattribute setup cost to throughput. + for _ in nlp.pipe(texts[:args.batch_size], batch_size=args.batch_size): + pass + + wps = [] + for _ in range(args.runs): + t0 = time.perf_counter() + for _ in nlp.pipe(texts, batch_size=args.batch_size): + pass + elapsed = time.perf_counter() - t0 + wps.append(n_words / elapsed) + + median = statistics.median(wps) + record = { + "model": args.model, + "pipeline": list(nlp.pipe_names), + "device": device, + "batch_size": args.batch_size, + "docs": len(texts), + "words": n_words, + "runs": [round(w, 1) for w in wps], + "wps_median": round(median, 1), + "spacy_version": spacy.__version__, + } + print(json.dumps(record, indent=2, ensure_ascii=False)) + if args.output: + p = Path(args.output) + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text(json.dumps(record, indent=2, ensure_ascii=False) + "\n") + print(f"wrote {p}") + + +if __name__ == "__main__": + main() diff --git a/scripts/finalize_pipeline.py b/scripts/finalize_pipeline.py index d21a7a6..24fa3b9 100644 --- a/scripts/finalize_pipeline.py +++ b/scripts/finalize_pipeline.py @@ -73,11 +73,15 @@ FLORET_LG = { "author": "Kiyarash Fazeli", "license": "CC BY-SA 4.0", } -TRANSFORMER = { - "name": "HooshvareLab/roberta-fa-zwnj-base", - "url": "https://huggingface.co/HooshvareLab/roberta-fa-zwnj-base", - "author": "Hooshvare Team", - "license": "Apache-2.0", +# Whatever encoder the config actually names wins; hardcoding one would silently mislabel a +# wheel the moment configs/fa_core_news_trf.cfg's `name` changes. Licences are recorded per +# encoder because they differ sharply, and two of the Persian ones have none at all. +ENCODER_LICENSES = { + "HooshvareLab/roberta-fa-zwnj-base": ("Hooshvare Team", "Apache-2.0"), + "HooshvareLab/bert-fa-zwnj-base": ("Hooshvare Team", "Apache-2.0"), + "m3hrdadfi/albert-fa-base-v2": ("Mehrdad Farahani", "Apache-2.0"), + "HooshvareLab/bert-base-parsbert-uncased": ("Hooshvare Team", "no licence stated on the model card"), + "sbunlp/fabert": ("SBU NLP Lab", "no licence stated on the model card"), } NER_NOTE = ( @@ -126,12 +130,48 @@ def vectors_note_lg(nlp): ) -TRANSFORMER_NOTE = ( - "This is the `trf` tier: no static vectors; contextual embeddings instead come from a " - "fine-tuned HooshvareLab/roberta-fa-zwnj-base (Apache-2.0) transformer via " - "spacy-transformers. Not ParsBERT: its model card carries no licence. GPU is recommended " - "for both training and inference." -) +def encoder_name(nlp): + """Read the encoder out of the trained pipeline's own config.""" + try: + return nlp.config["components"]["transformer"]["model"]["name"] + except KeyError: + raise SystemExit( + "--size trf expects a pipeline with a `transformer` component whose model names " + f"an encoder; got pipeline {list(nlp.pipe_names)}" + ) + + +def transformer_source(nlp): + name = encoder_name(nlp) + author, license_ = ENCODER_LICENSES.get(name, ("unknown", "unknown, check the model card")) + return { + "name": name, + "url": f"https://huggingface.co/{name}", + "author": author, + "license": license_, + } + + +def transformer_note(nlp): + name = encoder_name(nlp) + _, license_ = ENCODER_LICENSES.get(name, ("unknown", "unknown, check the model card")) + note = ( + f"This is the `trf` tier: no static vectors. Contextual embeddings come from a " + f"fine-tuned {name} ({license_}) via spacy-transformers, shared by every component " + f"through a TransformerListener, so one encoder forward pass serves the tagger, " + f"morphologizer, lemmatizer, parser and ner. Unlike the sm/md/lg tiers the ner is " + f"trained jointly rather than sourced, because a shared encoder cannot be fine-tuned " + f"twice and then merged. GPU is strongly recommended for both training and inference." + ) + if "no licence" in license_ or license_.startswith("unknown"): + note += ( + f" REDISTRIBUTION WARNING: {name} states no licence, so this wheel embeds weights " + f"whose terms are unknown and must not be republished. Retrain against " + f"HooshvareLab/roberta-fa-zwnj-base (Apache-2.0) for a publishable artifact." + ) + return note + + # CC BY-SA 4.0 on the treebank propagates to anything derived from it. PERDT_LICENSE = "CC BY-SA 4.0" ATTRIBUTION = ( @@ -222,8 +262,15 @@ def main(): sources.append(FLORET_LG) notes = " ".join([notes, vectors_note_lg(nlp)]) elif args.size == "trf": - sources.append(TRANSFORMER) - notes = " ".join([notes, TRANSFORMER_NOTE]) + sources.append(transformer_source(nlp)) + notes = " ".join([notes, transformer_note(nlp)]) + # The stock description advertises a CPU tok2vec pipeline, which is wrong here. + description = ( + "Persian pipeline built on a fine-tuned " + f"{encoder_name(nlp)} transformer. Components: transformer, tagger, " + "morphologizer, trainable_lemmatizer, parser, ner. Entity labels: PER, LOC, ORG, " + "DAT, MON, TIM, PCT. GPU recommended." + ) if args.add_ner: ner_nlp = spacy.load(args.add_ner) if ner_nlp.pipe_names != ["ner"]: diff --git a/scripts/make_model_card.py b/scripts/make_model_card.py new file mode 100644 index 0000000..aa5e6d6 --- /dev/null +++ b/scripts/make_model_card.py @@ -0,0 +1,145 @@ +"""Build the Hugging Face model card for a packaged pipeline. + +`spacy package` already writes a README into the wheel, and `spacy huggingface-hub push` +uploads it as the card. That card is a metadata dump: no install line, no usage, no +throughput, and no YAML frontmatter, so the Hub cannot index the model by language or task. + +This composes a card from the same sources of truth (`meta.json` and the JSON written by +scripts/benchmark_throughput.py) rather than from hand-copied numbers, so the card cannot +drift from the artifact it describes. +""" + +import argparse +import json +from pathlib import Path + +# meta.json key -> (row label, reference note). Only keys the pipeline actually evidences +# are emitted; a missing key means the corpus could not score it. +METRICS = [ + ("token_acc", "Tokenization accuracy", ""), + ("tag_acc", "XPOS tag accuracy", ""), + ("pos_acc", "UPOS tag accuracy", ""), + ("morph_acc", "Morphological features", ""), + ("lemma_acc", "Lemma accuracy", ""), + ("dep_uas", "Unlabelled attachment (UAS)", ""), + ("dep_las", "Labelled attachment (LAS)", ""), + ("sents_f", "Sentence segmentation F", ""), + ("ents_p", "NER precision", ""), + ("ents_r", "NER recall", ""), + ("ents_f", "NER F-score", ""), +] + + +def load(path): + return json.loads(Path(path).read_text()) + + +def throughput_rows(paths): + rows = [] + for p in paths: + if not Path(p).exists(): + continue + d = load(p) + rows.append((d["device"], d["batch_size"], d["wps_median"])) + return rows + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--meta", required=True, help="meta.json of the finalized pipeline") + ap.add_argument("--throughput", nargs="*", default=[], help="benchmark_throughput JSONs") + ap.add_argument("--repo-id", required=True, help="e.g. Phazel/fa_core_news_trf") + ap.add_argument("--wheel-name", required=True) + ap.add_argument("--out", required=True) + args = ap.parse_args() + + meta = load(args.meta) + name = f"{meta['lang']}_{meta['name']}" + perf = meta.get("performance", {}) + + lines = [] + # Frontmatter: without this the Hub cannot filter the model by language or library. + lines += [ + "---", + "language:", + "- fa", + f"license: {meta.get('license', 'cc-by-sa-4.0').lower().replace(' ', '-')}", + "library_name: spacy", + "pipeline_tag: token-classification", + "tags:", + "- spacy", + "- token-classification", + "- persian", + "- farsi", + "---", + "", + f"# {name}", + "", + meta.get("description", "").strip(), + "", + ] + + lines += [ + "## Install", + "", + "```bash", + f"pip install https://huggingface.co/{args.repo_id}/resolve/main/{args.wheel_name}", + "```", + "", + "```python", + "import spacy", + f'nlp = spacy.load("{name}")', + 'doc = nlp("شرکت ایران خودرو اعلام کرد که تولید خود را افزایش می\u200cدهد.")', + "print([(t.text, t.pos_, t.lemma_, t.dep_) for t in doc])", + "print([(e.text, e.label_) for e in doc.ents])", + "```", + "", + ] + + lines += ["## Accuracy", "", + "Scored with `spacy benchmark accuracy` on the held-out PerDT test split.", + "", "| Metric | Score |", "| --- | ---: |"] + for key, label, _ in METRICS: + v = perf.get(key) + if isinstance(v, (int, float)): + lines.append(f"| {label} | {v * 100:.2f} |") + lines.append("") + + rows = throughput_rows(args.throughput) + if rows: + lines += ["## Throughput", "", + "Median of repeated `nlp.pipe` passes over the 146-document PerDT test", + "split (23,825 tokens), timing the pipe only. Warmup pass discarded.", + "", "| Device | Batch | Words/s |", "| --- | ---: | ---: |"] + for device, batch, wps in rows: + lines.append(f"| {device} | {batch} | {wps:,.0f} |") + lines.append("") + gpu = next((r for r in rows if r[0].startswith("gpu")), None) + cpu = next((r for r in rows if r[0].startswith("cpu")), None) + if gpu and cpu: + lines += [ + f"A transformer pipeline is GPU-bound: the T4 is {gpu[2] / cpu[2]:.0f}x the " + f"CPU on the same machine. On CPU this runs roughly 25x slower than the " + f"`sm`/`md`/`lg` tiers, which is the price of the accuracy below.", + "", + ] + + lines += ["## Sources", "", "| Source | Author | Licence |", "| --- | --- | --- |"] + for s in meta.get("sources", []): + url, nm = s.get("url"), s.get("name", "") + label = f"[{nm}]({url})" if url else nm + lines.append(f"| {label} | {s.get('author', '')} | {s.get('license', '')} |") + lines.append("") + + notes = (meta.get("notes") or "").strip() + if notes: + lines += ["## Notes", "", notes, ""] + + out = Path(args.out) + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text("\n".join(lines), encoding="utf-8") + print(f"wrote {out} ({out.stat().st_size} bytes)") + + +if __name__ == "__main__": + main() diff --git a/scripts/merge_joint_corpus.py b/scripts/merge_joint_corpus.py new file mode 100644 index 0000000..cf777e2 --- /dev/null +++ b/scripts/merge_joint_corpus.py @@ -0,0 +1,68 @@ +"""Fuse the UD annotation layer and the transferred NER layer into one DocBin. + +The sm/md/lg tiers train `ner` as a separate pipeline with its own embedded tok2vec, then +source it into the dep model (project.yml `assemble-core`). That works because a hash-embed +tok2vec is cheap enough to train twice. + +A transformer is not. Fine-tuning ParsBERT once per component would double GPU cost and +produce a package carrying two independent 162M-parameter encoders, and sourcing the second +one would collide on the `transformer` component name. So the trf tier trains every component +against a single shared transformer via TransformerListener, which requires a single corpus +carrying both annotation layers on the same Doc. + +That fusion is exact, not approximate: `corpus/perdt-ner/` was produced by +scripts/transfer_perdt_ner.py from the same `--merge-subtokens` CoNLL-U as `corpus/merged/`, +then converted with the same `--n-sents`, so the two DocBins are token-for-token identical +(verified below and asserted at runtime). Only `doc.ents` is copied across; every other +annotation stays on the UD doc. +""" + +import argparse +from pathlib import Path + +import spacy +from spacy.tokens import DocBin, Span + +SPLITS = (("train", "fa_perdt-ud-train"), ("dev", "fa_perdt-ud-dev"), ("test", "fa_perdt-ud-test")) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--ud-dir", default="corpus/merged") + ap.add_argument("--ner-dir", default="corpus/perdt-ner") + ap.add_argument("--out", default="corpus/joint") + ap.add_argument("--lang", default="fa") + args = ap.parse_args() + + nlp = spacy.blank(args.lang) + out = Path(args.out) + out.mkdir(parents=True, exist_ok=True) + + for split, ud_stem in SPLITS: + ud_docs = list(DocBin().from_disk(Path(args.ud_dir) / f"{ud_stem}.spacy").get_docs(nlp.vocab)) + ner_docs = list(DocBin().from_disk(Path(args.ner_dir) / f"{split}.spacy").get_docs(nlp.vocab)) + if len(ud_docs) != len(ner_docs): + raise SystemExit( + f"{split}: {len(ud_docs)} UD docs vs {len(ner_docs)} NER docs; the two corpora " + "were not converted from the same source with the same --n-sents" + ) + + db = DocBin(store_user_data=True) + n_ents = 0 + for i, (ud, ner) in enumerate(zip(ud_docs, ner_docs)): + if [t.text for t in ud] != [t.text for t in ner]: + raise SystemExit(f"{split} doc {i}: tokenization differs between UD and NER layers") + # Tokens are index-aligned, so rebuild by token index. Char offsets are NOT + # safe here: the two converters can differ in trailing whitespace, which shifts + # `char_span` off the token grid and silently yields None. + ud.ents = [Span(ud, e.start, e.end, label=e.label_) for e in ner.ents] + n_ents += len(ud.ents) + db.add(ud) + + dest = out / f"{split}.spacy" + db.to_disk(dest) + print(f"{dest}: {len(ud_docs)} docs, {n_ents} entities") + + +if __name__ == "__main__": + main()