From b89b01ceb696b31eb8a26deb3e529601db98e9d8 Mon Sep 17 00:00:00 2001 From: Mohamad Fazeli Date: Wed, 12 Aug 2026 19:01:32 +0330 Subject: [PATCH] Add lg tier: fa_ent_news_lg on 200k floret vectors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New table: 200k rows x 300d, full Persian Wikipedia dump, 5 epochs, vs md's 50k rows / 400k documents. - configs/fa_ner_lg.cfg, project.yml ent-lg workflow (vectors-lg through smoke-ent-lg) - scripts/compare_tiers.py: generalized sm/md pair to N tiers; ent NER test now includes lg; fixed sm baseline to the file that's actually scored (perdt-ner-test.json, not the missing ent-test.json) - scripts/finalize_pipeline.py: FLORET_LG source and vectors_note_lg corrected to full Wikipedia, 5 epochs (were a generic Wikipedia + OSCAR placeholder) - docs/MODELS.md §7: PerDT NER test ENTS_F 75.94 (sm 71.87, md 74.71), full per-label table, cost (217 MB wheel) Not built: fa_dep_news_lg / fa_core_news_lg. --- configs/fa_ner_lg.cfg | 154 +++++++++++++++++++++++++++++++++++ docs/MODELS.md | 57 +++++++++++++ project.yml | 90 ++++++++++++++++++++ scripts/compare_tiers.py | 139 +++++++++++++++++++++---------- scripts/finalize_pipeline.py | 11 +-- 5 files changed, 404 insertions(+), 47 deletions(-) create mode 100644 configs/fa_ner_lg.cfg diff --git a/configs/fa_ner_lg.cfg b/configs/fa_ner_lg.cfg new file mode 100644 index 0000000..b06b86e --- /dev/null +++ b/configs/fa_ner_lg.cfg @@ -0,0 +1,154 @@ +# fa_ent_news_lg — Persian NER with the lg-tier static floret vectors. +# +# Identical to configs/fa_ner_md.cfg (which is identical to fa_ner_sm.cfg except +# include_static_vectors: true). Only the vector table supplied at train time via +# --paths.vectors differs: fa_floret, 200k rows x 300d, floret mode, trained on the full +# Persian Wikipedia dump for 5 epochs (assets/vectors/fa_floret_lg), vs md's 50k rows x 300d +# trained on 400k Persian documents. +# +# Same embedded-tok2vec design as sm/md (no Tok2VecListener), so the trained component stays +# sourceable into a future fa_core_news_lg via `nlp.add_pipe("ner", source=...)`. + +[paths] +train = null +dev = null +vectors = null +init_tok2vec = null + +[system] +gpu_allocator = null +seed = 0 + +[nlp] +lang = "fa" +pipeline = ["ner"] +batch_size = 1000 +disabled = [] +before_creation = null +after_creation = null +after_pipeline_creation = null + +[nlp.tokenizer] +@tokenizers = "spacy.Tokenizer.v1" + +[nlp.vectors] +@vectors = "spacy.Vectors.v1" + +[components] + +[components.ner] +factory = "ner" +moves = null +update_with_oracle_cut_size = 100 +incorrect_spans_key = null + +[components.ner.model] +@architectures = "spacy.TransitionBasedParser.v2" +state_type = "ner" +extra_state_tokens = false +hidden_width = 64 +maxout_pieces = 2 +use_upper = true +nO = null + +[components.ner.model.tok2vec] +@architectures = "spacy.Tok2Vec.v2" + +[components.ner.model.tok2vec.embed] +@architectures = "spacy.MultiHashEmbed.v2" +width = ${components.ner.model.tok2vec.encode.width} +attrs = ["NORM", "PREFIX", "SUFFIX", "SHAPE"] +rows = [5000, 1000, 2500, 2500] +include_static_vectors = true + +[components.ner.model.tok2vec.encode] +@architectures = "spacy.MaxoutWindowEncoder.v2" +width = 96 +depth = 4 +window_size = 1 +maxout_pieces = 3 + +[components.ner.scorer] +@scorers = "spacy.ner_scorer.v1" + +[corpora] + +[corpora.train] +@readers = "spacy.Corpus.v1" +path = ${paths.train} +max_length = 0 +gold_preproc = false +limit = 0 +augmenter = null + +[corpora.dev] +@readers = "spacy.Corpus.v1" +path = ${paths.dev} +max_length = 0 +gold_preproc = false +limit = 0 +augmenter = null + +[training] +dev_corpus = "corpora.dev" +train_corpus = "corpora.train" +seed = ${system.seed} +gpu_allocator = ${system.gpu_allocator} +dropout = 0.1 +accumulate_gradient = 1 +patience = 1600 +max_epochs = 0 +max_steps = 20000 +eval_frequency = 400 +frozen_components = [] +annotating_components = [] +before_to_disk = null +before_update = null + +[training.optimizer] +@optimizers = "Adam.v1" +beta1 = 0.9 +beta2 = 0.999 +L2_is_weight_decay = true +L2 = 0.01 +grad_clip = 1.0 +use_averages = false +eps = 1e-08 +learn_rate = 0.001 + +[training.batcher] +@batchers = "spacy.batch_by_words.v1" +discard_oversize = false +tolerance = 0.2 +get_length = null + +[training.batcher.size] +@schedules = "compounding.v1" +start = 100 +stop = 1000 +compound = 1.001 +t = 0.0 + +[training.logger] +@loggers = "spacy.ConsoleLogger.v1" +progress_bar = false + +[training.score_weights] +ents_f = 1.0 +ents_p = 0.0 +ents_r = 0.0 +ents_per_type = null + +[initialize] +vectors = ${paths.vectors} +init_tok2vec = ${paths.init_tok2vec} +vocab_data = null +lookups = null +before_init = null +after_init = null + +[initialize.tokenizer] + +[initialize.components] + +[pretraining] diff --git a/docs/MODELS.md b/docs/MODELS.md index a56f071..8a63e9f 100644 --- a/docs/MODELS.md +++ b/docs/MODELS.md @@ -44,6 +44,7 @@ pipelines such as `de_core_news_sm` as `news`. | `fa_core_web_sm` | same as core, mixed-genre training data | hash embeddings | not built; would add ParsTwiNER to cover social media | | `fa_dep_news_md` | same as `fa_dep_news_sm` | floret, 50k rows / 300d | built, shipping | | `fa_core_news_md` | same as `fa_core_news_sm` | floret, 50k rows / 300d | built, shipping | +| `fa_ent_news_lg` | ner (own internal tok2vec) | floret, 200k rows / 300d, full-wiki 5 epochs | built, optional | | `fa_core_news_lg` | same | floret, 200k rows | not built; bigger table, same recipe as md | | `fa_core_news_trf` | transformer instead of tok2vec | `HooshvareLab/roberta-fa-zwnj-base` (Apache-2.0) | not on this hardware; 2 GB VRAM cannot fine-tune a 125M-param encoder | @@ -349,3 +350,59 @@ best checkpoint near 10,800). Whether that trade is worth it depends on deployment. For a 1.19 LAS and 2.85 NER F gain, a 9x larger download and 16% slower parse is a good deal on a server and a bad one in a browser or a Lambda cold start. Both tiers ship; pick per target. + +## 7. The `lg` tier: bigger floret table, `ent` only + +Built after `md`, from a new `fa_floret` table — 200,000 rows x 300d, floret mode, +`minn=maxn=5`, `hash_count=2`, trained on the full Persian Wikipedia dump for 5 epochs (4x +the rows of `md`'s 50k-row table trained on 400k documents). Unpacked the same way as `md` +via `scripts/unpack_vectors.py`, into `assets/vectors/fa_floret_lg`. + +`configs/fa_ner_lg.cfg` is `fa_ner_md.cfg` unchanged except `--paths.vectors`. Only `ent` was +trained at this tier (`fa_ent_news_lg`), not `dep`/`core`: the point of this run was to check +whether a 4x larger table is worth it before spending the CPU time on `dep`/`core` too. Same +seed, same corpus, same architecture as `sm`/`md`. Reproduce with `spacy project run ent-lg`, +or the table alone with `python scripts/compare_tiers.py`. + +### PerDT NER test split, `fa_ent_news_lg` + +| Metric | `sm` | `md` | `lg` | Delta (lg vs sm) | +| --- | --- | --- | --- | --- | +| `ENTS_P` | 77.67 | 76.56 | 81.51 | +3.84 | +| `ENTS_R` | 66.87 | 72.95 | 71.09 | +4.22 | +| `ENTS_F` | 71.87 | 74.71 | 75.94 | +4.08 | + +`lg` beats both `sm` and `md` on `ENTS_F`, and unlike `md`'s recall-only gain over `sm`, `lg` +improves precision too (+3.84 over `sm`, whereas `md` cost -1.10). Consistent with a bigger, +less collision-prone floret table giving both better recall on rare proper nouns and fewer +false positives from hash collisions. + +| Label | Gold in test | `sm` F | `md` F | `lg` F | Delta (lg vs sm) | +| --- | --- | --- | --- | --- | --- | +| `PER` | 297 | 65.29 | 68.18 | 72.63 | +7.33 | +| `LOC` | 273 | 80.24 | 84.05 | 83.66 | +3.42 | +| `ORG` | 144 | 68.77 | 70.25 | 71.01 | +2.24 | +| `DAT` | 69 | 74.45 | 76.19 | 70.83 | -3.62 | +| `MON` | 10 | 73.68 | 84.21 | 88.89 | +15.20 | +| `TIM` | 9 | 66.67 | 66.67 | 61.54 | -5.13 | +| `PCT` | 4 | 57.14 | 33.33 | 57.14 | +0.00 | + +`PER`, `LOC` and `ORG` (714 entities, the labels with real support) all improve over both +smaller tiers. `DAT` and `TIM` regress a few points against `md`; `MON`/`TIM`/`PCT` swings are +one-or-two-entity noise, same caveat as §6. + +### Cost + +The bigger table dominates the artifact even more than `md`'s did: `fa_ent_news_lg` is a +217 MB wheel against 5.6 MB for `sm` and 58 MB for `md` — the 200k x 300d float32 vector +table alone is ~240 MB uncompressed. Training cost was comparable to `sm`/`md` (early stop +at step 7,200 of 20,000, best checkpoint at step 5,600). The `spacy benchmark accuracy` +words/s figures swung in `lg`'s favor in this run (15,614 vs 8,500 `sm` / 7,149 `md`); given +`lg`'s tok2vec architecture is identical to `md`'s and only the static-vector table lookup +differs, treat that as single-run CPU contention noise on shared hardware, not a real +architectural speedup, and re-benchmark before citing a number. + +For a 4x download over `md` (and 39x over `sm`) buying +4.08 ENTS_F over `sm` (+1.23 over +`md`), `lg` is a server/offline-batch pipeline, not something to ship to a browser or a +cold-start function. `dep`/`core` at this tier are not yet built; the `ent`-only result above +is the signal for whether that investment is worth making. diff --git a/project.yml b/project.yml index 797973a..7805fda 100644 --- a/project.yml +++ b/project.yml @@ -34,6 +34,12 @@ vars: core_md_package_name: "core_news_md" floret_wheel: "fa_floret-0.1.0-py3-none-any-400k-documents.whl" vectors_dir: "assets/vectors/fa_floret_400k" + # lg tier. Same architecture as sm/md, but ner only (no dep/core trained yet), with a + # larger floret table: 200k rows x 300d, trained on the full Persian Wikipedia dump for + # 5 epochs (vs md's 50k rows / 400k documents). + ent_lg_package_name: "ent_news_lg" + floret_lg_wheel: "fa_floret-0.1.0-py3-none-any-full-wiki-200k-5epoch.whl" + vectors_lg_dir: "assets/vectors/fa_floret_lg" directories: - "assets" @@ -93,6 +99,16 @@ workflows: - finalize-ent - evaluate-ent - package-ent + # The lg tier: ner only, same corpus as sm/md, with a bigger floret table (200k rows, + # full Persian Wikipedia, 5 epochs) than md's (50k rows, 400k documents). + ent-lg: + - vectors-lg + - train-ner-lg + - finalize-ent-lg + - evaluate-ent-lg + - compare-lg + - package-ent-lg + - smoke-ent-lg # The md tier: same corpus and architecture, plus the fa_floret static vectors. md: - vectors-md @@ -314,6 +330,80 @@ commands: outputs: - "packages/${vars.lang}_${vars.ent_package_name}-${vars.package_version}" + # ---------------------------------------------------------------- lg tier (ner only) + + - name: "vectors-lg" + help: > + Unpack the lg-tier fa_floret wheel into a plain spaCy model directory. 200k rows x + 300d in floret mode, trained on the full Persian Wikipedia dump for 5 epochs, vs + vectors-md's 50k rows / 400k documents. + script: + - "python scripts/unpack_vectors.py ${vars.floret_lg_wheel} ${vars.vectors_lg_dir}" + deps: + - "${vars.floret_lg_wheel}" + - "scripts/unpack_vectors.py" + outputs: + - "${vars.vectors_lg_dir}" + + - name: "train-ner-lg" + help: "Train the NER component with the lg-tier static floret vectors" + script: + - "python -m spacy train configs/fa_ner_lg.cfg --output training/perdt-ner-lg --paths.train corpus/perdt-ner/train.spacy --paths.dev corpus/perdt-ner/dev.spacy --paths.vectors ${vars.vectors_lg_dir} --gpu-id ${vars.gpu}" + deps: + - "corpus/perdt-ner/train.spacy" + - "corpus/perdt-ner/dev.spacy" + - "configs/fa_ner_lg.cfg" + - "${vars.vectors_lg_dir}" + outputs: + - "training/perdt-ner-lg/model-best" + + - name: "finalize-ent-lg" + help: "Write fa_ent_news_lg metadata onto the trained lg model" + script: + - "python scripts/finalize_pipeline.py training/perdt-ner-lg/model-best training/fa_ent_news_lg --variant ent --size lg --version ${vars.package_version}" + deps: + - "training/perdt-ner-lg/model-best" + - "scripts/finalize_pipeline.py" + outputs: + - "training/fa_ent_news_lg" + + - name: "evaluate-ent-lg" + help: "Score fa_ent_news_lg on the held-out PerDT NER test split" + script: + - "python -m spacy benchmark accuracy training/fa_ent_news_lg corpus/perdt-ner/test.spacy --output metrics/lg-perdt-ner-test.json --gpu-id ${vars.gpu}" + - "python scripts/finalize_pipeline.py training/perdt-ner-lg/model-best training/fa_ent_news_lg --variant ent --size lg --version ${vars.package_version} --ner-metrics metrics/lg-perdt-ner-test.json" + deps: + - "training/fa_ent_news_lg" + - "corpus/perdt-ner/test.spacy" + outputs: + - "metrics/lg-perdt-ner-test.json" + + - name: "compare-lg" + help: "Table the sm vs md vs lg ent NER deltas from the metrics/ JSON reports" + script: + - "python scripts/compare_tiers.py" + deps: + - "metrics/perdt-ner-test.json" + - "metrics/md-perdt-ner-test.json" + - "metrics/lg-perdt-ner-test.json" + - "scripts/compare_tiers.py" + + - name: "package-ent-lg" + help: "Build the installable fa_ent_news_lg wheel + sdist" + script: + - "python -m spacy package training/fa_ent_news_lg packages --name ${vars.ent_lg_package_name} --version ${vars.package_version} --build sdist,wheel --force" + deps: + - "training/fa_ent_news_lg" + outputs: + - "packages/${vars.lang}_${vars.ent_lg_package_name}-${vars.package_version}" + + - name: "smoke-ent-lg" + help: "Load fa_ent_news_lg and run it over real Persian text" + script: + - "python scripts/smoke_test.py training/fa_ent_news_lg" + deps: + - "training/fa_ent_news_lg" + # ---------------------------------------------------------------- md tier - name: "vectors-md" diff --git a/scripts/compare_tiers.py b/scripts/compare_tiers.py index 9bbc727..d032a09 100644 --- a/scripts/compare_tiers.py +++ b/scripts/compare_tiers.py @@ -1,8 +1,9 @@ -"""Table the sm vs md test-set deltas. +"""Table the sm vs md vs lg test-set deltas. -Both tiers are trained from the same corpus, the same seed and the same architecture; the -only difference is `include_static_vectors`. So the delta printed here is attributable to the -fa_floret vector table and nothing else. +All tiers are trained from the same corpus, the same seed and the same architecture; the +only difference is the static vector table (none for sm, fa_floret 50k rows for md, fa_floret +200k rows for lg) via `include_static_vectors`. So the delta printed here is attributable to +the vector table and nothing else. Reads the `spacy benchmark accuracy` reports written by the `evaluate-*` targets. Missing files are reported rather than fatal, so this is runnable mid-build. @@ -15,11 +16,18 @@ import argparse import json from pathlib import Path -# (label, sm report, md report) -PAIRS = [ - ("dep pipeline, UD test", "ud-test.json", "md-ud-test.json"), - ("core pipeline, UD test", "core-ud-test.json", "md-core-ud-test.json"), - ("core pipeline, NER test", "perdt-ner-test.json", "md-perdt-ner-test.json"), +# (label, {tier_label: report_filename}) +GROUPS = [ + ("dep pipeline, UD test", {"sm": "ud-test.json", "md": "md-ud-test.json"}), + ("core pipeline, UD test", {"sm": "core-ud-test.json", "md": "md-core-ud-test.json"}), + ( + "ent NER test", + { + "sm": "perdt-ner-test.json", + "md": "md-perdt-ner-test.json", + "lg": "lg-perdt-ner-test.json", + }, + ), ] SCALARS = [ @@ -40,41 +48,81 @@ def load(path): return json.loads(path.read_text(encoding="utf8")) if path.exists() else None -def table(title, sm, md, rows): +def table(title, tiers, rows): + """tiers: list of (label, data-dict-or-None), first tier is the baseline for deltas.""" + labels = [label for label, _ in tiers] + base_label, base = tiers[0] print(f"\n## {title}\n") - print(f"| {'metric':<12} | {'sm':>7} | {'md':>7} | {'delta':>7} |") - print(f"| {'-' * 12} | {'-' * 7} | {'-' * 7} | {'-' * 7} |") + header = " | ".join(f"{label:>7}" for label in labels) + delta_header = " | ".join(f"{'d(' + label + ')':>9}" for label, _ in tiers[1:]) + print(f"| {'metric':<12} | {header} | {delta_header} |") + sep = " | ".join("-" * 7 for _ in labels) + delta_sep = " | ".join("-" * 9 for _ in tiers[1:]) + print(f"| {'-' * 12} | {sep} | {delta_sep} |") for key, label in rows: - a, b = sm.get(key), md.get(key) - if a is None and b is None: + values = [d.get(key) if d is not None else None for _, d in tiers] + if all(v is None for v in values): continue # The NER report scores tag_acc 0.0 because its corpus has no gold tags. - if a == 0.0 and b == 0.0: + if all(v == 0.0 for v in values): continue - cells = [f"{v * 100:.2f}" if isinstance(v, float) else "-" for v in (a, b)] - delta = f"{(b - a) * 100:+.2f}" if isinstance(a, float) and isinstance(b, float) else "-" - print(f"| {label:<12} | {cells[0]:>7} | {cells[1]:>7} | {delta:>7} |") - for key, label in (("speed", "words/s"),): - a, b = sm.get(key), md.get(key) - if isinstance(a, float) and isinstance(b, float): - print(f"| {label:<12} | {a:>7.0f} | {b:>7.0f} | {b / a - 1:>+6.1%} |") + cells = [f"{v * 100:.2f}" if isinstance(v, float) else "-" for v in values] + deltas = [] + for v in values[1:]: + a, b = values[0], v + deltas.append( + f"{(b - a) * 100:+.2f}" if isinstance(a, float) and isinstance(b, float) else "-" + ) + row = " | ".join(f"{c:>7}" for c in cells) + drow = " | ".join(f"{d:>9}" for d in deltas) + print(f"| {label:<12} | {row} | {drow} |") + speeds = [d.get("speed") if d is not None else None for _, d in tiers] + if isinstance(speeds[0], float): + cells = [f"{s:.0f}" if isinstance(s, float) else "-" for s in speeds] + deltas = [ + f"{s / speeds[0] - 1:+.1%}" if isinstance(s, float) else "-" for s in speeds[1:] + ] + row = " | ".join(f"{c:>7}" for c in cells) + drow = " | ".join(f"{d:>9}" for d in deltas) + print(f"| {'words/s':<12} | {row} | {drow} |") -def per_type(title, sm, md): - a, b = sm.get("ents_per_type"), md.get("ents_per_type") - if not a or not b: +def per_type(title, tiers): + per_types = [(label, (d or {}).get("ents_per_type")) for label, d in tiers] + if not any(pt for _, pt in per_types): return + labels = [label for label, _ in tiers] def pct(v): return f"{v * 100:.2f}" if v is not None else "-" + all_labels = set() + for _, pt in per_types: + if pt: + all_labels |= set(pt) + print(f"\n### {title}, per label\n") - print(f"| {'label':<6} | {'sm F':>7} | {'md F':>7} | {'delta':>7} |") - print(f"| {'-' * 6} | {'-' * 7} | {'-' * 7} | {'-' * 7} |") - for label in sorted(set(a) | set(b), key=lambda k: -b.get(k, {}).get("f", 0)): - fa, fb = a.get(label, {}).get("f"), b.get(label, {}).get("f") - delta = f"{(fb - fa) * 100:+.2f}" if fa is not None and fb is not None else "-" - print(f"| {label:<6} | {pct(fa):>7} | {pct(fb):>7} | {delta:>7} |") + header = " | ".join(f"{label + ' F':>7}" for label in labels) + delta_header = " | ".join(f"{'d(' + label + ')':>9}" for label in labels[1:]) + print(f"| {'label':<6} | {header} | {delta_header} |") + sep = " | ".join("-" * 7 for _ in labels) + delta_sep = " | ".join("-" * 9 for _ in labels[1:]) + print(f"| {'-' * 6} | {sep} | {delta_sep} |") + + def sort_key(entity_label): + last_pt = per_types[-1][1] or {} + return -last_pt.get(entity_label, {}).get("f", 0) + + for entity_label in sorted(all_labels, key=sort_key): + fs = [(pt or {}).get(entity_label, {}).get("f") for _, pt in per_types] + cells = [pct(f) for f in fs] + deltas = [] + for f in fs[1:]: + a = fs[0] + deltas.append(f"{(f - a) * 100:+.2f}" if a is not None and f is not None else "-") + row = " | ".join(f"{c:>7}" for c in cells) + drow = " | ".join(f"{d:>9}" for d in deltas) + print(f"| {entity_label:<6} | {row} | {drow} |") def main(): @@ -82,19 +130,26 @@ def main(): ap.add_argument("--metrics-dir", type=Path, default=Path("metrics")) args = ap.parse_args() - print("# sm vs md (fa_floret 400k static vectors)") - print("\nSame corpus, same seed, same architecture. Only difference:") - print("`include_static_vectors = false -> true`.") + print("# sm vs md vs lg (fa_floret static vectors)") + print("\nSame corpus, same seed, same architecture per group. Only difference:") + print("`include_static_vectors = false -> true`, and which floret table (md: 50k rows,") + print("400k documents; lg: 200k rows, full Persian Wikipedia, 5 epochs).") - for title, sm_name, md_name in PAIRS: - sm = load(args.metrics_dir / sm_name) - md = load(args.metrics_dir / md_name) - if sm is None or md is None: - missing = [n for n, d in ((sm_name, sm), (md_name, md)) if d is None] - print(f"\n## {title}\n\n (skipped, missing {', '.join(missing)})") + for title, reports in GROUPS: + tiers = [] + missing = [] + for label, fname in reports.items(): + data = load(args.metrics_dir / fname) + if data is None: + missing.append(fname) + tiers.append((label, data)) + if tiers[0][1] is None: + print(f"\n## {title}\n\n (skipped, missing baseline {reports[list(reports)[0]]})") continue - table(title, sm, md, SCALARS) - per_type(title, sm, md) + table(title, tiers, SCALARS) + per_type(title, tiers) + if missing: + print(f"\n (missing: {', '.join(missing)})") if __name__ == "__main__": diff --git a/scripts/finalize_pipeline.py b/scripts/finalize_pipeline.py index 89c5b84..70b75dc 100644 --- a/scripts/finalize_pipeline.py +++ b/scripts/finalize_pipeline.py @@ -67,8 +67,8 @@ FLORET = { "license": "CC BY-SA 4.0", } FLORET_LG = { - "name": "fa_floret static vectors (lg tier: larger floret table trained on fa Wikipedia + " - "OSCAR via spacy-vectors-builder)", + "name": "fa_floret static vectors (lg tier: 200k rows x 300d floret table trained on " + "the full Persian Wikipedia dump, 5 epochs, via spacy-vectors-builder)", "url": PROJECT_URL, "author": "Kiyarash Fazeli", "license": "CC BY-SA 4.0", @@ -119,9 +119,10 @@ def vectors_note_lg(nlp): return ( f"This is the `lg` tier: identical architecture to `sm`/`md` but a larger static " f"floret vector table ({rows:,} rows x {dim} dimensions, minn=maxn=5, hash_count=2) " - f"trained on Persian Wikipedia + OSCAR via spacy-vectors-builder. Same zero-OOV " - f"rationale as `md` (see docs/MODELS.md): floret hashes subwords into a fixed table, " - f"so `token.has_vector` is always True despite Persian's ZWNJ (U+200C) inconsistency." + f"trained on the full Persian Wikipedia dump for 5 epochs via spacy-vectors-builder. " + f"Same zero-OOV rationale as `md` (see docs/MODELS.md): floret hashes subwords into " + f"a fixed table, so `token.has_vector` is always True despite Persian's ZWNJ " + f"(U+200C) inconsistency." )