Add lg tier: fa_ent_news_lg on 200k floret vectors
New table: 200k rows x 300d, full Persian Wikipedia dump, 5 epochs, vs md's 50k rows / 400k documents. - configs/fa_ner_lg.cfg, project.yml ent-lg workflow (vectors-lg through smoke-ent-lg) - scripts/compare_tiers.py: generalized sm/md pair to N tiers; ent NER test now includes lg; fixed sm baseline to the file that's actually scored (perdt-ner-test.json, not the missing ent-test.json) - scripts/finalize_pipeline.py: FLORET_LG source and vectors_note_lg corrected to full Wikipedia, 5 epochs (were a generic Wikipedia + OSCAR placeholder) - docs/MODELS.md §7: PerDT NER test ENTS_F 75.94 (sm 71.87, md 74.71), full per-label table, cost (217 MB wheel) Not built: fa_dep_news_lg / fa_core_news_lg.
This commit is contained in:
parent
c3cb02d9c3
commit
b89b01ceb6
|
|
@ -0,0 +1,154 @@
|
||||||
|
# fa_ent_news_lg — Persian NER with the lg-tier static floret vectors.
|
||||||
|
#
|
||||||
|
# Identical to configs/fa_ner_md.cfg (which is identical to fa_ner_sm.cfg except
|
||||||
|
# include_static_vectors: true). Only the vector table supplied at train time via
|
||||||
|
# --paths.vectors differs: fa_floret, 200k rows x 300d, floret mode, trained on the full
|
||||||
|
# Persian Wikipedia dump for 5 epochs (assets/vectors/fa_floret_lg), vs md's 50k rows x 300d
|
||||||
|
# trained on 400k Persian documents.
|
||||||
|
#
|
||||||
|
# Same embedded-tok2vec design as sm/md (no Tok2VecListener), so the trained component stays
|
||||||
|
# sourceable into a future fa_core_news_lg via `nlp.add_pipe("ner", source=...)`.
|
||||||
|
|
||||||
|
[paths]
|
||||||
|
train = null
|
||||||
|
dev = null
|
||||||
|
vectors = null
|
||||||
|
init_tok2vec = null
|
||||||
|
|
||||||
|
[system]
|
||||||
|
gpu_allocator = null
|
||||||
|
seed = 0
|
||||||
|
|
||||||
|
[nlp]
|
||||||
|
lang = "fa"
|
||||||
|
pipeline = ["ner"]
|
||||||
|
batch_size = 1000
|
||||||
|
disabled = []
|
||||||
|
before_creation = null
|
||||||
|
after_creation = null
|
||||||
|
after_pipeline_creation = null
|
||||||
|
|
||||||
|
[nlp.tokenizer]
|
||||||
|
@tokenizers = "spacy.Tokenizer.v1"
|
||||||
|
|
||||||
|
[nlp.vectors]
|
||||||
|
@vectors = "spacy.Vectors.v1"
|
||||||
|
|
||||||
|
[components]
|
||||||
|
|
||||||
|
[components.ner]
|
||||||
|
factory = "ner"
|
||||||
|
moves = null
|
||||||
|
update_with_oracle_cut_size = 100
|
||||||
|
incorrect_spans_key = null
|
||||||
|
|
||||||
|
[components.ner.model]
|
||||||
|
@architectures = "spacy.TransitionBasedParser.v2"
|
||||||
|
state_type = "ner"
|
||||||
|
extra_state_tokens = false
|
||||||
|
hidden_width = 64
|
||||||
|
maxout_pieces = 2
|
||||||
|
use_upper = true
|
||||||
|
nO = null
|
||||||
|
|
||||||
|
[components.ner.model.tok2vec]
|
||||||
|
@architectures = "spacy.Tok2Vec.v2"
|
||||||
|
|
||||||
|
[components.ner.model.tok2vec.embed]
|
||||||
|
@architectures = "spacy.MultiHashEmbed.v2"
|
||||||
|
width = ${components.ner.model.tok2vec.encode.width}
|
||||||
|
attrs = ["NORM", "PREFIX", "SUFFIX", "SHAPE"]
|
||||||
|
rows = [5000, 1000, 2500, 2500]
|
||||||
|
include_static_vectors = true
|
||||||
|
|
||||||
|
[components.ner.model.tok2vec.encode]
|
||||||
|
@architectures = "spacy.MaxoutWindowEncoder.v2"
|
||||||
|
width = 96
|
||||||
|
depth = 4
|
||||||
|
window_size = 1
|
||||||
|
maxout_pieces = 3
|
||||||
|
|
||||||
|
[components.ner.scorer]
|
||||||
|
@scorers = "spacy.ner_scorer.v1"
|
||||||
|
|
||||||
|
[corpora]
|
||||||
|
|
||||||
|
[corpora.train]
|
||||||
|
@readers = "spacy.Corpus.v1"
|
||||||
|
path = ${paths.train}
|
||||||
|
max_length = 0
|
||||||
|
gold_preproc = false
|
||||||
|
limit = 0
|
||||||
|
augmenter = null
|
||||||
|
|
||||||
|
[corpora.dev]
|
||||||
|
@readers = "spacy.Corpus.v1"
|
||||||
|
path = ${paths.dev}
|
||||||
|
max_length = 0
|
||||||
|
gold_preproc = false
|
||||||
|
limit = 0
|
||||||
|
augmenter = null
|
||||||
|
|
||||||
|
[training]
|
||||||
|
dev_corpus = "corpora.dev"
|
||||||
|
train_corpus = "corpora.train"
|
||||||
|
seed = ${system.seed}
|
||||||
|
gpu_allocator = ${system.gpu_allocator}
|
||||||
|
dropout = 0.1
|
||||||
|
accumulate_gradient = 1
|
||||||
|
patience = 1600
|
||||||
|
max_epochs = 0
|
||||||
|
max_steps = 20000
|
||||||
|
eval_frequency = 400
|
||||||
|
frozen_components = []
|
||||||
|
annotating_components = []
|
||||||
|
before_to_disk = null
|
||||||
|
before_update = null
|
||||||
|
|
||||||
|
[training.optimizer]
|
||||||
|
@optimizers = "Adam.v1"
|
||||||
|
beta1 = 0.9
|
||||||
|
beta2 = 0.999
|
||||||
|
L2_is_weight_decay = true
|
||||||
|
L2 = 0.01
|
||||||
|
grad_clip = 1.0
|
||||||
|
use_averages = false
|
||||||
|
eps = 1e-08
|
||||||
|
learn_rate = 0.001
|
||||||
|
|
||||||
|
[training.batcher]
|
||||||
|
@batchers = "spacy.batch_by_words.v1"
|
||||||
|
discard_oversize = false
|
||||||
|
tolerance = 0.2
|
||||||
|
get_length = null
|
||||||
|
|
||||||
|
[training.batcher.size]
|
||||||
|
@schedules = "compounding.v1"
|
||||||
|
start = 100
|
||||||
|
stop = 1000
|
||||||
|
compound = 1.001
|
||||||
|
t = 0.0
|
||||||
|
|
||||||
|
[training.logger]
|
||||||
|
@loggers = "spacy.ConsoleLogger.v1"
|
||||||
|
progress_bar = false
|
||||||
|
|
||||||
|
[training.score_weights]
|
||||||
|
ents_f = 1.0
|
||||||
|
ents_p = 0.0
|
||||||
|
ents_r = 0.0
|
||||||
|
ents_per_type = null
|
||||||
|
|
||||||
|
[initialize]
|
||||||
|
vectors = ${paths.vectors}
|
||||||
|
init_tok2vec = ${paths.init_tok2vec}
|
||||||
|
vocab_data = null
|
||||||
|
lookups = null
|
||||||
|
before_init = null
|
||||||
|
after_init = null
|
||||||
|
|
||||||
|
[initialize.tokenizer]
|
||||||
|
|
||||||
|
[initialize.components]
|
||||||
|
|
||||||
|
[pretraining]
|
||||||
|
|
@ -44,6 +44,7 @@ pipelines such as `de_core_news_sm` as `news`.
|
||||||
| `fa_core_web_sm` | same as core, mixed-genre training data | hash embeddings | not built; would add ParsTwiNER to cover social media |
|
| `fa_core_web_sm` | same as core, mixed-genre training data | hash embeddings | not built; would add ParsTwiNER to cover social media |
|
||||||
| `fa_dep_news_md` | same as `fa_dep_news_sm` | floret, 50k rows / 300d | built, shipping |
|
| `fa_dep_news_md` | same as `fa_dep_news_sm` | floret, 50k rows / 300d | built, shipping |
|
||||||
| `fa_core_news_md` | same as `fa_core_news_sm` | floret, 50k rows / 300d | built, shipping |
|
| `fa_core_news_md` | same as `fa_core_news_sm` | floret, 50k rows / 300d | built, shipping |
|
||||||
|
| `fa_ent_news_lg` | ner (own internal tok2vec) | floret, 200k rows / 300d, full-wiki 5 epochs | built, optional |
|
||||||
| `fa_core_news_lg` | same | floret, 200k rows | not built; bigger table, same recipe as md |
|
| `fa_core_news_lg` | same | floret, 200k rows | not built; bigger table, same recipe as md |
|
||||||
| `fa_core_news_trf` | transformer instead of tok2vec | `HooshvareLab/roberta-fa-zwnj-base` (Apache-2.0) | not on this hardware; 2 GB VRAM cannot fine-tune a 125M-param encoder |
|
| `fa_core_news_trf` | transformer instead of tok2vec | `HooshvareLab/roberta-fa-zwnj-base` (Apache-2.0) | not on this hardware; 2 GB VRAM cannot fine-tune a 125M-param encoder |
|
||||||
|
|
||||||
|
|
@ -349,3 +350,59 @@ best checkpoint near 10,800).
|
||||||
Whether that trade is worth it depends on deployment. For a 1.19 LAS and 2.85 NER F gain, a
|
Whether that trade is worth it depends on deployment. For a 1.19 LAS and 2.85 NER F gain, a
|
||||||
9x larger download and 16% slower parse is a good deal on a server and a bad one in a browser
|
9x larger download and 16% slower parse is a good deal on a server and a bad one in a browser
|
||||||
or a Lambda cold start. Both tiers ship; pick per target.
|
or a Lambda cold start. Both tiers ship; pick per target.
|
||||||
|
|
||||||
|
## 7. The `lg` tier: bigger floret table, `ent` only
|
||||||
|
|
||||||
|
Built after `md`, from a new `fa_floret` table — 200,000 rows x 300d, floret mode,
|
||||||
|
`minn=maxn=5`, `hash_count=2`, trained on the full Persian Wikipedia dump for 5 epochs (4x
|
||||||
|
the rows of `md`'s 50k-row table trained on 400k documents). Unpacked the same way as `md`
|
||||||
|
via `scripts/unpack_vectors.py`, into `assets/vectors/fa_floret_lg`.
|
||||||
|
|
||||||
|
`configs/fa_ner_lg.cfg` is `fa_ner_md.cfg` unchanged except `--paths.vectors`. Only `ent` was
|
||||||
|
trained at this tier (`fa_ent_news_lg`), not `dep`/`core`: the point of this run was to check
|
||||||
|
whether a 4x larger table is worth it before spending the CPU time on `dep`/`core` too. Same
|
||||||
|
seed, same corpus, same architecture as `sm`/`md`. Reproduce with `spacy project run ent-lg`,
|
||||||
|
or the table alone with `python scripts/compare_tiers.py`.
|
||||||
|
|
||||||
|
### PerDT NER test split, `fa_ent_news_lg`
|
||||||
|
|
||||||
|
| Metric | `sm` | `md` | `lg` | Delta (lg vs sm) |
|
||||||
|
| --- | --- | --- | --- | --- |
|
||||||
|
| `ENTS_P` | 77.67 | 76.56 | 81.51 | +3.84 |
|
||||||
|
| `ENTS_R` | 66.87 | 72.95 | 71.09 | +4.22 |
|
||||||
|
| `ENTS_F` | 71.87 | 74.71 | 75.94 | +4.08 |
|
||||||
|
|
||||||
|
`lg` beats both `sm` and `md` on `ENTS_F`, and unlike `md`'s recall-only gain over `sm`, `lg`
|
||||||
|
improves precision too (+3.84 over `sm`, whereas `md` cost -1.10). Consistent with a bigger,
|
||||||
|
less collision-prone floret table giving both better recall on rare proper nouns and fewer
|
||||||
|
false positives from hash collisions.
|
||||||
|
|
||||||
|
| Label | Gold in test | `sm` F | `md` F | `lg` F | Delta (lg vs sm) |
|
||||||
|
| --- | --- | --- | --- | --- | --- |
|
||||||
|
| `PER` | 297 | 65.29 | 68.18 | 72.63 | +7.33 |
|
||||||
|
| `LOC` | 273 | 80.24 | 84.05 | 83.66 | +3.42 |
|
||||||
|
| `ORG` | 144 | 68.77 | 70.25 | 71.01 | +2.24 |
|
||||||
|
| `DAT` | 69 | 74.45 | 76.19 | 70.83 | -3.62 |
|
||||||
|
| `MON` | 10 | 73.68 | 84.21 | 88.89 | +15.20 |
|
||||||
|
| `TIM` | 9 | 66.67 | 66.67 | 61.54 | -5.13 |
|
||||||
|
| `PCT` | 4 | 57.14 | 33.33 | 57.14 | +0.00 |
|
||||||
|
|
||||||
|
`PER`, `LOC` and `ORG` (714 entities, the labels with real support) all improve over both
|
||||||
|
smaller tiers. `DAT` and `TIM` regress a few points against `md`; `MON`/`TIM`/`PCT` swings are
|
||||||
|
one-or-two-entity noise, same caveat as §6.
|
||||||
|
|
||||||
|
### Cost
|
||||||
|
|
||||||
|
The bigger table dominates the artifact even more than `md`'s did: `fa_ent_news_lg` is a
|
||||||
|
217 MB wheel against 5.6 MB for `sm` and 58 MB for `md` — the 200k x 300d float32 vector
|
||||||
|
table alone is ~240 MB uncompressed. Training cost was comparable to `sm`/`md` (early stop
|
||||||
|
at step 7,200 of 20,000, best checkpoint at step 5,600). The `spacy benchmark accuracy`
|
||||||
|
words/s figures swung in `lg`'s favor in this run (15,614 vs 8,500 `sm` / 7,149 `md`); given
|
||||||
|
`lg`'s tok2vec architecture is identical to `md`'s and only the static-vector table lookup
|
||||||
|
differs, treat that as single-run CPU contention noise on shared hardware, not a real
|
||||||
|
architectural speedup, and re-benchmark before citing a number.
|
||||||
|
|
||||||
|
For a 4x download over `md` (and 39x over `sm`) buying +4.08 ENTS_F over `sm` (+1.23 over
|
||||||
|
`md`), `lg` is a server/offline-batch pipeline, not something to ship to a browser or a
|
||||||
|
cold-start function. `dep`/`core` at this tier are not yet built; the `ent`-only result above
|
||||||
|
is the signal for whether that investment is worth making.
|
||||||
|
|
|
||||||
90
project.yml
90
project.yml
|
|
@ -34,6 +34,12 @@ vars:
|
||||||
core_md_package_name: "core_news_md"
|
core_md_package_name: "core_news_md"
|
||||||
floret_wheel: "fa_floret-0.1.0-py3-none-any-400k-documents.whl"
|
floret_wheel: "fa_floret-0.1.0-py3-none-any-400k-documents.whl"
|
||||||
vectors_dir: "assets/vectors/fa_floret_400k"
|
vectors_dir: "assets/vectors/fa_floret_400k"
|
||||||
|
# lg tier. Same architecture as sm/md, but ner only (no dep/core trained yet), with a
|
||||||
|
# larger floret table: 200k rows x 300d, trained on the full Persian Wikipedia dump for
|
||||||
|
# 5 epochs (vs md's 50k rows / 400k documents).
|
||||||
|
ent_lg_package_name: "ent_news_lg"
|
||||||
|
floret_lg_wheel: "fa_floret-0.1.0-py3-none-any-full-wiki-200k-5epoch.whl"
|
||||||
|
vectors_lg_dir: "assets/vectors/fa_floret_lg"
|
||||||
|
|
||||||
directories:
|
directories:
|
||||||
- "assets"
|
- "assets"
|
||||||
|
|
@ -93,6 +99,16 @@ workflows:
|
||||||
- finalize-ent
|
- finalize-ent
|
||||||
- evaluate-ent
|
- evaluate-ent
|
||||||
- package-ent
|
- package-ent
|
||||||
|
# The lg tier: ner only, same corpus as sm/md, with a bigger floret table (200k rows,
|
||||||
|
# full Persian Wikipedia, 5 epochs) than md's (50k rows, 400k documents).
|
||||||
|
ent-lg:
|
||||||
|
- vectors-lg
|
||||||
|
- train-ner-lg
|
||||||
|
- finalize-ent-lg
|
||||||
|
- evaluate-ent-lg
|
||||||
|
- compare-lg
|
||||||
|
- package-ent-lg
|
||||||
|
- smoke-ent-lg
|
||||||
# The md tier: same corpus and architecture, plus the fa_floret static vectors.
|
# The md tier: same corpus and architecture, plus the fa_floret static vectors.
|
||||||
md:
|
md:
|
||||||
- vectors-md
|
- vectors-md
|
||||||
|
|
@ -314,6 +330,80 @@ commands:
|
||||||
outputs:
|
outputs:
|
||||||
- "packages/${vars.lang}_${vars.ent_package_name}-${vars.package_version}"
|
- "packages/${vars.lang}_${vars.ent_package_name}-${vars.package_version}"
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------- lg tier (ner only)
|
||||||
|
|
||||||
|
- name: "vectors-lg"
|
||||||
|
help: >
|
||||||
|
Unpack the lg-tier fa_floret wheel into a plain spaCy model directory. 200k rows x
|
||||||
|
300d in floret mode, trained on the full Persian Wikipedia dump for 5 epochs, vs
|
||||||
|
vectors-md's 50k rows / 400k documents.
|
||||||
|
script:
|
||||||
|
- "python scripts/unpack_vectors.py ${vars.floret_lg_wheel} ${vars.vectors_lg_dir}"
|
||||||
|
deps:
|
||||||
|
- "${vars.floret_lg_wheel}"
|
||||||
|
- "scripts/unpack_vectors.py"
|
||||||
|
outputs:
|
||||||
|
- "${vars.vectors_lg_dir}"
|
||||||
|
|
||||||
|
- name: "train-ner-lg"
|
||||||
|
help: "Train the NER component with the lg-tier static floret vectors"
|
||||||
|
script:
|
||||||
|
- "python -m spacy train configs/fa_ner_lg.cfg --output training/perdt-ner-lg --paths.train corpus/perdt-ner/train.spacy --paths.dev corpus/perdt-ner/dev.spacy --paths.vectors ${vars.vectors_lg_dir} --gpu-id ${vars.gpu}"
|
||||||
|
deps:
|
||||||
|
- "corpus/perdt-ner/train.spacy"
|
||||||
|
- "corpus/perdt-ner/dev.spacy"
|
||||||
|
- "configs/fa_ner_lg.cfg"
|
||||||
|
- "${vars.vectors_lg_dir}"
|
||||||
|
outputs:
|
||||||
|
- "training/perdt-ner-lg/model-best"
|
||||||
|
|
||||||
|
- name: "finalize-ent-lg"
|
||||||
|
help: "Write fa_ent_news_lg metadata onto the trained lg model"
|
||||||
|
script:
|
||||||
|
- "python scripts/finalize_pipeline.py training/perdt-ner-lg/model-best training/fa_ent_news_lg --variant ent --size lg --version ${vars.package_version}"
|
||||||
|
deps:
|
||||||
|
- "training/perdt-ner-lg/model-best"
|
||||||
|
- "scripts/finalize_pipeline.py"
|
||||||
|
outputs:
|
||||||
|
- "training/fa_ent_news_lg"
|
||||||
|
|
||||||
|
- name: "evaluate-ent-lg"
|
||||||
|
help: "Score fa_ent_news_lg on the held-out PerDT NER test split"
|
||||||
|
script:
|
||||||
|
- "python -m spacy benchmark accuracy training/fa_ent_news_lg corpus/perdt-ner/test.spacy --output metrics/lg-perdt-ner-test.json --gpu-id ${vars.gpu}"
|
||||||
|
- "python scripts/finalize_pipeline.py training/perdt-ner-lg/model-best training/fa_ent_news_lg --variant ent --size lg --version ${vars.package_version} --ner-metrics metrics/lg-perdt-ner-test.json"
|
||||||
|
deps:
|
||||||
|
- "training/fa_ent_news_lg"
|
||||||
|
- "corpus/perdt-ner/test.spacy"
|
||||||
|
outputs:
|
||||||
|
- "metrics/lg-perdt-ner-test.json"
|
||||||
|
|
||||||
|
- name: "compare-lg"
|
||||||
|
help: "Table the sm vs md vs lg ent NER deltas from the metrics/ JSON reports"
|
||||||
|
script:
|
||||||
|
- "python scripts/compare_tiers.py"
|
||||||
|
deps:
|
||||||
|
- "metrics/perdt-ner-test.json"
|
||||||
|
- "metrics/md-perdt-ner-test.json"
|
||||||
|
- "metrics/lg-perdt-ner-test.json"
|
||||||
|
- "scripts/compare_tiers.py"
|
||||||
|
|
||||||
|
- name: "package-ent-lg"
|
||||||
|
help: "Build the installable fa_ent_news_lg wheel + sdist"
|
||||||
|
script:
|
||||||
|
- "python -m spacy package training/fa_ent_news_lg packages --name ${vars.ent_lg_package_name} --version ${vars.package_version} --build sdist,wheel --force"
|
||||||
|
deps:
|
||||||
|
- "training/fa_ent_news_lg"
|
||||||
|
outputs:
|
||||||
|
- "packages/${vars.lang}_${vars.ent_lg_package_name}-${vars.package_version}"
|
||||||
|
|
||||||
|
- name: "smoke-ent-lg"
|
||||||
|
help: "Load fa_ent_news_lg and run it over real Persian text"
|
||||||
|
script:
|
||||||
|
- "python scripts/smoke_test.py training/fa_ent_news_lg"
|
||||||
|
deps:
|
||||||
|
- "training/fa_ent_news_lg"
|
||||||
|
|
||||||
# ---------------------------------------------------------------- md tier
|
# ---------------------------------------------------------------- md tier
|
||||||
|
|
||||||
- name: "vectors-md"
|
- name: "vectors-md"
|
||||||
|
|
|
||||||
|
|
@ -1,8 +1,9 @@
|
||||||
"""Table the sm vs md test-set deltas.
|
"""Table the sm vs md vs lg test-set deltas.
|
||||||
|
|
||||||
Both tiers are trained from the same corpus, the same seed and the same architecture; the
|
All tiers are trained from the same corpus, the same seed and the same architecture; the
|
||||||
only difference is `include_static_vectors`. So the delta printed here is attributable to the
|
only difference is the static vector table (none for sm, fa_floret 50k rows for md, fa_floret
|
||||||
fa_floret vector table and nothing else.
|
200k rows for lg) via `include_static_vectors`. So the delta printed here is attributable to
|
||||||
|
the vector table and nothing else.
|
||||||
|
|
||||||
Reads the `spacy benchmark accuracy` reports written by the `evaluate-*` targets. Missing
|
Reads the `spacy benchmark accuracy` reports written by the `evaluate-*` targets. Missing
|
||||||
files are reported rather than fatal, so this is runnable mid-build.
|
files are reported rather than fatal, so this is runnable mid-build.
|
||||||
|
|
@ -15,11 +16,18 @@ import argparse
|
||||||
import json
|
import json
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
# (label, sm report, md report)
|
# (label, {tier_label: report_filename})
|
||||||
PAIRS = [
|
GROUPS = [
|
||||||
("dep pipeline, UD test", "ud-test.json", "md-ud-test.json"),
|
("dep pipeline, UD test", {"sm": "ud-test.json", "md": "md-ud-test.json"}),
|
||||||
("core pipeline, UD test", "core-ud-test.json", "md-core-ud-test.json"),
|
("core pipeline, UD test", {"sm": "core-ud-test.json", "md": "md-core-ud-test.json"}),
|
||||||
("core pipeline, NER test", "perdt-ner-test.json", "md-perdt-ner-test.json"),
|
(
|
||||||
|
"ent NER test",
|
||||||
|
{
|
||||||
|
"sm": "perdt-ner-test.json",
|
||||||
|
"md": "md-perdt-ner-test.json",
|
||||||
|
"lg": "lg-perdt-ner-test.json",
|
||||||
|
},
|
||||||
|
),
|
||||||
]
|
]
|
||||||
|
|
||||||
SCALARS = [
|
SCALARS = [
|
||||||
|
|
@ -40,41 +48,81 @@ def load(path):
|
||||||
return json.loads(path.read_text(encoding="utf8")) if path.exists() else None
|
return json.loads(path.read_text(encoding="utf8")) if path.exists() else None
|
||||||
|
|
||||||
|
|
||||||
def table(title, sm, md, rows):
|
def table(title, tiers, rows):
|
||||||
|
"""tiers: list of (label, data-dict-or-None), first tier is the baseline for deltas."""
|
||||||
|
labels = [label for label, _ in tiers]
|
||||||
|
base_label, base = tiers[0]
|
||||||
print(f"\n## {title}\n")
|
print(f"\n## {title}\n")
|
||||||
print(f"| {'metric':<12} | {'sm':>7} | {'md':>7} | {'delta':>7} |")
|
header = " | ".join(f"{label:>7}" for label in labels)
|
||||||
print(f"| {'-' * 12} | {'-' * 7} | {'-' * 7} | {'-' * 7} |")
|
delta_header = " | ".join(f"{'d(' + label + ')':>9}" for label, _ in tiers[1:])
|
||||||
|
print(f"| {'metric':<12} | {header} | {delta_header} |")
|
||||||
|
sep = " | ".join("-" * 7 for _ in labels)
|
||||||
|
delta_sep = " | ".join("-" * 9 for _ in tiers[1:])
|
||||||
|
print(f"| {'-' * 12} | {sep} | {delta_sep} |")
|
||||||
for key, label in rows:
|
for key, label in rows:
|
||||||
a, b = sm.get(key), md.get(key)
|
values = [d.get(key) if d is not None else None for _, d in tiers]
|
||||||
if a is None and b is None:
|
if all(v is None for v in values):
|
||||||
continue
|
continue
|
||||||
# The NER report scores tag_acc 0.0 because its corpus has no gold tags.
|
# The NER report scores tag_acc 0.0 because its corpus has no gold tags.
|
||||||
if a == 0.0 and b == 0.0:
|
if all(v == 0.0 for v in values):
|
||||||
continue
|
continue
|
||||||
cells = [f"{v * 100:.2f}" if isinstance(v, float) else "-" for v in (a, b)]
|
cells = [f"{v * 100:.2f}" if isinstance(v, float) else "-" for v in values]
|
||||||
delta = f"{(b - a) * 100:+.2f}" if isinstance(a, float) and isinstance(b, float) else "-"
|
deltas = []
|
||||||
print(f"| {label:<12} | {cells[0]:>7} | {cells[1]:>7} | {delta:>7} |")
|
for v in values[1:]:
|
||||||
for key, label in (("speed", "words/s"),):
|
a, b = values[0], v
|
||||||
a, b = sm.get(key), md.get(key)
|
deltas.append(
|
||||||
if isinstance(a, float) and isinstance(b, float):
|
f"{(b - a) * 100:+.2f}" if isinstance(a, float) and isinstance(b, float) else "-"
|
||||||
print(f"| {label:<12} | {a:>7.0f} | {b:>7.0f} | {b / a - 1:>+6.1%} |")
|
)
|
||||||
|
row = " | ".join(f"{c:>7}" for c in cells)
|
||||||
|
drow = " | ".join(f"{d:>9}" for d in deltas)
|
||||||
|
print(f"| {label:<12} | {row} | {drow} |")
|
||||||
|
speeds = [d.get("speed") if d is not None else None for _, d in tiers]
|
||||||
|
if isinstance(speeds[0], float):
|
||||||
|
cells = [f"{s:.0f}" if isinstance(s, float) else "-" for s in speeds]
|
||||||
|
deltas = [
|
||||||
|
f"{s / speeds[0] - 1:+.1%}" if isinstance(s, float) else "-" for s in speeds[1:]
|
||||||
|
]
|
||||||
|
row = " | ".join(f"{c:>7}" for c in cells)
|
||||||
|
drow = " | ".join(f"{d:>9}" for d in deltas)
|
||||||
|
print(f"| {'words/s':<12} | {row} | {drow} |")
|
||||||
|
|
||||||
|
|
||||||
def per_type(title, sm, md):
|
def per_type(title, tiers):
|
||||||
a, b = sm.get("ents_per_type"), md.get("ents_per_type")
|
per_types = [(label, (d or {}).get("ents_per_type")) for label, d in tiers]
|
||||||
if not a or not b:
|
if not any(pt for _, pt in per_types):
|
||||||
return
|
return
|
||||||
|
labels = [label for label, _ in tiers]
|
||||||
|
|
||||||
def pct(v):
|
def pct(v):
|
||||||
return f"{v * 100:.2f}" if v is not None else "-"
|
return f"{v * 100:.2f}" if v is not None else "-"
|
||||||
|
|
||||||
|
all_labels = set()
|
||||||
|
for _, pt in per_types:
|
||||||
|
if pt:
|
||||||
|
all_labels |= set(pt)
|
||||||
|
|
||||||
print(f"\n### {title}, per label\n")
|
print(f"\n### {title}, per label\n")
|
||||||
print(f"| {'label':<6} | {'sm F':>7} | {'md F':>7} | {'delta':>7} |")
|
header = " | ".join(f"{label + ' F':>7}" for label in labels)
|
||||||
print(f"| {'-' * 6} | {'-' * 7} | {'-' * 7} | {'-' * 7} |")
|
delta_header = " | ".join(f"{'d(' + label + ')':>9}" for label in labels[1:])
|
||||||
for label in sorted(set(a) | set(b), key=lambda k: -b.get(k, {}).get("f", 0)):
|
print(f"| {'label':<6} | {header} | {delta_header} |")
|
||||||
fa, fb = a.get(label, {}).get("f"), b.get(label, {}).get("f")
|
sep = " | ".join("-" * 7 for _ in labels)
|
||||||
delta = f"{(fb - fa) * 100:+.2f}" if fa is not None and fb is not None else "-"
|
delta_sep = " | ".join("-" * 9 for _ in labels[1:])
|
||||||
print(f"| {label:<6} | {pct(fa):>7} | {pct(fb):>7} | {delta:>7} |")
|
print(f"| {'-' * 6} | {sep} | {delta_sep} |")
|
||||||
|
|
||||||
|
def sort_key(entity_label):
|
||||||
|
last_pt = per_types[-1][1] or {}
|
||||||
|
return -last_pt.get(entity_label, {}).get("f", 0)
|
||||||
|
|
||||||
|
for entity_label in sorted(all_labels, key=sort_key):
|
||||||
|
fs = [(pt or {}).get(entity_label, {}).get("f") for _, pt in per_types]
|
||||||
|
cells = [pct(f) for f in fs]
|
||||||
|
deltas = []
|
||||||
|
for f in fs[1:]:
|
||||||
|
a = fs[0]
|
||||||
|
deltas.append(f"{(f - a) * 100:+.2f}" if a is not None and f is not None else "-")
|
||||||
|
row = " | ".join(f"{c:>7}" for c in cells)
|
||||||
|
drow = " | ".join(f"{d:>9}" for d in deltas)
|
||||||
|
print(f"| {entity_label:<6} | {row} | {drow} |")
|
||||||
|
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
|
|
@ -82,19 +130,26 @@ def main():
|
||||||
ap.add_argument("--metrics-dir", type=Path, default=Path("metrics"))
|
ap.add_argument("--metrics-dir", type=Path, default=Path("metrics"))
|
||||||
args = ap.parse_args()
|
args = ap.parse_args()
|
||||||
|
|
||||||
print("# sm vs md (fa_floret 400k static vectors)")
|
print("# sm vs md vs lg (fa_floret static vectors)")
|
||||||
print("\nSame corpus, same seed, same architecture. Only difference:")
|
print("\nSame corpus, same seed, same architecture per group. Only difference:")
|
||||||
print("`include_static_vectors = false -> true`.")
|
print("`include_static_vectors = false -> true`, and which floret table (md: 50k rows,")
|
||||||
|
print("400k documents; lg: 200k rows, full Persian Wikipedia, 5 epochs).")
|
||||||
|
|
||||||
for title, sm_name, md_name in PAIRS:
|
for title, reports in GROUPS:
|
||||||
sm = load(args.metrics_dir / sm_name)
|
tiers = []
|
||||||
md = load(args.metrics_dir / md_name)
|
missing = []
|
||||||
if sm is None or md is None:
|
for label, fname in reports.items():
|
||||||
missing = [n for n, d in ((sm_name, sm), (md_name, md)) if d is None]
|
data = load(args.metrics_dir / fname)
|
||||||
print(f"\n## {title}\n\n (skipped, missing {', '.join(missing)})")
|
if data is None:
|
||||||
|
missing.append(fname)
|
||||||
|
tiers.append((label, data))
|
||||||
|
if tiers[0][1] is None:
|
||||||
|
print(f"\n## {title}\n\n (skipped, missing baseline {reports[list(reports)[0]]})")
|
||||||
continue
|
continue
|
||||||
table(title, sm, md, SCALARS)
|
table(title, tiers, SCALARS)
|
||||||
per_type(title, sm, md)
|
per_type(title, tiers)
|
||||||
|
if missing:
|
||||||
|
print(f"\n (missing: {', '.join(missing)})")
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
|
|
|
||||||
|
|
@ -67,8 +67,8 @@ FLORET = {
|
||||||
"license": "CC BY-SA 4.0",
|
"license": "CC BY-SA 4.0",
|
||||||
}
|
}
|
||||||
FLORET_LG = {
|
FLORET_LG = {
|
||||||
"name": "fa_floret static vectors (lg tier: larger floret table trained on fa Wikipedia + "
|
"name": "fa_floret static vectors (lg tier: 200k rows x 300d floret table trained on "
|
||||||
"OSCAR via spacy-vectors-builder)",
|
"the full Persian Wikipedia dump, 5 epochs, via spacy-vectors-builder)",
|
||||||
"url": PROJECT_URL,
|
"url": PROJECT_URL,
|
||||||
"author": "Kiyarash Fazeli",
|
"author": "Kiyarash Fazeli",
|
||||||
"license": "CC BY-SA 4.0",
|
"license": "CC BY-SA 4.0",
|
||||||
|
|
@ -119,9 +119,10 @@ def vectors_note_lg(nlp):
|
||||||
return (
|
return (
|
||||||
f"This is the `lg` tier: identical architecture to `sm`/`md` but a larger static "
|
f"This is the `lg` tier: identical architecture to `sm`/`md` but a larger static "
|
||||||
f"floret vector table ({rows:,} rows x {dim} dimensions, minn=maxn=5, hash_count=2) "
|
f"floret vector table ({rows:,} rows x {dim} dimensions, minn=maxn=5, hash_count=2) "
|
||||||
f"trained on Persian Wikipedia + OSCAR via spacy-vectors-builder. Same zero-OOV "
|
f"trained on the full Persian Wikipedia dump for 5 epochs via spacy-vectors-builder. "
|
||||||
f"rationale as `md` (see docs/MODELS.md): floret hashes subwords into a fixed table, "
|
f"Same zero-OOV rationale as `md` (see docs/MODELS.md): floret hashes subwords into "
|
||||||
f"so `token.has_vector` is always True despite Persian's ZWNJ (U+200C) inconsistency."
|
f"a fixed table, so `token.has_vector` is always True despite Persian's ZWNJ "
|
||||||
|
f"(U+200C) inconsistency."
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue