From 0d2648fb6a579ca46a8408c0356d3766ec8d79c3 Mon Sep 17 00:00:00 2001 From: Ronald Tse Date: Wed, 29 Jul 2026 00:25:19 +0800 Subject: [PATCH 1/8] chore: Ruby 3.3+ floor, modern GHA, gemspec hygiene --- .github/dependabot.yml | 10 +++ .github/workflows/python-arabic.yml | 98 ++++++++++++++--------------- .github/workflows/release.yml | 20 +++--- .github/workflows/ruby.yml | 24 ++++--- rababa.gemspec | 25 +++++--- 5 files changed, 93 insertions(+), 84 deletions(-) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..9eb31f3 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,10 @@ +version: 2 +updates: + - package-ecosystem: github-actions + directory: "/" + schedule: + interval: weekly + - package-ecosystem: bundler + directory: "/" + schedule: + interval: weekly diff --git a/.github/workflows/python-arabic.yml b/.github/workflows/python-arabic.yml index 68488c9..46607f3 100644 --- a/.github/workflows/python-arabic.yml +++ b/.github/workflows/python-arabic.yml @@ -11,71 +11,65 @@ jobs: strategy: fail-fast: false matrix: - python-version: ['3.6', '3.7', '3.8', '3.9'] + python-version: ["3.9"] steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} + - uses: actions/checkout@v7 + - uses: actions/setup-python@v7 + with: + python-version: ${{ matrix.python-version }} + cache: pip + cache-dependency-path: python/arabic/requirements.txt - - uses: actions/cache@v2 - with: - path: ${{ env.pythonLocation }} - key: ${{ env.pythonLocation }}-${{ hashFiles('python/arabic/setup.py') }}-${{ hashFiles('python/arabic/requirements.txt') }} + - name: Install requirements + working-directory: ./python/arabic + run: | + pip install --upgrade --upgrade-strategy eager -r requirements.txt -e . - - name: Install requirements - working-directory: ./python/arabic - run: | - pip install --upgrade --upgrade-strategy eager -r requirements.txt -e . + - name: Download PyTorch model + working-directory: ./python/arabic + run: | + curl -sSL https://github.com/secryst/rababa-models/releases/download/0.1/2000000-snapshot.pt \ + -o log_dir/CA_MSA.base.cbhg/models/2000000-snapshot.pt - - name: Download PyTorch model - working-directory: ./python/arabic - run: | - curl -sSL https://github.com/secryst/rababa-models/releases/download/0.1/2000000-snapshot.pt \ - -o log_dir/CA_MSA.base.cbhg/models/2000000-snapshot.pt - - - name: Run diacriticization - working-directory: ./python/arabic - run: | - python diacritize.py --model_kind "cbhg" --config config/cbhg.yml --text 'قطر' + - name: Run diacriticization + working-directory: ./python/arabic + run: | + python diacritize.py --model_kind "cbhg" --config config/cbhg.yml --text 'قطر' train: runs-on: ubuntu-latest strategy: fail-fast: false matrix: - python-version: ['3.6', '3.7', '3.8', '3.9'] + python-version: ["3.9"] steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - - uses: actions/cache@v2 - with: - path: ${{ env.pythonLocation }} - key: ${{ env.pythonLocation }}-${{ hashFiles('python/setup.py') }}-${{ hashFiles('python/requirements.txt') }} + - uses: actions/checkout@v7 + - uses: actions/setup-python@v7 + with: + python-version: ${{ matrix.python-version }} + cache: pip + cache-dependency-path: python/arabic/requirements.txt - - name: Install requirements - working-directory: ./python/arabic - run: | - pip install --upgrade --upgrade-strategy eager -r requirements.txt -e . + - name: Install requirements + working-directory: ./python/arabic + run: | + pip install --upgrade --upgrade-strategy eager -r requirements.txt -e . - - name: Prepare dataset - working-directory: ./python/arabic - run: | - mkdir -p data/CA_MSA - touch data/CA_MSA/{eval,train,test}.csv - cd data - curl -sSL https://github.com/interscript/rababa-tashkeela/archive/refs/tags/v1.0.zip -o tashkeela.zip - unzip tashkeela.zip - for d in `ls rababa-tashkeela-1.0/tashkeela_val/*`; do cat $d >> CA_MSA/eval.csv; done - for d in `ls rababa-tashkeela-1.0/tashkeela_train/*`; do cat $d >> CA_MSA/train.csv; done - for d in `ls rababa-tashkeela-1.0/tashkeela_test/*`; do cat $d >> CA_MSA/test.csv; done + - name: Prepare dataset + working-directory: ./python/arabic + run: | + mkdir -p data/CA_MSA + touch data/CA_MSA/{eval,train,test}.csv + cd data + curl -sSL https://github.com/interscript/rababa-tashkeela/archive/refs/tags/v1.0.zip -o tashkeela.zip + unzip tashkeela.zip + for d in `ls rababa-tashkeela-1.0/tashkeela_val/*`; do cat $d >> CA_MSA/eval.csv; done + for d in `ls rababa-tashkeela-1.0/tashkeela_train/*`; do cat $d >> CA_MSA/train.csv; done + for d in `ls rababa-tashkeela-1.0/tashkeela_test/*`; do cat $d >> CA_MSA/test.csv; done - - name: Try training (WIP) - working-directory: ./python/arabic - run: | - python train.py --model "cbhg" --config config/test_cbhg.yml + - name: Try training (WIP) + working-directory: ./python/arabic + run: | + python train.py --model "cbhg" --config config/test_cbhg.yml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a8165a1..6cf4e72 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -9,12 +9,12 @@ jobs: release: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v7 - - uses: actions/setup-ruby@v1 + - uses: ruby/setup-ruby@v1 with: - ruby-version: '2.7' - architecture: 'x64' + ruby-version: '3.3' + bundler-cache: true - run: bundle install --jobs 4 --retry 3 @@ -23,14 +23,10 @@ jobs: - name: Publish to rubygems.org env: - RUBYGEMS_API_KEY: ${{secrets.INTERSCRIPT_RUBYGEMS_API_KEY}} + RUBYGEMS_API_KEY: ${{ secrets.INTERSCRIPT_RUBYGEMS_API_KEY }} run: | - gem install gem-release - touch ~/.gem/credentials - cat > ~/.gem/credentials << EOF - --- - :rubygems_api_key: ${RUBYGEMS_API_KEY} - EOF + mkdir -p ~/.gem + printf -- "---\n:rubygems_api_key: %s\n" "$RUBYGEMS_API_KEY" > ~/.gem/credentials chmod 0600 ~/.gem/credentials - git status + gem install gem-release gem release diff --git a/.github/workflows/ruby.yml b/.github/workflows/ruby.yml index f8ebdbf..df63646 100644 --- a/.github/workflows/ruby.yml +++ b/.github/workflows/ruby.yml @@ -11,21 +11,19 @@ jobs: strategy: fail-fast: false matrix: - ruby-version: ['2.6', '2.7', '3.0', '3.1', '3.2'] + ruby-version: ["3.3", "3.4"] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v7 - - name: Set up Ruby - uses: ruby/setup-ruby@v1 - with: - ruby-version: ${{ matrix.ruby-version }} - bundler-cache: true + - name: Set up Ruby + uses: ruby/setup-ruby@v1 + with: + ruby-version: ${{ matrix.ruby-version }} + bundler-cache: true - - name: Run rake - run: | - bundle exec rake + - name: Run rake + run: bundle exec rake - - name: Run standardrb (use bundle exec standardrb --fix) - run: | - bundle exec standardrb + - name: Run standardrb + run: bundle exec standardrb diff --git a/rababa.gemspec b/rababa.gemspec index 76322ac..ca8be83 100644 --- a/rababa.gemspec +++ b/rababa.gemspec @@ -9,16 +9,27 @@ Gem::Specification.new do |spec| spec.email = ["open.source@ribose.com"] spec.summary = "Middle Eastern Languages diacriticizer from Interscript." - # spec.description = "TODO: Write a longer description or delete this line." + spec.description = "Middle Eastern Languages diacriticizer from Interscript." spec.homepage = "https://www.interscript.org" - spec.required_ruby_version = Gem::Requirement.new(">= 2.5.0") + spec.required_ruby_version = ">= 3.3.0" - spec.metadata["homepage_uri"] = spec.homepage - spec.metadata["source_code_uri"] = "https://github.com/interscript/rababa" - spec.metadata["changelog_uri"] = "https://github.com/interscript/rababa" + spec.metadata["homepage_uri"] = spec.homepage + spec.metadata["source_code_uri"] = "https://github.com/interscript/rababa" + spec.metadata["changelog_uri"] = "https://github.com/interscript/rababa/releases" + spec.metadata["bug_tracker_uri"] = "https://github.com/interscript/rababa/issues" + spec.metadata["rubygems_mfa_required"] = "true" - spec.files = Dir.chdir(File.expand_path(__dir__)) do - `git ls-files -z`.split("\x0").reject { |f| f.match(%r{\A(?:test|spec|features)/}) } + spec.files = Dir.chdir(__dir__) do + Dir[ + "lib/**/*", + "exe/**/*", + "config/**/*", + "data/**/*", + "models-data/**/*", + "README*", + "LICENSE*", + "*.gemspec" + ].select { |f| File.file?(f) } end spec.bindir = "exe" spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) } From 205638ef51243846ce1954045b95db6b8674553d Mon Sep 17 00:00:00 2001 From: Ronald Tse Date: Wed, 29 Jul 2026 12:17:13 +0800 Subject: [PATCH 2/8] docs: add SECURITY.md --- SECURITY.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 SECURITY.md diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..4c1f49b --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,20 @@ +# Security Policy + +## Supported Versions + +The latest released version of this project receives security fixes. + +## Reporting a Vulnerability + +Please **do not** open public GitHub issues for security vulnerabilities. + +Report privately via one of: + +- **GitHub Security Advisories** — Security tab → "Report a vulnerability" (preferred) +- **Email** — open.source@ribose.com + +We acknowledge reports within 72 hours and aim to ship a fix within 30 days for critical issues. Coordinated disclosure is supported. + +## Disclosure + +Public disclosure happens after a fix is released, on a timeline agreed with the reporter. From 83b1b3c9840f70a8915ffb36e0f307bd8ceb181e Mon Sep 17 00:00:00 2001 From: Ronald Tse Date: Wed, 29 Jul 2026 12:25:34 +0800 Subject: [PATCH 3/8] chore(python): add pyproject.toml + ruff config; autofix imports MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - pyproject.toml (PEP 621): name, version, license, requires-python - tool.ruff: conservative starter (E/F/W/I/UP); ignore E501/E402/E741 - tool.pytest config ready for future tests - 165 import-sort / unused-import fixes auto-applied - 116 remaining violations are research-code semantics (F811 dup defs in trainer.py, etc.) — manual review needed, not in this PR Refs: TODO.complete/08-ruff-rababa-python.md --- python/arabic/config_manager.py | 11 ++--- python/arabic/convert_torch_model_to_onnx.py | 7 +-- python/arabic/dataset.py | 9 +--- python/arabic/diacritize.py | 4 +- python/arabic/diacritizer.py | 15 +++---- python/arabic/models/baseline.py | 5 ++- python/arabic/models/cbhg.py | 5 +-- python/arabic/models/seq2seq.py | 9 ++-- python/arabic/models/tacotron_based.py | 6 ++- python/arabic/modules/attention.py | 7 ++- python/arabic/modules/layers.py | 10 ++--- python/arabic/modules/tacotron_modules.py | 17 ++++--- python/arabic/setup.py | 2 +- python/arabic/test.py | 3 +- python/arabic/tester.py | 10 ++--- python/arabic/train.py | 5 +-- python/arabic/trainer.py | 20 ++++----- python/arabic/util/learning_rates.py | 3 +- .../reconcile_original_plus_diacritized.py | 5 +-- python/arabic/util/text_cleaners.py | 4 +- python/arabic/util/text_encoders.py | 4 +- python/arabic/util/utils.py | 8 ++-- python/hebrew/config_manager.py | 7 +-- python/hebrew/convert_torch_model_to_onnx.py | 9 +--- python/hebrew/dataset.py | 10 +---- python/hebrew/diacritize.py | 4 +- python/hebrew/diacritizer.py | 14 ++---- python/hebrew/models/baseline.py | 5 ++- python/hebrew/models/cbhg.py | 5 +-- python/hebrew/models/seq2seq.py | 9 ++-- python/hebrew/models/tacotron_based.py | 6 ++- python/hebrew/modules/attention.py | 7 ++- python/hebrew/modules/layers.py | 10 ++--- python/hebrew/modules/tacotron_modules.py | 17 ++++--- python/hebrew/run_experiments_wandb.py | 32 ++++++-------- python/hebrew/setup.py | 2 +- python/hebrew/test.py | 3 +- python/hebrew/tester.py | 14 ++---- python/hebrew/train.py | 6 +-- python/hebrew/trainer.py | 37 ++++++---------- python/hebrew/util/learning_rates.py | 3 +- python/hebrew/util/nakdimon_dataset.py | 3 +- python/hebrew/util/nakdimon_hebrew_model.py | 11 ++--- python/hebrew/util/nakdimon_metrics.py | 5 +-- python/hebrew/util/nakdimon_utils.py | 8 ++-- python/hebrew/util/text_encoders.py | 2 - python/hebrew/util/utils.py | 8 ++-- python/pyproject.toml | 44 +++++++++++++++++++ 48 files changed, 202 insertions(+), 248 deletions(-) create mode 100644 python/pyproject.toml diff --git a/python/arabic/config_manager.py b/python/arabic/config_manager.py index 2a486c7..a735a3e 100644 --- a/python/arabic/config_manager.py +++ b/python/arabic/config_manager.py @@ -1,17 +1,14 @@ -from enum import Enum import os -from pathlib import Path import shutil import subprocess +from enum import Enum +from pathlib import Path from typing import Any, Dict import ruamel.yaml import torch - from models.baseline import BaseLineModel from models.cbhg import CBHGModel - - from options import AttentionType, LossType, OptimizerType from util.text_encoders import ( ArabicEncoderWithStartSymbol, @@ -188,9 +185,9 @@ def load_model(self, model_path: str = None): return model, 1 else: last_model_path = model_path - + saved_model = torch.load(last_model_path) if torch.cuda.is_available() else torch.load(last_model_path, map_location=torch.device('cpu')) - + out = model.load_state_dict(saved_model["model_state_dict"]) # print(out) check... global_step = saved_model["global_step"] + 1 diff --git a/python/arabic/convert_torch_model_to_onnx.py b/python/arabic/convert_torch_model_to_onnx.py index 48f7296..0b86a5d 100644 --- a/python/arabic/convert_torch_model_to_onnx.py +++ b/python/arabic/convert_torch_model_to_onnx.py @@ -1,11 +1,9 @@ -import torch -import pickle import numpy as np +import torch import yaml from diacritizer import Diacritizer - """ Key Params: max_len: @@ -49,10 +47,9 @@ Load ONNX libs and export models into onnx """ -import torch import onnx import onnxruntime - +import torch # export model torch.onnx.export( diff --git a/python/arabic/dataset.py b/python/arabic/dataset.py index 3b2e54d..21098ce 100644 --- a/python/arabic/dataset.py +++ b/python/arabic/dataset.py @@ -4,16 +4,11 @@ import os -import util.text_cleaners as cleaners import pandas as pd import torch -import random -import warnings -from diacritization_evaluation import util - -from torch.utils.data import DataLoader, Dataset - +import util.text_cleaners as cleaners from config_manager import ConfigManager +from torch.utils.data import DataLoader, Dataset class DiacritizationDataset(Dataset): diff --git a/python/arabic/diacritize.py b/python/arabic/diacritize.py index 30f1222..f92f521 100644 --- a/python/arabic/diacritize.py +++ b/python/arabic/diacritize.py @@ -1,11 +1,9 @@ import argparse -from diacritizer import Diacritizer -from itertools import repeat import random import numpy as np import torch - +from diacritizer import Diacritizer SEED = 1234 random.seed(SEED) diff --git a/python/arabic/diacritizer.py b/python/arabic/diacritizer.py index cd2606d..c468835 100644 --- a/python/arabic/diacritizer.py +++ b/python/arabic/diacritizer.py @@ -1,15 +1,12 @@ -from typing import Dict -import torch import warnings -import tqdm + import pandas as pd -import numpy as np -from config_manager import ConfigManager -from dataset import (DiacritizationDataset, - collate_fn) -from torch.utils.data import (DataLoader, - Dataset) +import torch +import tqdm import util.reconcile_original_plus_diacritized as reconcile +from config_manager import ConfigManager +from dataset import DiacritizationDataset, collate_fn +from torch.utils.data import DataLoader class Diacritizer: diff --git a/python/arabic/models/baseline.py b/python/arabic/models/baseline.py index 690af57..af78120 100644 --- a/python/arabic/models/baseline.py +++ b/python/arabic/models/baseline.py @@ -1,6 +1,7 @@ from typing import List -from torch import nn + import torch +from torch import nn class BaseLineModel(nn.Module): @@ -12,7 +13,7 @@ def __init__( layers_units: List[int] = [256, 256, 256], use_batch_norm: bool = False, ): - super(BaseLineModel, self).__init__() + super().__init__() self.targ_vocab_size = targ_vocab_size self.embedding = nn.Embedding(inp_vocab_size, embedding_dim) diff --git a/python/arabic/models/cbhg.py b/python/arabic/models/cbhg.py index 927a02d..b2263a5 100644 --- a/python/arabic/models/cbhg.py +++ b/python/arabic/models/cbhg.py @@ -3,10 +3,9 @@ """ from typing import List, Optional -from torch import nn import torch - from modules.tacotron_modules import CBHG, Prenet +from torch import nn class CBHGModel(nn.Module): @@ -41,7 +40,7 @@ def __init__( post_cbhg_layers_units: List[int] = [256, 256], post_cbhg_use_batch_norm: bool = True ): - super(CBHGModel, self).__init__() + super().__init__() self.use_prenet = use_prenet self.embedding = nn.Embedding(inp_vocab_size, embedding_dim) if self.use_prenet: diff --git a/python/arabic/models/seq2seq.py b/python/arabic/models/seq2seq.py index 5bef527..2e1fc37 100644 --- a/python/arabic/models/seq2seq.py +++ b/python/arabic/models/seq2seq.py @@ -1,14 +1,11 @@ -from typing import List from typing import List, Optional import torch -from torch import nn -from torch.autograd import Variable - from modules.attention import AttentionWrapper -from modules.layers import ConvNorm -from modules.tacotron_modules import CBHG, Prenet +from modules.tacotron_modules import Prenet from options import AttentionType +from torch import nn +from torch.autograd import Variable from util.utils import get_mask_from_lengths diff --git a/python/arabic/models/tacotron_based.py b/python/arabic/models/tacotron_based.py index 3feb034..3c02dc8 100644 --- a/python/arabic/models/tacotron_based.py +++ b/python/arabic/models/tacotron_based.py @@ -1,5 +1,7 @@ from typing import List -from models.seq2seq import Seq2Seq, Decoder as Seq2SeqDecoder + +from models.seq2seq import Decoder as Seq2SeqDecoder +from models.seq2seq import Seq2Seq from modules.tacotron_modules import CBHG, Prenet from torch import nn @@ -20,7 +22,7 @@ def __init__( cbhg_projections: List[int] = [128, 128], padding_idx: int = 0, ): - super(Encoder, self).__init__() + super().__init__() self.use_prenet = use_prenet self.embedding = nn.Embedding( diff --git a/python/arabic/modules/attention.py b/python/arabic/modules/attention.py index 06f84d2..f537806 100644 --- a/python/arabic/modules/attention.py +++ b/python/arabic/modules/attention.py @@ -1,15 +1,14 @@ from typing import Optional import torch -from torch import nn import torch.nn.functional as F - from options import AttentionType +from torch import nn class BahdanauAttention(nn.Module): def __init__(self, dim): - super(BahdanauAttention, self).__init__() + super().__init__() self.query_layer = nn.Linear(dim, dim, bias=False) self.tanh = nn.Tanh() self.v = nn.Linear(dim, 1, bias=False) @@ -35,7 +34,7 @@ def forward(self, query: torch.Tensor, keys: torch.Tensor): class LocationSensitive(nn.Module): def __init__(self, dim): - super(LocationSensitive, self).__init__() + super().__init__() self.query_layer = nn.Linear(dim, dim, bias=False) self.v = nn.Linear(dim, 1, bias=True) self.location_layer = nn.Linear(32, dim, bias=False) diff --git a/python/arabic/modules/layers.py b/python/arabic/modules/layers.py index e2905bc..e135522 100644 --- a/python/arabic/modules/layers.py +++ b/python/arabic/modules/layers.py @@ -1,9 +1,9 @@ -import torch -from torch import nn from copy import deepcopy - from typing import Any +import torch +from torch import nn + class BatchNormConv1d(nn.Module): """ @@ -19,7 +19,7 @@ def __init__( padding: int, activation: Any = None, ): - super(BatchNormConv1d, self).__init__() + super().__init__() self.conv1d = nn.Conv1d( in_dim, out_dim, @@ -39,7 +39,7 @@ def forward(self, x: Any): #x = self.activation(x) x = self.bn(x) - return x + return x class LinearNorm(torch.nn.Module): diff --git a/python/arabic/modules/tacotron_modules.py b/python/arabic/modules/tacotron_modules.py index d15db7f..875b924 100644 --- a/python/arabic/modules/tacotron_modules.py +++ b/python/arabic/modules/tacotron_modules.py @@ -1,13 +1,12 @@ """ Some custom modules that are used by the TTS model """ -from typing import List from copy import deepcopy +from typing import List import torch -from torch import nn - from modules.layers import BatchNormConv1d +from torch import nn class Prenet(nn.Module): @@ -100,7 +99,7 @@ def __init__( out_dim (int): the output size k (int): number of filters """ - super(CBHG, self).__init__() + super().__init__() self.in_dim = in_dim self.out_dim = out_dim @@ -128,9 +127,9 @@ def __init__( padding=k // 2, activation=self.relu, ) - + self.trafo = deepcopy(self.trafo_test) - + self.max_pool1d = nn.MaxPool1d(kernel_size=2, stride=1, padding=1) in_sizes = [K * in_dim] + projections[:-1] @@ -167,7 +166,7 @@ def forward(self, inputs, input_lengths=None): # (B, T_in, in_dim) # Back to the original shape x = x.transpose(1, 2) - + if x.size(-1) != self.in_dim: x = self.pre_highway(x) @@ -175,7 +174,7 @@ def forward(self, inputs, input_lengths=None): x += inputs for highway in self.highways: x = highway(x) - + if input_lengths is not None: x = nn.utils.rnn.pack_padded_sequence(x, input_lengths, batch_first=True) @@ -185,5 +184,5 @@ def forward(self, inputs, input_lengths=None): if input_lengths is not None: outputs, _ = nn.utils.rnn.pad_packed_sequence(outputs, batch_first=True) - + return outputs diff --git a/python/arabic/setup.py b/python/arabic/setup.py index 88bea0a..f3e3094 100644 --- a/python/arabic/setup.py +++ b/python/arabic/setup.py @@ -3,7 +3,7 @@ import setuptools -with open("README.adoc", "r", encoding="utf-8") as fh: +with open("README.adoc", encoding="utf-8") as fh: LONG_DESCRIPTION = fh.read() PKG_VERSION = "0.1.0" diff --git a/python/arabic/test.py b/python/arabic/test.py index d98834b..c5d4bde 100644 --- a/python/arabic/test.py +++ b/python/arabic/test.py @@ -1,10 +1,9 @@ import argparse import random -from tester import DiacritizationTester import numpy as np import torch - +from tester import DiacritizationTester SEED = 1234 random.seed(SEED) diff --git a/python/arabic/tester.py b/python/arabic/tester.py index 58eb282..9b975be 100644 --- a/python/arabic/tester.py +++ b/python/arabic/tester.py @@ -1,13 +1,9 @@ -from config_manager import ConfigManager -import os -import torch -from typing import Dict +import torch +from config_manager import ConfigManager +from dataset import load_iterators from torch import nn -from tqdm import tqdm from tqdm import trange - -from dataset import load_iterators from trainer import GeneralTrainer diff --git a/python/arabic/train.py b/python/arabic/train.py index 5dba6d8..811640f 100644 --- a/python/arabic/train.py +++ b/python/arabic/train.py @@ -4,10 +4,7 @@ import numpy as np import torch - -from trainer import ( - CBHGTrainer -) +from trainer import CBHGTrainer SEED = 1234 random.seed(SEED) diff --git a/python/arabic/trainer.py b/python/arabic/trainer.py index ac88969..beccd92 100644 --- a/python/arabic/trainer.py +++ b/python/arabic/trainer.py @@ -1,25 +1,21 @@ import os from typing import Dict -from diacritization_evaluation import der, wer import torch -from torch import nn -from torch import optim -from torch.cuda.amp import autocast -from torch.utils.tensorboard.writer import SummaryWriter -from tqdm import tqdm -from tqdm import trange - from config_manager import ConfigManager from dataset import load_iterators +from diacritization_evaluation import der, wer from diacritizer import Diacritizer -from util.learning_rates import LearningRateDecay from options import OptimizerType +from torch import nn, optim +from torch.cuda.amp import autocast +from torch.utils.tensorboard.writer import SummaryWriter +from tqdm import trange +from util.learning_rates import LearningRateDecay from util.utils import ( categorical_accuracy, count_parameters, initialize_weights, - plot_alignment, repeater, ) @@ -163,9 +159,9 @@ def evaluate_with_error_rates(self, iterator, tqdm): tqdm.update() summary_texts = [] - orig_path = os.path.join(self.config_manager.prediction_dir, f"original.txt") + orig_path = os.path.join(self.config_manager.prediction_dir, "original.txt") predicted_path = os.path.join( - self.config_manager.prediction_dir, f"predicted.txt" + self.config_manager.prediction_dir, "predicted.txt" ) with open(orig_path, "w", encoding="utf8") as file: diff --git a/python/arabic/util/learning_rates.py b/python/arabic/util/learning_rates.py index dd3325b..28e4fae 100644 --- a/python/arabic/util/learning_rates.py +++ b/python/arabic/util/learning_rates.py @@ -1,6 +1,7 @@ -import numpy as np import math +import numpy as np + class LearningRateDecay: def __init__(self, lr=0.002, warmup_steps=4000.0) -> None: diff --git a/python/arabic/util/reconcile_original_plus_diacritized.py b/python/arabic/util/reconcile_original_plus_diacritized.py index 2eee597..aba796f 100644 --- a/python/arabic/util/reconcile_original_plus_diacritized.py +++ b/python/arabic/util/reconcile_original_plus_diacritized.py @@ -1,5 +1,4 @@ -from util.constants import HARAQAT, ARAB_CHARS - +from util.constants import ARAB_CHARS, HARAQAT """ ################## @@ -65,7 +64,7 @@ def reconcile_strings(str_original, str_diacritized): """ # we model the strings as dict d_original = dict((i,c) for i,c in - enumerate(list([c for c in str_original if not c in HARAQAT]))) + enumerate(list([c for c in str_original if c not in HARAQAT]))) d_diacritized = dict((i,c) for i,c in enumerate(list(str_diacritized))) # matching positions diff --git a/python/arabic/util/text_cleaners.py b/python/arabic/util/text_cleaners.py index ead9783..b779d78 100644 --- a/python/arabic/util/text_cleaners.py +++ b/python/arabic/util/text_cleaners.py @@ -1,6 +1,6 @@ import re -from util.constants import VALID_ARABIC, BASIC_HARAQAT, ALL_POSSIBLE_HARAQAT -from diacritization_evaluation import util + +from util.constants import ALL_POSSIBLE_HARAQAT, BASIC_HARAQAT, VALID_ARABIC _whitespace_re = re.compile(r"\s+") diff --git a/python/arabic/util/text_encoders.py b/python/arabic/util/text_encoders.py index 5a5a0c1..c9113d1 100644 --- a/python/arabic/util/text_encoders.py +++ b/python/arabic/util/text_encoders.py @@ -1,7 +1,9 @@ -from util import text_cleaners from typing import Dict, List, Optional + from util.constants import ALL_POSSIBLE_HARAQAT +from util import text_cleaners + class TextEncoder: pad = "P" diff --git a/python/arabic/util/utils.py b/python/arabic/util/utils.py index 290d848..0726731 100644 --- a/python/arabic/util/utils.py +++ b/python/arabic/util/utils.py @@ -1,13 +1,13 @@ import os +from dataclasses import dataclass +from itertools import repeat from typing import Any import matplotlib.pyplot as plt +import numpy as np import torch from torch import nn -from itertools import repeat from util.decorators import ignore_exception -from dataclasses import dataclass -import numpy as np @dataclass @@ -199,7 +199,7 @@ def categorical_accuracy(preds, y, tag_pad_idx, device="cuda"): max_preds = preds.argmax( dim=1, keepdim=True ) # get the index of the max probability - non_pad_elements = torch.nonzero((y != tag_pad_idx)) + non_pad_elements = torch.nonzero(y != tag_pad_idx) correct = max_preds[non_pad_elements].squeeze(1).eq(y[non_pad_elements]) return correct.sum() / torch.FloatTensor([y[non_pad_elements].shape[0]]).to(device) diff --git a/python/hebrew/config_manager.py b/python/hebrew/config_manager.py index a16009c..2c8f94f 100644 --- a/python/hebrew/config_manager.py +++ b/python/hebrew/config_manager.py @@ -1,17 +1,14 @@ -from enum import Enum import os -from pathlib import Path import shutil import subprocess +from enum import Enum +from pathlib import Path from typing import Any, Dict import ruamel.yaml import torch - from models.baseline import BaseLineModel from models.cbhg import CBHGModel - - from options import AttentionType, LossType, OptimizerType from util.text_encoders import ( TextEncoder, diff --git a/python/hebrew/convert_torch_model_to_onnx.py b/python/hebrew/convert_torch_model_to_onnx.py index 1ece73f..56fe236 100644 --- a/python/hebrew/convert_torch_model_to_onnx.py +++ b/python/hebrew/convert_torch_model_to_onnx.py @@ -1,16 +1,11 @@ -import torch -import pickle import random -import torch +import numpy as np import onnx import onnxruntime - -import numpy as np - +import torch from diacritizer import Diacritizer - """ Key Params: max_len: diff --git a/python/hebrew/dataset.py b/python/hebrew/dataset.py index 5bfc78a..a7a0142 100644 --- a/python/hebrew/dataset.py +++ b/python/hebrew/dataset.py @@ -3,19 +3,13 @@ """ import os -import numpy as np -import pandas as pd -import torch -import random -import warnings - -from torch.utils.data import DataLoader, Dataset from config_manager import ConfigManager +from torch.utils.data import DataLoader, Dataset from util import nakdimon_dataset -from util import nakdimon_utils as utils from util import nakdimon_hebrew_model as hebrew +from util import nakdimon_utils as utils class DiacritizationDataset(Dataset): diff --git a/python/hebrew/diacritize.py b/python/hebrew/diacritize.py index fcda882..c628850 100644 --- a/python/hebrew/diacritize.py +++ b/python/hebrew/diacritize.py @@ -1,11 +1,9 @@ import argparse -from diacritizer import Diacritizer -from itertools import repeat import random import numpy as np import torch - +from diacritizer import Diacritizer SEED = 1234 random.seed(SEED) diff --git a/python/hebrew/diacritizer.py b/python/hebrew/diacritizer.py index 7cdfe91..08d5e82 100644 --- a/python/hebrew/diacritizer.py +++ b/python/hebrew/diacritizer.py @@ -1,17 +1,11 @@ -from typing import Dict import torch -import warnings import tqdm -import pandas as pd -import numpy as np - from config_manager import ConfigManager from dataset import DiacritizationDataset, collate_fn -from torch.utils.data import DataLoader, Dataset +from torch.utils.data import DataLoader from util import nakdimon_dataset # as dataset from util import nakdimon_hebrew_model as hebrew -from util import nakdimon_metrics from util import nakdimon_utils as utils @@ -112,13 +106,13 @@ def diacritize_data_iterator(self, data_iterator, criterion=None): raw_data.append(data_batch) preds, loss = self.predict_batch(data_batch, criterion) dia_data.append(preds) - if not criterion is None: + if criterion is not None: losses.append(loss) raw_data = nakdimon_dataset.Data.concatenate(raw_data) dia_data = nakdimon_dataset.Data.concatenate(dia_data) - if not criterion is None: + if criterion is not None: losses = ( [l[0] for l in losses], [l[1] for l in losses], @@ -137,7 +131,7 @@ def process_dim(dim): niqqud, dagesh, sin = self.model(data_batch.normalized) losses = None - if not criterion is None: + if criterion is not None: losses = [ criterion(process_dim(niqqud), data_batch.niqqud.long()), diff --git a/python/hebrew/models/baseline.py b/python/hebrew/models/baseline.py index 690af57..af78120 100644 --- a/python/hebrew/models/baseline.py +++ b/python/hebrew/models/baseline.py @@ -1,6 +1,7 @@ from typing import List -from torch import nn + import torch +from torch import nn class BaseLineModel(nn.Module): @@ -12,7 +13,7 @@ def __init__( layers_units: List[int] = [256, 256, 256], use_batch_norm: bool = False, ): - super(BaseLineModel, self).__init__() + super().__init__() self.targ_vocab_size = targ_vocab_size self.embedding = nn.Embedding(inp_vocab_size, embedding_dim) diff --git a/python/hebrew/models/cbhg.py b/python/hebrew/models/cbhg.py index e0d08cf..54a3137 100644 --- a/python/hebrew/models/cbhg.py +++ b/python/hebrew/models/cbhg.py @@ -3,10 +3,9 @@ """ from typing import List, Optional -from torch import nn import torch - from modules.tacotron_modules import CBHG, Prenet +from torch import nn class CBHGModel(nn.Module): @@ -45,7 +44,7 @@ def __init__( post_cbhg_layers_units: List[int] = [256, 256], post_cbhg_use_batch_norm: bool = True ): - super(CBHGModel, self).__init__() + super().__init__() self.use_prenet = use_prenet self.embedding = nn.Embedding(inp_vocab_size, embedding_dim) diff --git a/python/hebrew/models/seq2seq.py b/python/hebrew/models/seq2seq.py index 5bef527..2e1fc37 100644 --- a/python/hebrew/models/seq2seq.py +++ b/python/hebrew/models/seq2seq.py @@ -1,14 +1,11 @@ -from typing import List from typing import List, Optional import torch -from torch import nn -from torch.autograd import Variable - from modules.attention import AttentionWrapper -from modules.layers import ConvNorm -from modules.tacotron_modules import CBHG, Prenet +from modules.tacotron_modules import Prenet from options import AttentionType +from torch import nn +from torch.autograd import Variable from util.utils import get_mask_from_lengths diff --git a/python/hebrew/models/tacotron_based.py b/python/hebrew/models/tacotron_based.py index 3feb034..3c02dc8 100644 --- a/python/hebrew/models/tacotron_based.py +++ b/python/hebrew/models/tacotron_based.py @@ -1,5 +1,7 @@ from typing import List -from models.seq2seq import Seq2Seq, Decoder as Seq2SeqDecoder + +from models.seq2seq import Decoder as Seq2SeqDecoder +from models.seq2seq import Seq2Seq from modules.tacotron_modules import CBHG, Prenet from torch import nn @@ -20,7 +22,7 @@ def __init__( cbhg_projections: List[int] = [128, 128], padding_idx: int = 0, ): - super(Encoder, self).__init__() + super().__init__() self.use_prenet = use_prenet self.embedding = nn.Embedding( diff --git a/python/hebrew/modules/attention.py b/python/hebrew/modules/attention.py index 06f84d2..f537806 100644 --- a/python/hebrew/modules/attention.py +++ b/python/hebrew/modules/attention.py @@ -1,15 +1,14 @@ from typing import Optional import torch -from torch import nn import torch.nn.functional as F - from options import AttentionType +from torch import nn class BahdanauAttention(nn.Module): def __init__(self, dim): - super(BahdanauAttention, self).__init__() + super().__init__() self.query_layer = nn.Linear(dim, dim, bias=False) self.tanh = nn.Tanh() self.v = nn.Linear(dim, 1, bias=False) @@ -35,7 +34,7 @@ def forward(self, query: torch.Tensor, keys: torch.Tensor): class LocationSensitive(nn.Module): def __init__(self, dim): - super(LocationSensitive, self).__init__() + super().__init__() self.query_layer = nn.Linear(dim, dim, bias=False) self.v = nn.Linear(dim, 1, bias=True) self.location_layer = nn.Linear(32, dim, bias=False) diff --git a/python/hebrew/modules/layers.py b/python/hebrew/modules/layers.py index e2905bc..e135522 100644 --- a/python/hebrew/modules/layers.py +++ b/python/hebrew/modules/layers.py @@ -1,9 +1,9 @@ -import torch -from torch import nn from copy import deepcopy - from typing import Any +import torch +from torch import nn + class BatchNormConv1d(nn.Module): """ @@ -19,7 +19,7 @@ def __init__( padding: int, activation: Any = None, ): - super(BatchNormConv1d, self).__init__() + super().__init__() self.conv1d = nn.Conv1d( in_dim, out_dim, @@ -39,7 +39,7 @@ def forward(self, x: Any): #x = self.activation(x) x = self.bn(x) - return x + return x class LinearNorm(torch.nn.Module): diff --git a/python/hebrew/modules/tacotron_modules.py b/python/hebrew/modules/tacotron_modules.py index d15db7f..875b924 100644 --- a/python/hebrew/modules/tacotron_modules.py +++ b/python/hebrew/modules/tacotron_modules.py @@ -1,13 +1,12 @@ """ Some custom modules that are used by the TTS model """ -from typing import List from copy import deepcopy +from typing import List import torch -from torch import nn - from modules.layers import BatchNormConv1d +from torch import nn class Prenet(nn.Module): @@ -100,7 +99,7 @@ def __init__( out_dim (int): the output size k (int): number of filters """ - super(CBHG, self).__init__() + super().__init__() self.in_dim = in_dim self.out_dim = out_dim @@ -128,9 +127,9 @@ def __init__( padding=k // 2, activation=self.relu, ) - + self.trafo = deepcopy(self.trafo_test) - + self.max_pool1d = nn.MaxPool1d(kernel_size=2, stride=1, padding=1) in_sizes = [K * in_dim] + projections[:-1] @@ -167,7 +166,7 @@ def forward(self, inputs, input_lengths=None): # (B, T_in, in_dim) # Back to the original shape x = x.transpose(1, 2) - + if x.size(-1) != self.in_dim: x = self.pre_highway(x) @@ -175,7 +174,7 @@ def forward(self, inputs, input_lengths=None): x += inputs for highway in self.highways: x = highway(x) - + if input_lengths is not None: x = nn.utils.rnn.pack_padded_sequence(x, input_lengths, batch_first=True) @@ -185,5 +184,5 @@ def forward(self, inputs, input_lengths=None): if input_lengths is not None: outputs, _ = nn.utils.rnn.pad_packed_sequence(outputs, batch_first=True) - + return outputs diff --git a/python/hebrew/run_experiments_wandb.py b/python/hebrew/run_experiments_wandb.py index 2ed0379..0e08df8 100644 --- a/python/hebrew/run_experiments_wandb.py +++ b/python/hebrew/run_experiments_wandb.py @@ -3,16 +3,12 @@ import random import numpy as np -import torch # import ruamel.yaml import ruamel.yaml as yaml - +import torch import wandb - -from trainer import ( - CBHGTrainer -) +from trainer import CBHGTrainer SEED = 1234 random.seed(SEED) @@ -38,7 +34,7 @@ def train_parser(): parser = train_parser() args = parser.parse_args() - + # Define Experiments using Wandb sweep_config = { # search method @@ -46,7 +42,7 @@ def train_parser(): # metric and objective 'metric': { 'name': 'dec', - 'goal': 'maximize' #'minimize' + 'goal': 'maximize' #'minimize' }, # define search parameters 'parameters': { @@ -69,7 +65,7 @@ def train_parser(): 'post_cbhg_layers_units': { 'values': [[256, 256]] }, - + 'optimizer': { 'values': ['Adam', 'SGD'] }, @@ -78,26 +74,26 @@ def train_parser(): }, 'prenet_sizes': { 'values': [[512, 256]] - } + } } } -# train code, with the search preprocessing logic +# train code, with the search preprocessing logic def train(): with open('config/train.yml', "rb") as model_yaml: config = yaml.load(model_yaml) - + # load default config - config_defaults = config + config_defaults = config wandb.init(config=config_defaults) # , magic=True) config_wandb = wandb.config - + # overwrite initial config - config = { **config, + config = { **config, **config_wandb } - + tmp_config_path = 'config/sweep_tmp.yml' with open(tmp_config_path, 'w') as yaml_file: yaml.dump(config, yaml_file, default_flow_style=False) @@ -108,9 +104,9 @@ def train(): raise ValueError("The model kind is not supported") trainer.run(config_wandb) - - + + ################################## # MAIN # ################################## diff --git a/python/hebrew/setup.py b/python/hebrew/setup.py index 88bea0a..f3e3094 100644 --- a/python/hebrew/setup.py +++ b/python/hebrew/setup.py @@ -3,7 +3,7 @@ import setuptools -with open("README.adoc", "r", encoding="utf-8") as fh: +with open("README.adoc", encoding="utf-8") as fh: LONG_DESCRIPTION = fh.read() PKG_VERSION = "0.1.0" diff --git a/python/hebrew/test.py b/python/hebrew/test.py index d98834b..c5d4bde 100644 --- a/python/hebrew/test.py +++ b/python/hebrew/test.py @@ -1,10 +1,9 @@ import argparse import random -from tester import DiacritizationTester import numpy as np import torch - +from tester import DiacritizationTester SEED = 1234 random.seed(SEED) diff --git a/python/hebrew/tester.py b/python/hebrew/tester.py index be80f66..2e8366a 100644 --- a/python/hebrew/tester.py +++ b/python/hebrew/tester.py @@ -1,19 +1,11 @@ -from config_manager import ConfigManager -import os -import torch -from typing import Dict +import torch +from config_manager import ConfigManager +from dataset import load_iterators from torch import nn -from tqdm import tqdm from tqdm import trange - -from dataset import load_iterators from trainer import GeneralTrainer -from util import nakdimon_dataset -from util import nakdimon_utils as utils -from util import nakdimon_hebrew_model as hebrew - class DiacritizationTester(GeneralTrainer): def __init__(self, config_path: str, model_kind: str) -> None: diff --git a/python/hebrew/train.py b/python/hebrew/train.py index 3e9ae93..811640f 100644 --- a/python/hebrew/train.py +++ b/python/hebrew/train.py @@ -4,11 +4,7 @@ import numpy as np import torch -import wandb - -from trainer import ( - CBHGTrainer -) +from trainer import CBHGTrainer SEED = 1234 random.seed(SEED) diff --git a/python/hebrew/trainer.py b/python/hebrew/trainer.py index 16e4feb..a8edb79 100644 --- a/python/hebrew/trainer.py +++ b/python/hebrew/trainer.py @@ -1,35 +1,24 @@ import os -from typing import Dict import torch -from torch import nn -from torch import optim -from torch.cuda.amp import autocast -from torch.utils.tensorboard.writer import SummaryWriter -from tqdm import tqdm -from tqdm import trange -import numpy as np - +import wandb from config_manager import ConfigManager from dataset import load_iterators from diacritizer import Diacritizer -from util.learning_rates import LearningRateDecay from options import OptimizerType - +from torch import nn, optim +from torch.cuda.amp import autocast +from torch.utils.tensorboard.writer import SummaryWriter +from tqdm import trange +from util.learning_rates import LearningRateDecay from util.utils import ( - categorical_accuracy, count_parameters, # initialize_weights, # plot_alignment, repeater, ) -from util import nakdimon_dataset -from util import nakdimon_utils as utils -from util import nakdimon_hebrew_model as hebrew -from util import nakdimon_metrics - -import wandb +from util import nakdimon_dataset, nakdimon_metrics class Trainer: @@ -144,12 +133,12 @@ def evaluate_with_error_rates(self, iterator, tqdm): self.config_manager.config["test_file_name"], ) - orig_path = os.path.join(self.config_manager.prediction_dir, f"original.txt") + orig_path = os.path.join(self.config_manager.prediction_dir, "original.txt") predicts_path = os.path.join( - self.config_manager.prediction_dir, f"predicted.txt" + self.config_manager.prediction_dir, "predicted.txt" ) - f = open(test_path, "r") + f = open(test_path) all_orig = f.readlines() f.close() @@ -249,7 +238,7 @@ def run(self, config_wandb=None): validation_iterator, tqdm_error_rates ) - if not config_wandb is None: + if config_wandb is not None: wandb.log({**d_scores, **scores}) print("scores:: ", scores) @@ -332,10 +321,10 @@ def load_model(self, model_path: str = None, load_optimizer: bool = True): ) = self.config_manager.load_model(model_path, load_optimizer) self.model = saved_model - if not optimizer_states_dict is None: + if optimizer_states_dict is not None: self.optimizer.load_state_dict(optimizer_states_dict) - self.global_step = global_step if not global_step is None else 0 + self.global_step = global_step if global_step is not None else 0 def get_optimizer(self): if self.config["optimizer"] == OptimizerType.Adam: diff --git a/python/hebrew/util/learning_rates.py b/python/hebrew/util/learning_rates.py index dd3325b..28e4fae 100644 --- a/python/hebrew/util/learning_rates.py +++ b/python/hebrew/util/learning_rates.py @@ -1,6 +1,7 @@ -import numpy as np import math +import numpy as np + class LearningRateDecay: def __init__(self, lr=0.002, warmup_steps=4000.0) -> None: diff --git a/python/hebrew/util/nakdimon_dataset.py b/python/hebrew/util/nakdimon_dataset.py index cfa1460..7ae42ed 100644 --- a/python/hebrew/util/nakdimon_dataset.py +++ b/python/hebrew/util/nakdimon_dataset.py @@ -1,5 +1,6 @@ -from typing import Tuple, List import random +from typing import List, Tuple + import numpy as np import torch diff --git a/python/hebrew/util/nakdimon_hebrew_model.py b/python/hebrew/util/nakdimon_hebrew_model.py index abd5121..2b7a971 100644 --- a/python/hebrew/util/nakdimon_hebrew_model.py +++ b/python/hebrew/util/nakdimon_hebrew_model.py @@ -1,12 +1,7 @@ -import itertools -from collections import defaultdict, Counter -from typing import NamedTuple, Iterator, Iterable, List, Tuple +from collections.abc import Iterable, Iterator from functools import lru_cache -import re - -from util import nakdimon_utils as utils - +from typing import List, NamedTuple # "rafe" denotes a letter to which it would have been valid to add a diacritic of some category # but instead it is decided not to. This makes the metrics less biased. @@ -236,7 +231,7 @@ def __bool__(self): def __eq__(self, other): return self.items == other.items - @lru_cache() + @lru_cache def to_undotted(self): return ''.join(str(c.letter) for c in self.items) diff --git a/python/hebrew/util/nakdimon_metrics.py b/python/hebrew/util/nakdimon_metrics.py index 35abbc8..f94c241 100644 --- a/python/hebrew/util/nakdimon_metrics.py +++ b/python/hebrew/util/nakdimon_metrics.py @@ -1,12 +1,9 @@ -from typing import Tuple, List from pathlib import Path - -import numpy as np +from typing import List, Tuple from util import nakdimon_hebrew_model as hebrew - basepath = Path('tests/validation/expected') diff --git a/python/hebrew/util/nakdimon_utils.py b/python/hebrew/util/nakdimon_utils.py index f19dfb8..da750f3 100644 --- a/python/hebrew/util/nakdimon_utils.py +++ b/python/hebrew/util/nakdimon_utils.py @@ -1,9 +1,9 @@ -from typing import List, Iterable - -import sys import contextlib import os +import sys +from collections.abc import Iterable +from typing import List import numpy as np @@ -20,7 +20,7 @@ def iterate_files(base_paths: Iterable[str]) -> List[str]: def read_file(filename): - with open(filename, 'r', encoding='utf-8') as f: + with open(filename, encoding='utf-8') as f: return f.read() diff --git a/python/hebrew/util/text_encoders.py b/python/hebrew/util/text_encoders.py index 2d5687b..3602bec 100644 --- a/python/hebrew/util/text_encoders.py +++ b/python/hebrew/util/text_encoders.py @@ -1,9 +1,7 @@ -from typing import Dict, List, Optional # from util import text_cleaners from util import nakdimon_dataset as dataset -from util import nakdimon_hebrew_model as hebrew class TextEncoder: diff --git a/python/hebrew/util/utils.py b/python/hebrew/util/utils.py index 65c7721..d42469c 100644 --- a/python/hebrew/util/utils.py +++ b/python/hebrew/util/utils.py @@ -1,14 +1,14 @@ import os +from dataclasses import dataclass +from itertools import repeat from typing import Any import matplotlib.pyplot as plt +import numpy as np import torch from torch import nn -from itertools import repeat from util.decorators import ignore_exception -from dataclasses import dataclass -import numpy as np @dataclass @@ -200,7 +200,7 @@ def categorical_accuracy(preds, y, tag_pad_idx, device="cuda"): max_preds = preds.argmax( dim=1, keepdim=True ) # get the index of the max probability - non_pad_elements = torch.nonzero((y != tag_pad_idx)) + non_pad_elements = torch.nonzero(y != tag_pad_idx) correct = max_preds[non_pad_elements].squeeze(1).eq(y[non_pad_elements]) return correct.sum() / torch.FloatTensor([y[non_pad_elements].shape[0]]).to(device) diff --git a/python/pyproject.toml b/python/pyproject.toml new file mode 100644 index 0000000..5ce64d5 --- /dev/null +++ b/python/pyproject.toml @@ -0,0 +1,44 @@ +[project] +name = "rababa-python" +version = "0.1.1" +description = "Middle Eastern language diacritization (Arabic, Hebrew) — Interscript" +readme = "README.md" +requires-python = ">=3.9" +license = { text = "BSD-2-Clause" } +authors = [{ name = "Ribose Inc.", email = "open.source@ribose.com" }] + +# Runtime dependencies are pinned per-language under python/{arabic,hebrew}/requirements.txt +# because torch and onnxruntime are version-coupled to the trained model weights. +# Bumping them requires re-validating inference — see TODO.complete/19-torch-2x.md. +dependencies = [] + +[optional-dependencies] +arabic = [] +hebrew = [] + +[tool.ruff] +line-length = 100 +target-version = "py39" +extend-exclude = ["python/**/data", "python/**/log_dir", "python/**/__pycache__"] + +[tool.ruff.lint] +# Conservative starter set — expand as code is cleaned up +select = [ + "E", # pycodestyle errors + "F", # pyflakes + "W", # pycodestyle warnings + "I", # isort + "UP", # pyupgrade (within reason) +] +ignore = [ + "E501", # line too long — formatter handles this + "E402", # module-level import not at top — research scripts often configure paths first + "E741", # ambiguous variable names — common in math/ML code +] + +[tool.ruff.format] +quote-style = "double" + +[tool.pytest.ini_options] +testpaths = ["python"] +python_files = ["test_*.py", "*_test.py"] From 2c99fe92aab2c186375efe4289b4b9dc729f7af4 Mon Sep 17 00:00:00 2001 From: Ronald Tse Date: Wed, 29 Jul 2026 12:33:46 +0800 Subject: [PATCH 4/8] chore(python): add pyproject.toml + ruff config; autofix 245 violations MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - pyproject.toml (PEP 621): name, version, license, requires-python - tool.ruff: conservative starter (E/F/W/I/UP); ignore E501/E402/E741 - tool.pytest config ready for future tests - 245 fixes auto-applied (165 safe + 80 unsafe): PEP 585 annotations, deprecated imports, unused vars, yield-in-for, isort - ruff format applied to all 56 .py files - CI: new lint job (non-blocking) runs ruff check + format check - .gitignore: exclude python/{log_dir,data,models}/ training artifacts 36 violations remain (F821 false positives on tuple-unpack assigns, F811 dup defs in trainer.py, E722 bare-except, E721 type-compare) — manual review needed. Refs: TODO.complete/08-ruff-rababa-python.md --- .github/workflows/python-arabic.yml | 12 ++ .gitignore | 15 ++ python/arabic/config_manager.py | 46 ++--- python/arabic/convert_torch_model_to_onnx.py | 5 - python/arabic/dataset.py | 40 ++-- python/arabic/diacritize.py | 4 +- python/arabic/diacritizer.py | 56 +++--- python/arabic/models/baseline.py | 4 +- python/arabic/models/cbhg.py | 14 +- python/arabic/models/seq2seq.py | 31 ++-- python/arabic/models/tacotron_based.py | 10 +- python/arabic/modules/layers.py | 39 ++-- python/arabic/modules/tacotron_modules.py | 52 +++--- python/arabic/options.py | 1 + python/arabic/setup.py | 44 ++--- python/arabic/tester.py | 5 +- python/arabic/train.py | 3 +- python/arabic/trainer.py | 69 +++---- python/arabic/util/constants.py | 3 +- python/arabic/util/learning_rates.py | 18 +- .../reconcile_original_plus_diacritized.py | 62 +++---- python/arabic/util/text_cleaners.py | 21 ++- python/arabic/util/text_encoders.py | 54 +++--- python/arabic/util/utils.py | 26 +-- python/hebrew/config_manager.py | 30 +-- python/hebrew/convert_torch_model_to_onnx.py | 32 +--- python/hebrew/dataset.py | 24 +-- python/hebrew/diacritizer.py | 25 +-- python/hebrew/models/baseline.py | 4 +- python/hebrew/models/cbhg.py | 23 +-- python/hebrew/models/seq2seq.py | 31 ++-- python/hebrew/models/tacotron_based.py | 10 +- python/hebrew/modules/layers.py | 39 ++-- python/hebrew/modules/tacotron_modules.py | 52 +++--- python/hebrew/options.py | 1 + python/hebrew/run_experiments_wandb.py | 65 +++---- python/hebrew/setup.py | 44 ++--- python/hebrew/tester.py | 14 +- python/hebrew/train.py | 3 +- python/hebrew/trainer.py | 36 +--- python/hebrew/util/decorators.py | 1 - python/hebrew/util/learning_rates.py | 18 +- python/hebrew/util/nakdimon_dataset.py | 34 +--- python/hebrew/util/nakdimon_hebrew_model.py | 171 ++++++++++-------- python/hebrew/util/nakdimon_metrics.py | 100 ++++++---- python/hebrew/util/nakdimon_utils.py | 16 +- python/hebrew/util/text_encoders.py | 4 +- python/hebrew/util/utils.py | 27 +-- 48 files changed, 637 insertions(+), 801 deletions(-) diff --git a/.github/workflows/python-arabic.yml b/.github/workflows/python-arabic.yml index 46607f3..f0ac6c3 100644 --- a/.github/workflows/python-arabic.yml +++ b/.github/workflows/python-arabic.yml @@ -6,6 +6,18 @@ on: pull_request: jobs: + lint: + runs-on: ubuntu-latest + continue-on-error: true # 36 pre-existing violations; see TODO.complete/08-ruff-rababa-python.md + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-python@v7 + with: + python-version: "3.11" + - run: pip install ruff + - run: ruff check python/ + - run: ruff format --check python/ + infer: runs-on: ubuntu-latest strategy: diff --git a/.gitignore b/.gitignore index 116a66c..1d6c6f5 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,18 @@ Gemfile.lock .eggs __pycache__ + +# Python training artifacts — never commit +python/log_dir/ +python/data/ +python/models/ +python/__pycache__/ +**/__pycache__/ +*.pyc +*.pyo +*.onnx +*.pt + +# Editor +.idea/ +.vscode/ diff --git a/python/arabic/config_manager.py b/python/arabic/config_manager.py index a735a3e..275a58b 100644 --- a/python/arabic/config_manager.py +++ b/python/arabic/config_manager.py @@ -3,7 +3,7 @@ import subprocess from enum import Enum from pathlib import Path -from typing import Any, Dict +from typing import Any import ruamel.yaml import torch @@ -30,7 +30,7 @@ def __init__(self, config_path: str, model_kind: str): self.config_path = Path(config_path) self.model_kind = model_kind self.yaml = ruamel.yaml.YAML() - self.config: Dict[str, Any] = self._load_config() + self.config: dict[str, Any] = self._load_config() # self.git_hash = self._get_git_hash() self.session_name = ".".join( [ @@ -40,12 +40,8 @@ def __init__(self, config_path: str, model_kind: str): ] ) - self.data_dir = Path( - os.path.join(self.config["data_directory"], self.config["data_type"]) - ) - self.base_dir = Path( - os.path.join(self.config["log_directory"], self.session_name) - ) + self.data_dir = Path(os.path.join(self.config["data_directory"], self.config["data_type"])) + self.base_dir = Path(os.path.join(self.config["log_directory"], self.session_name)) self.log_dir = Path(os.path.join(self.base_dir, "logs")) self.prediction_dir = Path(os.path.join(self.base_dir, "predictions")) self.plot_dir = Path(os.path.join(self.base_dir, "plots")) @@ -65,25 +61,17 @@ def _load_config(self): @staticmethod def _get_git_hash(): try: - return ( - subprocess.check_output(["git", "describe", "--always"]) - .strip() - .decode() - ) + return subprocess.check_output(["git", "describe", "--always"]).strip().decode() except Exception as e: print(f"WARNING: could not retrieve git hash. {e}") def _check_hash(self): try: - git_hash = ( - subprocess.check_output(["git", "describe", "--always"]) - .strip() - .decode() - ) + git_hash = subprocess.check_output(["git", "describe", "--always"]).strip().decode() if self.config["git_hash"] != git_hash: print( f"""WARNING: git hash mismatch. Current: {git_hash}. - Config hash: {self.config['git_hash']}""" + Config hash: {self.config["git_hash"]}""" ) except Exception as e: print(f"WARNING: could not check git hash. {e}") @@ -99,9 +87,7 @@ def _print_dictionary(self, dictionary, recursion_level=0): recursion_level += 1 self._print_dictionary(dictionary[key], recursion_level) else: - self._print_dict_values( - dictionary[key], key_name=key, level=recursion_level - ) + self._print_dict_values(dictionary[key], key_name=key, level=recursion_level) def print_config(self): print("\nCONFIGURATION", self.session_name) @@ -186,9 +172,13 @@ def load_model(self, model_path: str = None): else: last_model_path = model_path - saved_model = torch.load(last_model_path) if torch.cuda.is_available() else torch.load(last_model_path, map_location=torch.device('cpu')) + saved_model = ( + torch.load(last_model_path) + if torch.cuda.is_available() + else torch.load(last_model_path, map_location=torch.device("cpu")) + ) - out = model.load_state_dict(saved_model["model_state_dict"]) + model.load_state_dict(saved_model["model_state_dict"]) # print(out) check... global_step = saved_model["global_step"] + 1 return model, global_step @@ -241,13 +231,9 @@ def get_text_encoder(self): if self.config["text_encoder"] == "BasicArabicEncoder": text_encoder = BasicArabicEncoder(cleaner_fn=self.config["text_cleaner"]) elif self.config["text_encoder"] == "ArabicEncoderWithStartSymbol": - text_encoder = ArabicEncoderWithStartSymbol( - cleaner_fn=self.config["text_cleaner"] - ) + text_encoder = ArabicEncoderWithStartSymbol(cleaner_fn=self.config["text_cleaner"]) else: - raise Exception( - f"the text encoder is not found {self.config['text_encoder']}" - ) + raise Exception(f"the text encoder is not found {self.config['text_encoder']}") return text_encoder diff --git a/python/arabic/convert_torch_model_to_onnx.py b/python/arabic/convert_torch_model_to_onnx.py index 0b86a5d..52f112a 100644 --- a/python/arabic/convert_torch_model_to_onnx.py +++ b/python/arabic/convert_torch_model_to_onnx.py @@ -1,4 +1,3 @@ - import numpy as np import torch import yaml @@ -151,7 +150,6 @@ print("***** Test MAX size :: Random Boolean vectors: *****") for test_run in range(3): - vec = [[random.randint(0, 1) for i in range(max_len)] for i in range(batch_size)] src = torch.Tensor(vec).long() lengths = torch.Tensor([max_len for i in range(batch_size)]).long() @@ -182,7 +180,6 @@ print("***** Test MAX size :: Random float, vectors within 0:16 *****") for test_run in range(3): - vec = [[random.randint(0, 17) for i in range(max_len)] for i in range(batch_size)] src = torch.Tensor(vec).long() torch_out = dia.model(src, lengths) @@ -209,7 +206,6 @@ print("***** Test Dynamical sizes :: Random Boolean vectors: *****") for l in [2, 10, 40, 100, 150]: - print("length:: ", l) vec = [[1 for i in range(l)] for i in range(batch_size)] # random.randint(0,1) @@ -242,7 +238,6 @@ print("***** Test Dynamical sizes :: Random float, vectors within 0:16 *****") for l in [2, 10, 40, 100, 150]: - vec = [[random.randint(0, 17) for i in range(l)] for i in range(batch_size)] src = torch.Tensor(vec).long() lengths = torch.Tensor([l for i in range(batch_size)]).long() diff --git a/python/arabic/dataset.py b/python/arabic/dataset.py index 21098ce..4d85263 100644 --- a/python/arabic/dataset.py +++ b/python/arabic/dataset.py @@ -33,13 +33,10 @@ def __getitem__(self, index): # Select sample id = self.list_ids[index] data_orig = self.data[id].strip() - text, inputs, diacritics = cleaners.extract_haraqat( - self.text_encoder.clean(data_orig)) + text, inputs, diacritics = cleaners.extract_haraqat(self.text_encoder.clean(data_orig)) - inputs = torch.Tensor( - self.text_encoder.input_to_sequence("".join(inputs))) - diacritics = torch.Tensor( - self.text_encoder.target_to_sequence(diacritics)) + inputs = torch.Tensor(self.text_encoder.input_to_sequence("".join(inputs))) + diacritics = torch.Tensor(self.text_encoder.target_to_sequence(diacritics)) return inputs, diacritics, data_orig @@ -94,24 +91,18 @@ def load_training_data(config_manager: ConfigManager, loader_parameters): ) # train_data = train_data[train_data[0] <= config_manager.config["max_len"]] - training_set = DiacritizationDataset( - config_manager, train_data.index, train_data - ) + training_set = DiacritizationDataset(config_manager, train_data.index, train_data) else: with open(path, encoding="utf8") as file: train_data = file.readlines() train_data = [ - text - for text in train_data - if len(text) <= config_manager.config["max_len"] + text for text in train_data if len(text) <= config_manager.config["max_len"] ] training_set = DiacritizationDataset( config_manager, [idx for idx in range(len(train_data))], train_data ) - train_iterator = DataLoader( - training_set, collate_fn=collate_fn, **loader_parameters - ) + train_iterator = DataLoader(training_set, collate_fn=collate_fn, **loader_parameters) print(f"Length of training iterator = {len(train_iterator)}") return train_iterator @@ -138,15 +129,12 @@ def load_test_data(config_manager: ConfigManager, loader_parameters): else: with open(path, encoding="utf8") as file: test_data = file.readlines() - test_data = [ - text for text in test_data if len(text) <= config_manager.config["max_len"] - ] + test_data = [text for text in test_data if len(text) <= config_manager.config["max_len"]] test_dataset = DiacritizationDataset( config_manager, [idx for idx in range(len(test_data))], test_data ) - test_iterator = DataLoader(test_dataset, collate_fn=collate_fn, - **loader_parameters) + test_iterator = DataLoader(test_dataset, collate_fn=collate_fn, **loader_parameters) print(f"Length of test iterator = {len(test_iterator)}") return test_iterator @@ -170,23 +158,17 @@ def load_validation_data(config_manager: ConfigManager, loader_parameters): ) # valid_data = valid_data[valid_data[0] <= config_manager.config["max_len"]] - valid_dataset = DiacritizationDataset( - config_manager, valid_data.index, valid_data - ) + valid_dataset = DiacritizationDataset(config_manager, valid_data.index, valid_data) else: with open(path, encoding="utf8") as file: valid_data = file.readlines() - valid_data = [ - text for text in valid_data if len(text) <= config_manager.config["max_len"] - ] + valid_data = [text for text in valid_data if len(text) <= config_manager.config["max_len"]] valid_dataset = DiacritizationDataset( config_manager, [idx for idx in range(len(valid_data))], valid_data ) - valid_iterator = DataLoader( - valid_dataset, collate_fn=collate_fn, **loader_parameters - ) + valid_iterator = DataLoader(valid_dataset, collate_fn=collate_fn, **loader_parameters) print(f"Length of valid iterator = {len(valid_iterator)}") return valid_iterator diff --git a/python/arabic/diacritize.py b/python/arabic/diacritize.py index f92f521..fb518c4 100644 --- a/python/arabic/diacritize.py +++ b/python/arabic/diacritize.py @@ -30,9 +30,9 @@ def diacritization_parser(): raise ValueError("text or text_file params required!") if args.model_kind == "cbhg": - diacritizer = Diacritizer(args.config, args.model_kind, 'log_dir') + diacritizer = Diacritizer(args.config, args.model_kind, "log_dir") elif args.model_kind == "baseline": - diacritizer = Diacritizer(args.config, args.model_kind, 'log_dir') + diacritizer = Diacritizer(args.config, args.model_kind, "log_dir") else: raise ValueError("The model kind is not supported") diff --git a/python/arabic/diacritizer.py b/python/arabic/diacritizer.py index c468835..cae8a5e 100644 --- a/python/arabic/diacritizer.py +++ b/python/arabic/diacritizer.py @@ -10,14 +10,10 @@ class Diacritizer: - def __init__( - self, config_path: str, model_kind: str, load_model: bool = False - ) -> None: + def __init__(self, config_path: str, model_kind: str, load_model: bool = False) -> None: self.config_path = config_path self.model_kind = model_kind - self.config_manager = ConfigManager( - config_path=config_path, model_kind=model_kind - ) + self.config_manager = ConfigManager(config_path=config_path, model_kind=model_kind) self.config = self.config_manager.config self.text_encoder = self.config_manager.text_encoder @@ -37,40 +33,39 @@ def diacritize_text(self, text: str): text = text.strip() seq = self.text_encoder.input_to_sequence(text) # transform indices into "batch data" - batch_data = {'original': [text], - 'src': torch.Tensor([seq]).long(), - 'lengths': torch.Tensor([len(seq)]).long()} + batch_data = { + "original": [text], + "src": torch.Tensor([seq]).long(), + "lengths": torch.Tensor([len(seq)]).long(), + } return self.diacritize_batch(batch_data)[0] def get_data_from_file(self, path): """get data from relative path""" - loader_params = {"batch_size": self.config_manager.config["batch_size"], - "shuffle": False, - "num_workers": 2} + {"batch_size": self.config_manager.config["batch_size"], "shuffle": False, "num_workers": 2} - data_tmp = pd.read_csv(path, - encoding="utf-8", - sep=self.config_manager.config["data_separator"], - header=None) + data_tmp = pd.read_csv( + path, encoding="utf-8", sep=self.config_manager.config["data_separator"], header=None + ) data = [] max_len = self.config_manager.config["max_len"] for txt in [d[0] for d in data_tmp.values.tolist()]: if len(txt) > max_len: txt = txt[:max_len] - warnings.warn('Warning: text length cut for sentence: \n'+txt) + warnings.warn("Warning: text length cut for sentence: \n" + txt) data.append(txt) list_ids = [idx for idx in range(len(data))] - dataset = DiacritizationDataset(self.config_manager, - list_ids, - data) + dataset = DiacritizationDataset(self.config_manager, list_ids, data) - data_iterator = DataLoader(dataset, - collate_fn=collate_fn, - # **loader_params, - shuffle=False) + data_iterator = DataLoader( + dataset, + collate_fn=collate_fn, + # **loader_params, + shuffle=False, + ) # print(f"Length of data iterator = {len(data_iterator)}") return data_iterator @@ -80,10 +75,9 @@ def diacritize_file(self, path: str): data_iterator = self.get_data_from_file(path) diacritized_data = [] for batch_inputs in tqdm.tqdm(data_iterator): - - #batch_inputs["original"] = batch_inputs["original"].to(self.device) + # batch_inputs["original"] = batch_inputs["original"].to(self.device) batch_inputs["src"] = batch_inputs["src"].to(self.device) - batch_inputs["lengths"] = batch_inputs["lengths"].to('cpu') + batch_inputs["lengths"] = batch_inputs["lengths"].to("cpu") batch_inputs["target"] = batch_inputs["target"].to(self.device) for d in self.diacritize_batch(batch_inputs): @@ -94,7 +88,7 @@ def diacritize_file(self, path: str): def diacritize_batch(self, batch): # print('batch: ',batch) self.model.eval() - originals = batch['original'] + originals = batch["original"] inputs = batch["src"] lengths = batch["lengths"] outputs = self.model(inputs.to(self.device), lengths.to("cpu")) @@ -104,12 +98,12 @@ def diacritize_batch(self, batch): sentences = [] for src, prediction, original in zip(inputs, predictions, originals): sentence = self.text_encoder.combine_text_and_haraqat( - list(src.detach().cpu().numpy()), - list(prediction.detach().cpu().numpy())) + list(src.detach().cpu().numpy()), list(prediction.detach().cpu().numpy()) + ) # Diacritized strings, sentence have to be "reconciled" # with original strings, because the non arabic strings are removed # before being processed in nnet - if self.config['reconcile']: + if self.config["reconcile"]: sentence = reconcile.reconcile_strings(original, sentence) sentences.append(sentence) diff --git a/python/arabic/models/baseline.py b/python/arabic/models/baseline.py index af78120..06f569b 100644 --- a/python/arabic/models/baseline.py +++ b/python/arabic/models/baseline.py @@ -1,5 +1,3 @@ -from typing import List - import torch from torch import nn @@ -10,7 +8,7 @@ def __init__( inp_vocab_size: int, targ_vocab_size: int, embedding_dim: int = 512, - layers_units: List[int] = [256, 256, 256], + layers_units: list[int] = [256, 256, 256], use_batch_norm: bool = False, ): super().__init__() diff --git a/python/arabic/models/cbhg.py b/python/arabic/models/cbhg.py index b2263a5..cffe1c8 100644 --- a/python/arabic/models/cbhg.py +++ b/python/arabic/models/cbhg.py @@ -1,7 +1,8 @@ """ The CBHG model implementation """ -from typing import List, Optional + +from typing import Optional import torch from modules.tacotron_modules import CBHG, Prenet @@ -33,12 +34,12 @@ def __init__( targ_vocab_size: int, embedding_dim: int = 512, use_prenet: bool = True, - prenet_sizes: List[int] = [512, 256], + prenet_sizes: list[int] = [512, 256], cbhg_gru_units: int = 512, cbhg_filters: int = 16, - cbhg_projections: List[int] = [128, 256], - post_cbhg_layers_units: List[int] = [256, 256], - post_cbhg_use_batch_norm: bool = True + cbhg_projections: list[int] = [128, 256], + post_cbhg_layers_units: list[int] = [256, 256], + post_cbhg_use_batch_norm: bool = True, ): super().__init__() self.use_prenet = use_prenet @@ -73,12 +74,11 @@ def __init__( self.post_cbhg_layers_units = post_cbhg_layers_units self.post_cbhg_use_batch_norm = post_cbhg_use_batch_norm - def forward( self, src: torch.Tensor, lengths: Optional[torch.Tensor] = None, - target: Optional[torch.Tensor] = None # not required in this model + target: Optional[torch.Tensor] = None, # not required in this model ): """Compute forward propagation""" diff --git a/python/arabic/models/seq2seq.py b/python/arabic/models/seq2seq.py index 2e1fc37..f42a5ef 100644 --- a/python/arabic/models/seq2seq.py +++ b/python/arabic/models/seq2seq.py @@ -1,4 +1,4 @@ -from typing import List, Optional +from typing import Optional import torch from modules.attention import AttentionWrapper @@ -37,7 +37,7 @@ def __init__( self, inp_vocab_size: int, embedding_dim: int = 512, - layers_units: List[int] = [256, 256, 256], + layers_units: list[int] = [256, 256, 256], use_batch_norm: bool = False, ): super().__init__() @@ -81,6 +81,7 @@ def forward(self, inputs: torch.Tensor, inputs_lengths: torch.Tensor): return outputs + class Decoder(nn.Module): """A seq2seq decoder that decode a diacritic at a time , Args: @@ -100,7 +101,7 @@ def __init__( attention_units: int = 256, attention_type: AttentionType = AttentionType.LocationSensitive, is_attention_accumulative: bool = False, - prenet_depth: List[int] = [256, 128], + prenet_depth: list[int] = [256, 128], use_prenet: bool = True, teacher_forcing_probability: float = 0.0, ): @@ -193,9 +194,7 @@ def inference(self): """Generate diacritics one at a time""" batch_size = self.encoder_outputs.size(0) trg_len = self.encoder_outputs.size(1) - diacritic = ( - torch.full((batch_size,), self.start_symbol_id).to(self.device).long() - ) + diacritic = torch.full((batch_size,), self.start_symbol_id).to(self.device).long() outputs, alignments = [], [] self.initialize() @@ -239,18 +238,16 @@ def forward( self.initialize() - diacritic = ( - torch.full((batch_size,), self.start_symbol_id).to(self.device).long() - ) + diacritic = torch.full((batch_size,), self.start_symbol_id).to(self.device).long() for time in range(trg_len): output, alignment = self.decode(diacritic=diacritic) outputs += [output] alignments += [alignment] - #if random.random() > self.teacher_forcing_probability: + # if random.random() > self.teacher_forcing_probability: diacritic = diacritics[:, time] # use training input - #else: - #diacritic = torch.max(output, 1).indices # use last output + # else: + # diacritic = torch.max(output, 1).indices # use last output alignments = torch.stack(alignments).transpose(0, 1) outputs = torch.stack(outputs).transpose(0, 1).contiguous() @@ -261,14 +258,12 @@ def initialize(self): """Initialize the first step variables""" batch_size = self.encoder_outputs.size(0) src_len = self.encoder_outputs.size(1) - self.attention_hidden = Variable( - torch.zeros(batch_size, self.attention_units) - ).to(self.device) + self.attention_hidden = Variable(torch.zeros(batch_size, self.attention_units)).to( + self.device + ) self.decoder_hiddens = [ Variable(torch.zeros(batch_size, self.decoder_units)).to(self.device) for _ in range(len(self.decoder_rnns)) ] - self.prev_attention = Variable(torch.zeros(batch_size, self.encoder_dim)).to( - self.device - ) + self.prev_attention = Variable(torch.zeros(batch_size, self.encoder_dim)).to(self.device) self.prev_alignment = Variable(torch.zeros(batch_size, src_len)).to(self.device) diff --git a/python/arabic/models/tacotron_based.py b/python/arabic/models/tacotron_based.py index 3c02dc8..1e7feec 100644 --- a/python/arabic/models/tacotron_based.py +++ b/python/arabic/models/tacotron_based.py @@ -1,5 +1,3 @@ -from typing import List - from models.seq2seq import Decoder as Seq2SeqDecoder from models.seq2seq import Seq2Seq from modules.tacotron_modules import CBHG, Prenet @@ -16,18 +14,16 @@ def __init__( inp_vocab_size: int, embedding_dim: int = 512, use_prenet: bool = True, - prenet_sizes: List[int] = [256, 128], + prenet_sizes: list[int] = [256, 128], cbhg_gru_units: int = 128, cbhg_filters: int = 16, - cbhg_projections: List[int] = [128, 128], + cbhg_projections: list[int] = [128, 128], padding_idx: int = 0, ): super().__init__() self.use_prenet = use_prenet - self.embedding = nn.Embedding( - inp_vocab_size, embedding_dim, padding_idx=padding_idx - ) + self.embedding = nn.Embedding(inp_vocab_size, embedding_dim, padding_idx=padding_idx) if use_prenet: self.prenet = Prenet(embedding_dim, prenet_depth=prenet_sizes) self.cbhg = CBHG( diff --git a/python/arabic/modules/layers.py b/python/arabic/modules/layers.py index e135522..e23674d 100644 --- a/python/arabic/modules/layers.py +++ b/python/arabic/modules/layers.py @@ -36,40 +36,55 @@ def forward(self, x: Any): x = self.conv1d(x) if self.activation is not None: x = self.activation(x) - #x = self.activation(x) + # x = self.activation(x) x = self.bn(x) return x class LinearNorm(torch.nn.Module): - def __init__(self, in_dim, out_dim, bias=True, w_init_gain='linear'): + def __init__(self, in_dim, out_dim, bias=True, w_init_gain="linear"): super().__init__() self.linear_layer = torch.nn.Linear(in_dim, out_dim, bias=bias) torch.nn.init.xavier_uniform_( - self.linear_layer.weight, - gain=torch.nn.init.calculate_gain(w_init_gain)) + self.linear_layer.weight, gain=torch.nn.init.calculate_gain(w_init_gain) + ) def forward(self, x): return self.linear_layer(x) class ConvNorm(torch.nn.Module): - def __init__(self, in_channels, out_channels, kernel_size=1, stride=1, - padding=None, dilation=1, bias=True, w_init_gain='linear'): + def __init__( + self, + in_channels, + out_channels, + kernel_size=1, + stride=1, + padding=None, + dilation=1, + bias=True, + w_init_gain="linear", + ): super().__init__() if padding is None: - assert(kernel_size % 2 == 1) + assert kernel_size % 2 == 1 padding = int(dilation * (kernel_size - 1) / 2) - self.conv = torch.nn.Conv1d(in_channels, out_channels, - kernel_size=kernel_size, stride=stride, - padding=padding, dilation=dilation, - bias=bias) + self.conv = torch.nn.Conv1d( + in_channels, + out_channels, + kernel_size=kernel_size, + stride=stride, + padding=padding, + dilation=dilation, + bias=bias, + ) torch.nn.init.xavier_uniform_( - self.conv.weight, gain=torch.nn.init.calculate_gain(w_init_gain)) + self.conv.weight, gain=torch.nn.init.calculate_gain(w_init_gain) + ) def forward(self, signal): conv_signal = self.conv(signal) diff --git a/python/arabic/modules/tacotron_modules.py b/python/arabic/modules/tacotron_modules.py index 875b924..7e20bce 100644 --- a/python/arabic/modules/tacotron_modules.py +++ b/python/arabic/modules/tacotron_modules.py @@ -1,8 +1,8 @@ """ Some custom modules that are used by the TTS model """ + from copy import deepcopy -from typing import List import torch from modules.layers import BatchNormConv1d @@ -18,17 +18,12 @@ class Prenet(nn.Module): in_dim (int): the input dim """ - def __init__( - self, in_dim: int, prenet_depth: List[int] = [256, 128], dropout: int = 0.5 - ): - """ Initializing the prenet module """ + def __init__(self, in_dim: int, prenet_depth: list[int] = [256, 128], dropout: int = 0.5): + """Initializing the prenet module""" super().__init__() in_sizes = [in_dim] + prenet_depth[:-1] self.layers = nn.ModuleList( - [ - nn.Linear(in_size, out_size) - for (in_size, out_size) in zip(in_sizes, prenet_depth) - ] + [nn.Linear(in_size, out_size) for (in_size, out_size) in zip(in_sizes, prenet_depth)] ) self.relu = nn.ReLU() self.dropout = nn.Dropout(dropout) @@ -90,7 +85,7 @@ def __init__( in_dim: int, out_dim: int, K: int, - projections: List[int], + projections: list[int], highway_layers: int = 4, ): """Initializing the CBHG module @@ -108,25 +103,26 @@ def __init__( [ deepcopy( BatchNormConv1d( - in_dim, - in_dim, - kernel_size=k, - stride=1, - padding=k // 2, - activation=self.relu, - )) + in_dim, + in_dim, + kernel_size=k, + stride=1, + padding=k // 2, + activation=self.relu, + ) + ) for k in range(1, K + 1) ] ) k = 2 self.trafo_test = BatchNormConv1d( - in_dim, - in_dim, - kernel_size=k, - stride=1, - padding=k // 2, - activation=self.relu, - ) + in_dim, + in_dim, + kernel_size=k, + stride=1, + padding=k // 2, + activation=self.relu, + ) self.trafo = deepcopy(self.trafo_test) @@ -136,9 +132,11 @@ def __init__( activations = [self.relu] * (len(projections) - 1) + [None] self.conv1d_projections = nn.ModuleList( [ - deepcopy(BatchNormConv1d( - in_size, out_size, kernel_size=3, stride=1, padding=1, activation=ac - )) + deepcopy( + BatchNormConv1d( + in_size, out_size, kernel_size=3, stride=1, padding=1, activation=ac + ) + ) for (in_size, out_size, ac) in zip(in_sizes, projections, activations) ] ) diff --git a/python/arabic/options.py b/python/arabic/options.py index 6b850c0..bc65f0b 100644 --- a/python/arabic/options.py +++ b/python/arabic/options.py @@ -1,6 +1,7 @@ """ Types of various choices used during training """ + from enum import Enum diff --git a/python/arabic/setup.py b/python/arabic/setup.py index f3e3094..0527c31 100644 --- a/python/arabic/setup.py +++ b/python/arabic/setup.py @@ -9,42 +9,42 @@ PKG_VERSION = "0.1.0" GIT_TAG = environ.get("GITHUB_REF", "") -TAG_VERSION = re.match(r'^refs/tags/v([0-9]+\.[0-9a-z]+\.[0-9a-z]+)$', GIT_TAG) +TAG_VERSION = re.match(r"^refs/tags/v([0-9]+\.[0-9a-z]+\.[0-9a-z]+)$", GIT_TAG) if TAG_VERSION: PKG_VERSION = TAG_VERSION.group(1) setuptools.setup( - name='rababa', + name="rababa", version=PKG_VERSION, author="Ribose", author_email="open.source@ribose.com", - license='MIT', - description='Rababa for Arabic diacriticization', + license="MIT", + description="Rababa for Arabic diacriticization", # packages=['rababa'], - url='https://www.interscript.org', - python_requires='>=3.6, <4', + url="https://www.interscript.org", + python_requires=">=3.6, <4", project_urls={ - 'Documentation': 'https://github.com/interscript/rababa', - 'Source': 'https://github.com/interscript/rababa', - 'Tracker': 'https://github.com/interscript/rababa/issues', + "Documentation": "https://github.com/interscript/rababa", + "Source": "https://github.com/interscript/rababa", + "Tracker": "https://github.com/interscript/rababa/issues", }, install_requires=[ - 'torch>=1.9.0', - 'numpy', - 'matplotlib', - 'pandas', - 'ruamel.yaml', - 'tensorboard', - 'diacritization-evaluation', - 'tqdm', - 'onnx', - 'onnxruntime', - 'pyyaml', + "torch>=1.9.0", + "numpy", + "matplotlib", + "pandas", + "ruamel.yaml", + "tensorboard", + "diacritization-evaluation", + "tqdm", + "onnx", + "onnxruntime", + "pyyaml", ], # extras_require={'plotting': ['matplotlib>=2.2.0', 'jupyter']}, - setup_requires=['pytest-runner'], - tests_require=['pytest'], + setup_requires=["pytest-runner"], + tests_require=["pytest"], # entry_points={ # 'console_scripts': ['my-command=exampleproject.example:main'] # }, diff --git a/python/arabic/tester.py b/python/arabic/tester.py index 9b975be..d24c59e 100644 --- a/python/arabic/tester.py +++ b/python/arabic/tester.py @@ -1,4 +1,3 @@ - import torch from config_manager import ConfigManager from dataset import load_iterators @@ -11,9 +10,7 @@ class DiacritizationTester(GeneralTrainer): def __init__(self, config_path: str, model_kind: str) -> None: self.config_path = config_path self.model_kind = model_kind - self.config_manager = ConfigManager( - config_path=config_path, model_kind=model_kind - ) + self.config_manager = ConfigManager(config_path=config_path, model_kind=model_kind) self.config = self.config_manager.config self.pad_idx = 0 self.criterion = nn.CrossEntropyLoss(ignore_index=self.pad_idx) diff --git a/python/arabic/train.py b/python/arabic/train.py index 811640f..dba06c6 100644 --- a/python/arabic/train.py +++ b/python/arabic/train.py @@ -1,4 +1,3 @@ - import argparse import random @@ -32,7 +31,7 @@ def train_parser(): args = parser.parse_args() -if args.model_kind in ['baseline',"cbhg"]: +if args.model_kind in ["baseline", "cbhg"]: trainer = CBHGTrainer(args.config, args.model_kind) else: raise ValueError("The model kind is not supported") diff --git a/python/arabic/trainer.py b/python/arabic/trainer.py index beccd92..ba8c4dd 100644 --- a/python/arabic/trainer.py +++ b/python/arabic/trainer.py @@ -1,5 +1,4 @@ import os -from typing import Dict import torch from config_manager import ConfigManager @@ -29,9 +28,7 @@ class GeneralTrainer(Trainer): def __init__(self, config_path: str, model_kind: str) -> None: self.config_path = config_path self.model_kind = model_kind - self.config_manager = ConfigManager( - config_path=config_path, model_kind=model_kind - ) + self.config_manager = ConfigManager(config_path=config_path, model_kind=model_kind) self.config = self.config_manager.config self.losses = [] self.lr = 0 @@ -77,7 +74,7 @@ def load_diacritizer(self): if self.model_kind in ["cbhg", "baseline"]: self.diacritizer = Diacritizer(self.config_path, self.model_kind) else: - print('model not found') + print("model not found") exit() def initialize_model(self): @@ -95,7 +92,6 @@ def print_losses(self, step_results, tqdm): tqdm.display(f"loss: {step_results['loss']}", pos=3) for pos, n_steps in enumerate(self.config["n_steps_avg_losses"]): if len(self.losses) > n_steps: - self.summary_manager.add_scalar( f"loss/loss-{n_steps}", sum(self.losses[-n_steps:]) / n_steps, @@ -160,9 +156,7 @@ def evaluate_with_error_rates(self, iterator, tqdm): summary_texts = [] orig_path = os.path.join(self.config_manager.prediction_dir, "original.txt") - predicted_path = os.path.join( - self.config_manager.prediction_dir, "predicted.txt" - ) + predicted_path = os.path.join(self.config_manager.prediction_dir, "predicted.txt") with open(orig_path, "w", encoding="utf8") as file: for sentence in all_orig: @@ -176,18 +170,12 @@ def evaluate_with_error_rates(self, iterator, tqdm): if i > len(all_predicted): break - summary_texts.append( - (f"eval-text/{i}", f"{ all_orig[i]} |-> {all_predicted[i]}") - ) + summary_texts.append((f"eval-text/{i}", f"{all_orig[i]} |-> {all_predicted[i]}")) results["DER"] = der.calculate_der_from_path(orig_path, predicted_path) - results["DER*"] = der.calculate_der_from_path( - orig_path, predicted_path, case_ending=False - ) + results["DER*"] = der.calculate_der_from_path(orig_path, predicted_path, case_ending=False) results["WER"] = wer.calculate_wer_from_path(orig_path, predicted_path) - results["WER*"] = wer.calculate_wer_from_path( - orig_path, predicted_path, case_ending=False - ) + results["WER*"] = wer.calculate_wer_from_path(orig_path, predicted_path, case_ending=False) tqdm.reset() return results, summary_texts @@ -206,9 +194,7 @@ def run(self): for batch_inputs in repeater(train_iterator): tqdm.set_description(f"Global Step {self.global_step}") if self.config["use_decay"]: - self.lr = self.adjust_learning_rate( - self.optimizer, global_step=self.global_step - ) + self.lr = self.adjust_learning_rate(self.optimizer, global_step=self.global_step) self.optimizer.zero_grad() if self.device == "cuda" and self.config["use_mixed_precision"]: with autocast(): @@ -216,9 +202,7 @@ def run(self): scaler.scale(step_results["loss"]).backward() scaler.unscale_(self.optimizer) if self.config.get("CLIP"): - torch.nn.utils.clip_grad_norm_( - self.model.parameters(), self.config["CLIP"] - ) + torch.nn.utils.clip_grad_norm_(self.model.parameters(), self.config["CLIP"]) scaler.step(self.optimizer) @@ -229,9 +213,7 @@ def run(self): loss = step_results["loss"] loss.backward() if self.config.get("CLIP"): - torch.nn.utils.clip_grad_norm_( - self.model.parameters(), self.config["CLIP"] - ) + torch.nn.utils.clip_grad_norm_(self.model.parameters(), self.config["CLIP"]) self.optimizer.step() self.losses.append(step_results["loss"].item()) @@ -257,21 +239,12 @@ def run(self): if self.global_step % self.config["evaluate_frequency"] == 0: loss, acc = self.evaluate(validation_iterator, tqdm_eval) - self.summary_manager.add_scalar( - "evaluate/loss", loss, global_step=self.global_step - ) - self.summary_manager.add_scalar( - "evaluate/acc", acc, global_step=self.global_step - ) - tqdm.display( - f"Evaluate {self.global_step}: accuracy, {acc}, loss: {loss}", pos=8 - ) + self.summary_manager.add_scalar("evaluate/loss", loss, global_step=self.global_step) + self.summary_manager.add_scalar("evaluate/acc", acc, global_step=self.global_step) + tqdm.display(f"Evaluate {self.global_step}: accuracy, {acc}, loss: {loss}", pos=8) self.model.train() - if ( - self.global_step % self.config["evaluate_with_error_rates_frequency"] - == 0 - ): + if self.global_step % self.config["evaluate_with_error_rates_frequency"] == 0: error_rates, summery_texts = self.evaluate_with_error_rates( validation_iterator, tqdm_error_rates ) @@ -322,7 +295,7 @@ def run(self): tqdm.update() - def run_one_step(self, batch_inputs: Dict[str, torch.Tensor]): + def run_one_step(self, batch_inputs: dict[str, torch.Tensor]): batch_inputs["src"] = batch_inputs["src"].to(self.device) batch_inputs["lengths"] = batch_inputs["lengths"].to("cpu") batch_inputs["target"] = batch_inputs["target"].to(self.device) @@ -339,8 +312,7 @@ def run_one_step(self, batch_inputs: Dict[str, torch.Tensor]): predictions = predictions.view(-1, predictions.shape[-1]) targets = targets.view(-1) - loss = self.criterion(predictions.to(self.device), - targets.to(self.device)) + loss = self.criterion(predictions.to(self.device), targets.to(self.device)) outputs.update({"loss": loss}) return outputs @@ -348,9 +320,7 @@ def predict(self, iterator): pass def load_model(self, model_path: str = None, load_optimizer: bool = True): - with open( - self.config_manager.base_dir / f"{self.model_kind}_network.txt", "w" - ) as file: + with open(self.config_manager.base_dir / f"{self.model_kind}_network.txt", "w") as file: file.write(str(self.model)) if model_path is None: @@ -362,8 +332,11 @@ def load_model(self, model_path: str = None, load_optimizer: bool = True): last_model_path = model_path print(f"loading from {last_model_path}") - saved_model = torch.load(last_model_path) if torch.cuda.is_available() \ - else torch.load(last_model_path, map_location=torch.device('cpu')) + saved_model = ( + torch.load(last_model_path) + if torch.cuda.is_available() + else torch.load(last_model_path, map_location=torch.device("cpu")) + ) self.model.load_state_dict(saved_model["model_state_dict"]) if load_optimizer: self.optimizer.load_state_dict(saved_model["optimizer_state_dict"]) diff --git a/python/arabic/util/constants.py b/python/arabic/util/constants.py index 3093b59..e9b33c4 100644 --- a/python/arabic/util/constants.py +++ b/python/arabic/util/constants.py @@ -1,8 +1,9 @@ """ Constants that are used by the model """ + HARAQAT = ["ْ", "ّ", "ٌ", "ٍ", "ِ", "ً", "َ", "ُ"] -ARAB_CHARS = '\u0649\u0639\u0638\u062D\u0631\u0633\u064A\u0634\u0636\u0642 \u062B\u0644\u0635\u0637\u0643\u0622\u0645\u0627\u0625\u0647\u0632\u0621\u0623\u0641\u0624\u063A\u062C\u0626\u062F\u0629\u062E\u0648\u0628\u0630\u062A\u0646' +ARAB_CHARS = "\u0649\u0639\u0638\u062d\u0631\u0633\u064a\u0634\u0636\u0642 \u062b\u0644\u0635\u0637\u0643\u0622\u0645\u0627\u0625\u0647\u0632\u0621\u0623\u0641\u0624\u063a\u062c\u0626\u062f\u0629\u062e\u0648\u0628\u0630\u062a\u0646" PUNCTUATIONS = [".", "،", ":", "؛", "-", "؟"] VALID_ARABIC = HARAQAT + list(ARAB_CHARS) + [".", "،", ":", "؛", "-", "؟"] BASIC_HARAQAT = { diff --git a/python/arabic/util/learning_rates.py b/python/arabic/util/learning_rates.py index 28e4fae..4078856 100644 --- a/python/arabic/util/learning_rates.py +++ b/python/arabic/util/learning_rates.py @@ -12,12 +12,13 @@ def __call__(self, global_step) -> float: step = global_step + 1.0 lr = ( self.lr - * self.warmup_steps ** 0.5 - * np.minimum(step * self.warmup_steps ** -1.5, step ** -0.5) + * self.warmup_steps**0.5 + * np.minimum(step * self.warmup_steps**-1.5, step**-0.5) ) return lr + class SquareRootScheduler: def __init__(self, lr=0.1): self.lr = lr @@ -28,9 +29,7 @@ def __call__(self, global_step): class CosineScheduler: - def __init__( - self, max_update, base_lr=0.02, final_lr=0, warmup_steps=0, warmup_begin_lr=0 - ): + def __init__(self, max_update, base_lr=0.02, final_lr=0, warmup_steps=0, warmup_begin_lr=0): self.base_lr_orig = base_lr self.max_update = max_update self.final_lr = final_lr @@ -53,19 +52,14 @@ def __call__(self, global_step): self.base_lr = ( self.final_lr + (self.base_lr_orig - self.final_lr) - * ( - 1 - + math.cos( - math.pi * (global_step - self.warmup_steps) / self.max_steps - ) - ) + * (1 + math.cos(math.pi * (global_step - self.warmup_steps) / self.max_steps)) / 2 ) return self.base_lr + def adjust_learning_rate(optimizer, global_step): lr = LearningRateDecay()(global_step=global_step) for param_group in optimizer.param_groups: param_group["lr"] = lr return lr - diff --git a/python/arabic/util/reconcile_original_plus_diacritized.py b/python/arabic/util/reconcile_original_plus_diacritized.py index aba796f..0d45de2 100644 --- a/python/arabic/util/reconcile_original_plus_diacritized.py +++ b/python/arabic/util/reconcile_original_plus_diacritized.py @@ -16,20 +16,20 @@ b. end of original """ + def build_pivot_map(d_original, d_diacritized): """build_pivot_map: - This function takes 2 strings and finds the "pivot points", - i.e the points where both strings are identical. - args: - d_original: dictionary modelling the original string abc -> {0:a,1:b,2:c} - d_diacritized: dictionary modelling diacritized as above - return: list of ids tuple where strings match + This function takes 2 strings and finds the "pivot points", + i.e the points where both strings are identical. + args: + d_original: dictionary modelling the original string abc -> {0:a,1:b,2:c} + d_diacritized: dictionary modelling diacritized as above + return: list of ids tuple where strings match """ l_map = [] idx_dia, idx_ori = 0, 0 while idx_dia < len(d_diacritized): - c_dia = d_diacritized[idx_dia] for i in range(idx_ori, len(d_original)): if c_dia == d_original[i]: @@ -47,51 +47,51 @@ def build_pivot_map(d_original, d_diacritized): def reconcile_strings(str_original, str_diacritized): """reconcile_strings: - This function takes original and diacritized string and merge them into a sensible output. - For instance: - original string: - # گيله پسمير الجديد 34 - diacritised string (with non arabic removed by the nnets preprocessing): - يَلِهُ سُمِيْرٌ الجَدِيدُ - reconcile_strings --> - '# گيَلِهُ پسُمِيْرٌ الجَدِيدُ 34' - - Other examples and tests can be found in the commented section below. - args: - str_original: original string - str_diacritized: diacritized string - return: reconciled string + This function takes original and diacritized string and merge them into a sensible output. + For instance: + original string: + # گيله پسمير الجديد 34 + diacritised string (with non arabic removed by the nnets preprocessing): + يَلِهُ سُمِيْرٌ الجَدِيدُ + reconcile_strings --> + '# گيَلِهُ پسُمِيْرٌ الجَدِيدُ 34' + + Other examples and tests can be found in the commented section below. + args: + str_original: original string + str_diacritized: diacritized string + return: reconciled string """ # we model the strings as dict - d_original = dict((i,c) for i,c in - enumerate(list([c for c in str_original if c not in HARAQAT]))) - d_diacritized = dict((i,c) for i,c in enumerate(list(str_diacritized))) + d_original = dict( + (i, c) for i, c in enumerate(list([c for c in str_original if c not in HARAQAT])) + ) + d_diacritized = dict((i, c) for i, c in enumerate(list(str_diacritized))) # matching positions l_pivot_map = build_pivot_map(d_original, d_diacritized) - str__ = '' # "accumulated" chars - pt_dia, pt_ori = 0, 0 # pointers for resp diacr and orig. strings + str__ = "" # "accumulated" chars + pt_dia, pt_ori = 0, 0 # pointers for resp diacr and orig. strings for x_dia, x_ori in l_pivot_map: - # We start to write characters from original strings if pt_ori < x_ori: - for i in range(pt_ori, x_ori): + for i in range(pt_ori, x_ori): str__ += d_original[i] # We then add chars from diacritized strings if pt_dia < x_dia: - for i in range(pt_dia, x_dia): + for i in range(pt_dia, x_dia): str__ += d_diacritized[i] # append matches str__ += d_original[x_ori] pt_dia, pt_ori = x_dia + 1, x_ori + 1 # Finalize by adding first last diacritized chars and then - for i in range(pt_dia, len(d_diacritized)): + for i in range(pt_dia, len(d_diacritized)): str__ += d_diacritized[i] # remaining chars for original string - for i in range(pt_ori, len(d_original)): + for i in range(pt_ori, len(d_original)): str__ += d_original[i] return str__.strip() diff --git a/python/arabic/util/text_cleaners.py b/python/arabic/util/text_cleaners.py index b779d78..1c1560f 100644 --- a/python/arabic/util/text_cleaners.py +++ b/python/arabic/util/text_cleaners.py @@ -9,15 +9,18 @@ def collapse_whitespace(text): text = re.sub(_whitespace_re, " ", text) return text + def basic_cleaners(text): text = collapse_whitespace(text) return text.strip() + def valid_arabic_cleaners(text): text = filter(lambda char: char in VALID_ARABIC, text) - text = collapse_whitespace(''.join(list(text))) + text = collapse_whitespace("".join(list(text))) return text.strip() + def extract_stack(stack, correct_reversed: bool = True): """ Given stack, we extract its content to string, and check whether this string is @@ -34,16 +37,17 @@ def extract_stack(stack, correct_reversed: bool = True): elif reversed_full_haraqah in ALL_POSSIBLE_HARAQAT and correct_reversed: out = reversed_full_haraqah else: - #raise ValueError(stack) + # raise ValueError(stack) - #raise ValueError( + # raise ValueError( # f"""The chart has the following haraqat which are not found in - #all possible haraqat: {'|'.join([ALL_POSSIBLE_HARAQAT[diacritic] + # all possible haraqat: {'|'.join([ALL_POSSIBLE_HARAQAT[diacritic] # for diacritic in full_haraqah ])}""" - #) - out = '' + # ) + out = "" return out + def extract_haraqat(text: str, correct_reversed: bool = True): """ Args: @@ -61,9 +65,8 @@ def extract_haraqat(text: str, correct_reversed: bool = True): for char in text: # if chart is a diacritic, then extract the stack and empty it if char not in BASIC_HARAQAT.keys(): - stack_content = extract_stack(stack, - correct_reversed=correct_reversed) - #if stack_content != '': + stack_content = extract_stack(stack, correct_reversed=correct_reversed) + # if stack_content != '': haraqat_list.append(stack_content) txt_list.append(char) stack = [] diff --git a/python/arabic/util/text_encoders.py b/python/arabic/util/text_encoders.py index c9113d1..3d09476 100644 --- a/python/arabic/util/text_encoders.py +++ b/python/arabic/util/text_encoders.py @@ -1,4 +1,4 @@ -from typing import Dict, List, Optional +from typing import Optional from util.constants import ALL_POSSIBLE_HARAQAT @@ -10,8 +10,8 @@ class TextEncoder: def __init__( self, - input_chars: List[str], - target_charts: List[str], + input_chars: list[str], + target_charts: list[str], cleaner_fn: Optional[str] = None, reverse_input: bool = False, reverse_target: bool = False, @@ -21,22 +21,14 @@ def __init__( else: self.cleaner_fn = None - self.input_symbols: List[str] = [TextEncoder.pad] + input_chars - self.target_symbols: List[str] = [TextEncoder.pad] + target_charts + self.input_symbols: list[str] = [TextEncoder.pad] + input_chars + self.target_symbols: list[str] = [TextEncoder.pad] + target_charts - self.input_symbol_to_id: Dict[str, int] = { - s: i for i, s in enumerate(self.input_symbols) - } - self.input_id_to_symbol: Dict[int, str] = { - i: s for i, s in enumerate(self.input_symbols) - } + self.input_symbol_to_id: dict[str, int] = {s: i for i, s in enumerate(self.input_symbols)} + self.input_id_to_symbol: dict[int, str] = {i: s for i, s in enumerate(self.input_symbols)} - self.target_symbol_to_id: Dict[str, int] = { - s: i for i, s in enumerate(self.target_symbols) - } - self.target_id_to_symbol: Dict[int, str] = { - i: s for i, s in enumerate(self.target_symbols) - } + self.target_symbol_to_id: dict[str, int] = {s: i for i, s in enumerate(self.target_symbols)} + self.target_id_to_symbol: dict[int, str] = {i: s for i, s in enumerate(self.target_symbols)} self.reverse_input = reverse_input self.reverse_target = reverse_target @@ -44,34 +36,36 @@ def __init__( self.target_pad_id = self.target_symbol_to_id[self.pad] self.start_symbol_id = None - def input_to_sequence(self, text: str) -> List[int]: + def input_to_sequence(self, text: str) -> list[int]: if self.reverse_input: text = "".join(list(reversed(text))) - sequence = [self.input_symbol_to_id[s] for s in text - if s not in [self.pad] and \ - self.input_symbol_to_id.get(s, False)] + sequence = [ + self.input_symbol_to_id[s] + for s in text + if s not in [self.pad] and self.input_symbol_to_id.get(s, False) + ] if len(sequence) == 0: # handle cases with zero length strings (no arabic symbols) - sequence = [self.input_symbol_to_id[s] for s in ' '] + sequence = [self.input_symbol_to_id[s] for s in " "] return sequence - def target_to_sequence(self, text: str) -> List[int]: + def target_to_sequence(self, text: str) -> list[int]: if self.reverse_target: text = "".join(list(reversed(text))) sequence = [self.target_symbol_to_id[s] for s in text if s not in [self.pad]] return sequence - def sequence_to_input(self, sequence: List[int]): + def sequence_to_input(self, sequence: list[int]): return [ self.input_id_to_symbol[symbol] for symbol in sequence if symbol in self.input_id_to_symbol and symbol not in [self.input_pad_id] ] - def sequence_to_target(self, sequence: List[int]): + def sequence_to_target(self, sequence: list[int]): return [ self.target_id_to_symbol[symbol] for symbol in sequence @@ -83,7 +77,7 @@ def clean(self, text): return self.cleaner_fn(text) return text - def combine_text_and_haraqat(self, input_ids: List[int], output_ids: List[int]): + def combine_text_and_haraqat(self, input_ids: list[int], output_ids: list[int]): """ Combines the input text with its corresponding haraqat Args: @@ -112,8 +106,8 @@ def __init__( reverse_input: bool = False, reverse_target: bool = False, ): - input_chars: List[str] = list("بض.غىهظخة؟:طس،؛فندؤلوئآك-يذاصشحزءمأجإ ترقعث") - target_charts: List[str] = list(ALL_POSSIBLE_HARAQAT.keys()) + input_chars: list[str] = list("بض.غىهظخة؟:طس،؛فندؤلوئآك-يذاصشحزءمأجإ ترقعث") + target_charts: list[str] = list(ALL_POSSIBLE_HARAQAT.keys()) super().__init__( input_chars, @@ -131,9 +125,9 @@ def __init__( reverse_input: bool = False, reverse_target: bool = False, ): - input_chars: List[str] = list("بض.غىهظخة؟:طس،؛فندؤلوئآك-يذاصشحزءمأجإ ترقعث") + input_chars: list[str] = list("بض.غىهظخة؟:طس،؛فندؤلوئآك-يذاصشحزءمأجإ ترقعث") # the only difference from the basic encoder is adding the start symbol - target_charts: List[str] = list(ALL_POSSIBLE_HARAQAT.keys()) + ["s"] + target_charts: list[str] = list(ALL_POSSIBLE_HARAQAT.keys()) + ["s"] super().__init__( input_chars, diff --git a/python/arabic/util/utils.py b/python/arabic/util/utils.py index 0726731..c3b2347 100644 --- a/python/arabic/util/utils.py +++ b/python/arabic/util/utils.py @@ -61,8 +61,7 @@ def get_mask_from_lengths(memory, memory_lengths): def repeater(data_loader): for loader in repeat(data_loader): - for data in loader: - yield data + yield from loader def count_parameters(model): @@ -89,19 +88,16 @@ def get_decoder_layers_attentions(model): return self_attns, src_attens -def display_attention( - attention, path, global_step: int, name="att", n_heads=4, n_rows=2, n_cols=2 -): +def display_attention(attention, path, global_step: int, name="att", n_heads=4, n_rows=2, n_cols=2): assert n_rows * n_cols == n_heads fig = plt.figure(figsize=(15, 15)) for i in range(n_heads): - ax = fig.add_subplot(n_rows, n_cols, i + 1) _attention = attention.squeeze(0)[i].transpose(0, 1).cpu().detach().numpy() - cax = ax.imshow(_attention, aspect="auto", origin="lower", interpolation="none") + ax.imshow(_attention, aspect="auto", origin="lower", interpolation="none") plot_name = f"{global_step}-{name}.png" plt.savefig(os.path.join(path, plot_name), dpi=300, format="png") @@ -112,17 +108,11 @@ def plot_multi_head(model, path, global_step): encoder_attentions = get_encoder_layers_attentions(model) decoder_attentions, attentions = get_decoder_layers_attentions(model) for i in range(len(attentions)): - display_attention( - attentions[0][0], path, global_step, f"encoder-decoder-layer{i + 1}" - ) + display_attention(attentions[0][0], path, global_step, f"encoder-decoder-layer{i + 1}") for i in range(len(decoder_attentions)): - display_attention( - decoder_attentions[0][0], path, global_step, f"decoder-layer{i + 1}" - ) + display_attention(decoder_attentions[0][0], path, global_step, f"decoder-layer{i + 1}") for i in range(len(encoder_attentions)): - display_attention( - encoder_attentions[0][0], path, global_step, f"encoder-layer {i + 1}" - ) + display_attention(encoder_attentions[0][0], path, global_step, f"encoder-layer {i + 1}") def make_src_mask(src, pad_idx=0): @@ -196,9 +186,7 @@ def categorical_accuracy(preds, y, tag_pad_idx, device="cuda"): """ Returns accuracy per batch, i.e. if you get 8/10 right, this returns 0.8, NOT 8 """ - max_preds = preds.argmax( - dim=1, keepdim=True - ) # get the index of the max probability + max_preds = preds.argmax(dim=1, keepdim=True) # get the index of the max probability non_pad_elements = torch.nonzero(y != tag_pad_idx) correct = max_preds[non_pad_elements].squeeze(1).eq(y[non_pad_elements]) return correct.sum() / torch.FloatTensor([y[non_pad_elements].shape[0]]).to(device) diff --git a/python/hebrew/config_manager.py b/python/hebrew/config_manager.py index 2c8f94f..02e0ee2 100644 --- a/python/hebrew/config_manager.py +++ b/python/hebrew/config_manager.py @@ -3,7 +3,7 @@ import subprocess from enum import Enum from pathlib import Path -from typing import Any, Dict +from typing import Any import ruamel.yaml import torch @@ -29,7 +29,7 @@ def __init__(self, config_path: str, model_kind: str): self.config_path = Path(config_path) self.model_kind = model_kind self.yaml = ruamel.yaml.YAML() - self.config: Dict[str, Any] = self._load_config() + self.config: dict[str, Any] = self._load_config() self.set_device() self.session_name = ".".join( [self.config["session_name"], f"{model_kind}"] # self.config["data_type"], @@ -37,9 +37,7 @@ def __init__(self, config_path: str, model_kind: str): self.data_dir = Path(os.path.join(self.config["data_directory"])) - self.base_dir = Path( - os.path.join(self.config["log_directory"], self.session_name) - ) + self.base_dir = Path(os.path.join(self.config["log_directory"], self.session_name)) self.log_dir = Path(os.path.join(self.base_dir, "logs")) self.prediction_dir = Path(os.path.join(self.base_dir, "predictions")) @@ -66,25 +64,17 @@ def set_device(self): @staticmethod def _get_git_hash(): try: - return ( - subprocess.check_output(["git", "describe", "--always"]) - .strip() - .decode() - ) + return subprocess.check_output(["git", "describe", "--always"]).strip().decode() except Exception as e: print(f"WARNING: could not retrieve git hash. {e}") def _check_hash(self): try: - git_hash = ( - subprocess.check_output(["git", "describe", "--always"]) - .strip() - .decode() - ) + git_hash = subprocess.check_output(["git", "describe", "--always"]).strip().decode() if self.config["git_hash"] != git_hash: print( f"""WARNING: git hash mismatch. Current: {git_hash}. - Config hash: {self.config['git_hash']}""" + Config hash: {self.config["git_hash"]}""" ) except Exception as e: print(f"WARNING: could not check git hash. {e}") @@ -100,9 +90,7 @@ def _print_dictionary(self, dictionary, recursion_level=0): recursion_level += 1 self._print_dictionary(dictionary[key], recursion_level) else: - self._print_dict_values( - dictionary[key], key_name=key, level=recursion_level - ) + self._print_dict_values(dictionary[key], key_name=key, level=recursion_level) def print_config(self): print("\nCONFIGURATION", self.session_name) @@ -189,9 +177,7 @@ def load_model(self, model_path: str = None, load_optimizer: bool = False): ) check = model.load_state_dict(saved_model["model_state_dict"]) print("Load model state dict:: ", check) # check... - optimizer_stat_dict = ( - saved_model["optimizer_state_dict"] if load_optimizer else None - ) + optimizer_stat_dict = saved_model["optimizer_state_dict"] if load_optimizer else None global_step = saved_model["global_step"] + 1 except: diff --git a/python/hebrew/convert_torch_model_to_onnx.py b/python/hebrew/convert_torch_model_to_onnx.py index 56fe236..df812ab 100644 --- a/python/hebrew/convert_torch_model_to_onnx.py +++ b/python/hebrew/convert_torch_model_to_onnx.py @@ -28,9 +28,7 @@ we found that populating all the data, removing the zeros gives better results. """ -normalized = torch.Tensor( - [[1 for i in range(max_len)] for i in range(batch_size)] -).long() +normalized = torch.Tensor([[1 for i in range(max_len)] for i in range(batch_size)]).long() """ @@ -90,9 +88,7 @@ """ # prepare onnx input -ort_inputs = { - ort_session.get_inputs()[0].name: normalized.detach().numpy().astype(np.int64) -} +ort_inputs = {ort_session.get_inputs()[0].name: normalized.detach().numpy().astype(np.int64)} # run onnx model ort_outs = ort_session.run(None, ort_inputs) @@ -116,9 +112,7 @@ vec = [[41, 12, 40] for i in range(batch_size)] normalized = torch.Tensor(vec).long() -ort_inputs = { - ort_session.get_inputs()[0].name: normalized.detach().numpy().astype(np.int64) -} +ort_inputs = {ort_session.get_inputs()[0].name: normalized.detach().numpy().astype(np.int64)} """ @@ -131,15 +125,12 @@ print(max_len) for test_run in range(3): - vec = [[random.randint(0, 1) for i in range(max_len)] for i in range(batch_size)] normalized = torch.Tensor(vec).long() torch_outs = dia.model(normalized) # prepare onnx input - ort_inputs = { - ort_session.get_inputs()[0].name: normalized.detach().numpy().astype(np.int64) - } + ort_inputs = {ort_session.get_inputs()[0].name: normalized.detach().numpy().astype(np.int64)} # run onnx model ort_outs = ort_session.run(None, ort_inputs) @@ -161,15 +152,12 @@ print(max_len) for test_run in range(3): - vec = [[random.randint(0, 17) for i in range(max_len)] for i in range(batch_size)] normalized = torch.Tensor(vec).long() torch_out = dia.model(normalized) # prepare onnx input - ort_inputs = { - ort_session.get_inputs()[0].name: normalized.detach().numpy().astype(np.int64) - } + ort_inputs = {ort_session.get_inputs()[0].name: normalized.detach().numpy().astype(np.int64)} # run onnx model ort_outs = ort_session.run(None, ort_inputs) @@ -187,7 +175,6 @@ print("***** Test Dynamical sizes :: Random Boolean vectors: *****") for l in [2, 10, 40, 100, 150]: - print("length:: ", l) vec = [[1 for i in range(l)] for i in range(batch_size)] # random.randint(0,1) @@ -196,9 +183,7 @@ torch_out = dia.model(normalized) # prepare onnx input - ort_inputs = { - ort_session.get_inputs()[0].name: normalized.detach().numpy().astype(np.int64) - } + ort_inputs = {ort_session.get_inputs()[0].name: normalized.detach().numpy().astype(np.int64)} # run onnx model ort_outs = ort_session.run(None, ort_inputs) @@ -219,16 +204,13 @@ print("***** Test Dynamical sizes :: Random float, vectors within 0:16 *****") for l in [2, 10, 40, 100, 150]: - vec = [[random.randint(0, 17) for i in range(l)] for i in range(batch_size)] normalized = torch.Tensor(vec).long() torch_out = dia.model(normalized) # prepare onnx input - ort_inputs = { - ort_session.get_inputs()[0].name: normalized.detach().numpy().astype(np.int64) - } + ort_inputs = {ort_session.get_inputs()[0].name: normalized.detach().numpy().astype(np.int64)} # run onnx model ort_outs = ort_session.run(None, ort_inputs) diff --git a/python/hebrew/dataset.py b/python/hebrew/dataset.py index a7a0142..9638ded 100644 --- a/python/hebrew/dataset.py +++ b/python/hebrew/dataset.py @@ -57,15 +57,11 @@ def load_training_data(config_manager: ConfigManager, loader_parameters): if not config_manager.config["load_training_data"]: return [] - path = os.path.join( - config_manager.data_dir, "train", config_manager.config["train_file_name"] - ) + path = os.path.join(config_manager.data_dir, "train", config_manager.config["train_file_name"]) training_set = DiacritizationDataset(config_manager, path) - train_iterator = DataLoader( - training_set.data, collate_fn=collate_fn, **loader_parameters - ) + train_iterator = DataLoader(training_set.data, collate_fn=collate_fn, **loader_parameters) print(f"Length of training iterator = {len(train_iterator)}") return train_iterator @@ -78,15 +74,11 @@ def load_test_data(config_manager: ConfigManager, loader_parameters): if not config_manager.config["load_test_data"]: return [] # test_file_name = config_manager.config.get("test_file_name", "test.csv") - path = os.path.join( - config_manager.data_dir, "test", config_manager.config["test_file_name"] - ) + path = os.path.join(config_manager.data_dir, "test", config_manager.config["test_file_name"]) test_dataset = DiacritizationDataset(config_manager, path) - test_iterator = DataLoader( - test_dataset.data, collate_fn=collate_fn, **loader_parameters - ) + test_iterator = DataLoader(test_dataset.data, collate_fn=collate_fn, **loader_parameters) print(f"Length of test iterator = {len(test_iterator)}") return test_iterator @@ -100,15 +92,11 @@ def load_validation_data(config_manager: ConfigManager, loader_parameters): if not config_manager.config["load_validation_data"]: return [] - path = os.path.join( - config_manager.data_dir, "eval", config_manager.config["eval_file_name"] - ) + path = os.path.join(config_manager.data_dir, "eval", config_manager.config["eval_file_name"]) valid_dataset = DiacritizationDataset(config_manager, path) - valid_iterator = DataLoader( - valid_dataset.data, collate_fn=collate_fn, **loader_parameters - ) + valid_iterator = DataLoader(valid_dataset.data, collate_fn=collate_fn, **loader_parameters) print(f"Length of valid iterator = {len(valid_iterator)}") return valid_iterator diff --git a/python/hebrew/diacritizer.py b/python/hebrew/diacritizer.py index 08d5e82..af3af92 100644 --- a/python/hebrew/diacritizer.py +++ b/python/hebrew/diacritizer.py @@ -10,14 +10,10 @@ class Diacritizer: - def __init__( - self, config_path: str, model_kind: str, load_model: bool = False - ) -> None: + def __init__(self, config_path: str, model_kind: str, load_model: bool = False) -> None: self.config_path = config_path self.model_kind = model_kind - self.config_manager = ConfigManager( - config_path=config_path, model_kind=model_kind - ) + self.config_manager = ConfigManager(config_path=config_path, model_kind=model_kind) self.config = self.config_manager.config self.text_encoder = self.config_manager.text_encoder self.device = self.config_manager.device @@ -44,12 +40,7 @@ def diacritize_text(self, text: str): dia_data.sin, ) - text = ( - " ".join(dia_total) - .replace("\ufeff", "") - .replace(" ", " ") - .replace(hebrew.RAFE, "") - ) + text = " ".join(dia_total).replace("\ufeff", "").replace(" ", " ").replace(hebrew.RAFE, "") return text def get_data_from_file(self, path): @@ -68,7 +59,7 @@ def get_data_from_file(self, path): def diacritize_file(self, path: str, path_out: str): """ - download data from relative path and diacritize it batch by batch + download data from relative path and diacritize it batch by batch """ data_iterator = self.get_data_from_file(path) @@ -86,12 +77,7 @@ def postprocess_data(raw_data): postprocess_data(dia_data.sin), ) - text = ( - " ".join(dia_total) - .replace("\ufeff", "") - .replace(" ", " ") - .replace(hebrew.RAFE, "") - ) + text = " ".join(dia_total).replace("\ufeff", "").replace(" ", " ").replace(hebrew.RAFE, "") with utils.smart_open(path_out, "w", encoding="utf-8") as f: f.write(text) @@ -132,7 +118,6 @@ def process_dim(dim): losses = None if criterion is not None: - losses = [ criterion(process_dim(niqqud), data_batch.niqqud.long()), criterion(process_dim(dagesh), data_batch.dagesh.long()), diff --git a/python/hebrew/models/baseline.py b/python/hebrew/models/baseline.py index af78120..06f569b 100644 --- a/python/hebrew/models/baseline.py +++ b/python/hebrew/models/baseline.py @@ -1,5 +1,3 @@ -from typing import List - import torch from torch import nn @@ -10,7 +8,7 @@ def __init__( inp_vocab_size: int, targ_vocab_size: int, embedding_dim: int = 512, - layers_units: List[int] = [256, 256, 256], + layers_units: list[int] = [256, 256, 256], use_batch_norm: bool = False, ): super().__init__() diff --git a/python/hebrew/models/cbhg.py b/python/hebrew/models/cbhg.py index 54a3137..e43b561 100644 --- a/python/hebrew/models/cbhg.py +++ b/python/hebrew/models/cbhg.py @@ -1,7 +1,8 @@ """ The CBHG model implementation """ -from typing import List, Optional + +from typing import Optional import torch from modules.tacotron_modules import CBHG, Prenet @@ -37,12 +38,12 @@ def __init__( targ_sin_size: int, embedding_dim: int = 512, use_prenet: bool = True, - prenet_sizes: List[int] = [512, 256], + prenet_sizes: list[int] = [512, 256], cbhg_gru_units: int = 512, cbhg_filters: int = 16, - cbhg_projections: List[int] = [128, 256], - post_cbhg_layers_units: List[int] = [256, 256], - post_cbhg_use_batch_norm: bool = True + cbhg_projections: list[int] = [128, 256], + post_cbhg_layers_units: list[int] = [256, 256], + post_cbhg_use_batch_norm: bool = True, ): super().__init__() self.use_prenet = use_prenet @@ -75,22 +76,18 @@ def __init__( self.post_cbhg_layers = nn.ModuleList(layers) - self.projections_niqqud = nn.Linear(post_cbhg_layers_units[-1] * 2, - targ_niqqud_size) - self.projections_dagesh = nn.Linear(post_cbhg_layers_units[-1] * 2, - targ_dagesh_size) - self.projections_sin = nn.Linear(post_cbhg_layers_units[-1] * 2, - targ_sin_size) + self.projections_niqqud = nn.Linear(post_cbhg_layers_units[-1] * 2, targ_niqqud_size) + self.projections_dagesh = nn.Linear(post_cbhg_layers_units[-1] * 2, targ_dagesh_size) + self.projections_sin = nn.Linear(post_cbhg_layers_units[-1] * 2, targ_sin_size) self.post_cbhg_layers_units = post_cbhg_layers_units self.post_cbhg_use_batch_norm = post_cbhg_use_batch_norm - def forward( self, src: torch.Tensor, lengths: Optional[torch.Tensor] = None, - target: Optional[torch.Tensor] = None # not required in this model + target: Optional[torch.Tensor] = None, # not required in this model ): """Compute forward propagation""" diff --git a/python/hebrew/models/seq2seq.py b/python/hebrew/models/seq2seq.py index 2e1fc37..f42a5ef 100644 --- a/python/hebrew/models/seq2seq.py +++ b/python/hebrew/models/seq2seq.py @@ -1,4 +1,4 @@ -from typing import List, Optional +from typing import Optional import torch from modules.attention import AttentionWrapper @@ -37,7 +37,7 @@ def __init__( self, inp_vocab_size: int, embedding_dim: int = 512, - layers_units: List[int] = [256, 256, 256], + layers_units: list[int] = [256, 256, 256], use_batch_norm: bool = False, ): super().__init__() @@ -81,6 +81,7 @@ def forward(self, inputs: torch.Tensor, inputs_lengths: torch.Tensor): return outputs + class Decoder(nn.Module): """A seq2seq decoder that decode a diacritic at a time , Args: @@ -100,7 +101,7 @@ def __init__( attention_units: int = 256, attention_type: AttentionType = AttentionType.LocationSensitive, is_attention_accumulative: bool = False, - prenet_depth: List[int] = [256, 128], + prenet_depth: list[int] = [256, 128], use_prenet: bool = True, teacher_forcing_probability: float = 0.0, ): @@ -193,9 +194,7 @@ def inference(self): """Generate diacritics one at a time""" batch_size = self.encoder_outputs.size(0) trg_len = self.encoder_outputs.size(1) - diacritic = ( - torch.full((batch_size,), self.start_symbol_id).to(self.device).long() - ) + diacritic = torch.full((batch_size,), self.start_symbol_id).to(self.device).long() outputs, alignments = [], [] self.initialize() @@ -239,18 +238,16 @@ def forward( self.initialize() - diacritic = ( - torch.full((batch_size,), self.start_symbol_id).to(self.device).long() - ) + diacritic = torch.full((batch_size,), self.start_symbol_id).to(self.device).long() for time in range(trg_len): output, alignment = self.decode(diacritic=diacritic) outputs += [output] alignments += [alignment] - #if random.random() > self.teacher_forcing_probability: + # if random.random() > self.teacher_forcing_probability: diacritic = diacritics[:, time] # use training input - #else: - #diacritic = torch.max(output, 1).indices # use last output + # else: + # diacritic = torch.max(output, 1).indices # use last output alignments = torch.stack(alignments).transpose(0, 1) outputs = torch.stack(outputs).transpose(0, 1).contiguous() @@ -261,14 +258,12 @@ def initialize(self): """Initialize the first step variables""" batch_size = self.encoder_outputs.size(0) src_len = self.encoder_outputs.size(1) - self.attention_hidden = Variable( - torch.zeros(batch_size, self.attention_units) - ).to(self.device) + self.attention_hidden = Variable(torch.zeros(batch_size, self.attention_units)).to( + self.device + ) self.decoder_hiddens = [ Variable(torch.zeros(batch_size, self.decoder_units)).to(self.device) for _ in range(len(self.decoder_rnns)) ] - self.prev_attention = Variable(torch.zeros(batch_size, self.encoder_dim)).to( - self.device - ) + self.prev_attention = Variable(torch.zeros(batch_size, self.encoder_dim)).to(self.device) self.prev_alignment = Variable(torch.zeros(batch_size, src_len)).to(self.device) diff --git a/python/hebrew/models/tacotron_based.py b/python/hebrew/models/tacotron_based.py index 3c02dc8..1e7feec 100644 --- a/python/hebrew/models/tacotron_based.py +++ b/python/hebrew/models/tacotron_based.py @@ -1,5 +1,3 @@ -from typing import List - from models.seq2seq import Decoder as Seq2SeqDecoder from models.seq2seq import Seq2Seq from modules.tacotron_modules import CBHG, Prenet @@ -16,18 +14,16 @@ def __init__( inp_vocab_size: int, embedding_dim: int = 512, use_prenet: bool = True, - prenet_sizes: List[int] = [256, 128], + prenet_sizes: list[int] = [256, 128], cbhg_gru_units: int = 128, cbhg_filters: int = 16, - cbhg_projections: List[int] = [128, 128], + cbhg_projections: list[int] = [128, 128], padding_idx: int = 0, ): super().__init__() self.use_prenet = use_prenet - self.embedding = nn.Embedding( - inp_vocab_size, embedding_dim, padding_idx=padding_idx - ) + self.embedding = nn.Embedding(inp_vocab_size, embedding_dim, padding_idx=padding_idx) if use_prenet: self.prenet = Prenet(embedding_dim, prenet_depth=prenet_sizes) self.cbhg = CBHG( diff --git a/python/hebrew/modules/layers.py b/python/hebrew/modules/layers.py index e135522..e23674d 100644 --- a/python/hebrew/modules/layers.py +++ b/python/hebrew/modules/layers.py @@ -36,40 +36,55 @@ def forward(self, x: Any): x = self.conv1d(x) if self.activation is not None: x = self.activation(x) - #x = self.activation(x) + # x = self.activation(x) x = self.bn(x) return x class LinearNorm(torch.nn.Module): - def __init__(self, in_dim, out_dim, bias=True, w_init_gain='linear'): + def __init__(self, in_dim, out_dim, bias=True, w_init_gain="linear"): super().__init__() self.linear_layer = torch.nn.Linear(in_dim, out_dim, bias=bias) torch.nn.init.xavier_uniform_( - self.linear_layer.weight, - gain=torch.nn.init.calculate_gain(w_init_gain)) + self.linear_layer.weight, gain=torch.nn.init.calculate_gain(w_init_gain) + ) def forward(self, x): return self.linear_layer(x) class ConvNorm(torch.nn.Module): - def __init__(self, in_channels, out_channels, kernel_size=1, stride=1, - padding=None, dilation=1, bias=True, w_init_gain='linear'): + def __init__( + self, + in_channels, + out_channels, + kernel_size=1, + stride=1, + padding=None, + dilation=1, + bias=True, + w_init_gain="linear", + ): super().__init__() if padding is None: - assert(kernel_size % 2 == 1) + assert kernel_size % 2 == 1 padding = int(dilation * (kernel_size - 1) / 2) - self.conv = torch.nn.Conv1d(in_channels, out_channels, - kernel_size=kernel_size, stride=stride, - padding=padding, dilation=dilation, - bias=bias) + self.conv = torch.nn.Conv1d( + in_channels, + out_channels, + kernel_size=kernel_size, + stride=stride, + padding=padding, + dilation=dilation, + bias=bias, + ) torch.nn.init.xavier_uniform_( - self.conv.weight, gain=torch.nn.init.calculate_gain(w_init_gain)) + self.conv.weight, gain=torch.nn.init.calculate_gain(w_init_gain) + ) def forward(self, signal): conv_signal = self.conv(signal) diff --git a/python/hebrew/modules/tacotron_modules.py b/python/hebrew/modules/tacotron_modules.py index 875b924..7e20bce 100644 --- a/python/hebrew/modules/tacotron_modules.py +++ b/python/hebrew/modules/tacotron_modules.py @@ -1,8 +1,8 @@ """ Some custom modules that are used by the TTS model """ + from copy import deepcopy -from typing import List import torch from modules.layers import BatchNormConv1d @@ -18,17 +18,12 @@ class Prenet(nn.Module): in_dim (int): the input dim """ - def __init__( - self, in_dim: int, prenet_depth: List[int] = [256, 128], dropout: int = 0.5 - ): - """ Initializing the prenet module """ + def __init__(self, in_dim: int, prenet_depth: list[int] = [256, 128], dropout: int = 0.5): + """Initializing the prenet module""" super().__init__() in_sizes = [in_dim] + prenet_depth[:-1] self.layers = nn.ModuleList( - [ - nn.Linear(in_size, out_size) - for (in_size, out_size) in zip(in_sizes, prenet_depth) - ] + [nn.Linear(in_size, out_size) for (in_size, out_size) in zip(in_sizes, prenet_depth)] ) self.relu = nn.ReLU() self.dropout = nn.Dropout(dropout) @@ -90,7 +85,7 @@ def __init__( in_dim: int, out_dim: int, K: int, - projections: List[int], + projections: list[int], highway_layers: int = 4, ): """Initializing the CBHG module @@ -108,25 +103,26 @@ def __init__( [ deepcopy( BatchNormConv1d( - in_dim, - in_dim, - kernel_size=k, - stride=1, - padding=k // 2, - activation=self.relu, - )) + in_dim, + in_dim, + kernel_size=k, + stride=1, + padding=k // 2, + activation=self.relu, + ) + ) for k in range(1, K + 1) ] ) k = 2 self.trafo_test = BatchNormConv1d( - in_dim, - in_dim, - kernel_size=k, - stride=1, - padding=k // 2, - activation=self.relu, - ) + in_dim, + in_dim, + kernel_size=k, + stride=1, + padding=k // 2, + activation=self.relu, + ) self.trafo = deepcopy(self.trafo_test) @@ -136,9 +132,11 @@ def __init__( activations = [self.relu] * (len(projections) - 1) + [None] self.conv1d_projections = nn.ModuleList( [ - deepcopy(BatchNormConv1d( - in_size, out_size, kernel_size=3, stride=1, padding=1, activation=ac - )) + deepcopy( + BatchNormConv1d( + in_size, out_size, kernel_size=3, stride=1, padding=1, activation=ac + ) + ) for (in_size, out_size, ac) in zip(in_sizes, projections, activations) ] ) diff --git a/python/hebrew/options.py b/python/hebrew/options.py index 6b850c0..bc65f0b 100644 --- a/python/hebrew/options.py +++ b/python/hebrew/options.py @@ -1,6 +1,7 @@ """ Types of various choices used during training """ + from enum import Enum diff --git a/python/hebrew/run_experiments_wandb.py b/python/hebrew/run_experiments_wandb.py index 0e08df8..fd52824 100644 --- a/python/hebrew/run_experiments_wandb.py +++ b/python/hebrew/run_experiments_wandb.py @@ -1,4 +1,3 @@ - import argparse import random @@ -38,67 +37,50 @@ def train_parser(): # Define Experiments using Wandb sweep_config = { # search method - 'method': 'random', #grid, random + "method": "random", # grid, random # metric and objective - 'metric': { - 'name': 'dec', - 'goal': 'maximize' #'minimize' + "metric": { + "name": "dec", + "goal": "maximize", #'minimize' }, # define search parameters - 'parameters': { - 'max_steps': { - 'values': [1000] - }, - 'batch_size': { - 'values': [32] #[128, 64, 32] - }, - 'cbhg_filters': { - 'values': [16] + "parameters": { + "max_steps": {"values": [1000]}, + "batch_size": { + "values": [32] # [128, 64, 32] }, - 'cbhg_gru_units': { - 'values': [256] - }, - 'cbhg_projections': { - 'values': [[128, 256]] #, [256, 512]] - }, - - 'post_cbhg_layers_units': { - 'values': [[256, 256]] + "cbhg_filters": {"values": [16]}, + "cbhg_gru_units": {"values": [256]}, + "cbhg_projections": { + "values": [[128, 256]] # , [256, 512]] }, - - 'optimizer': { - 'values': ['Adam', 'SGD'] - }, - 'use_prenet': { - 'values': ['false'] - }, - 'prenet_sizes': { - 'values': [[512, 256]] - } - } + "post_cbhg_layers_units": {"values": [[256, 256]]}, + "optimizer": {"values": ["Adam", "SGD"]}, + "use_prenet": {"values": ["false"]}, + "prenet_sizes": {"values": [[512, 256]]}, + }, } # train code, with the search preprocessing logic def train(): - with open('config/train.yml', "rb") as model_yaml: + with open("config/train.yml", "rb") as model_yaml: config = yaml.load(model_yaml) # load default config config_defaults = config - wandb.init(config=config_defaults) # , magic=True) + wandb.init(config=config_defaults) # , magic=True) config_wandb = wandb.config # overwrite initial config - config = { **config, - **config_wandb } + config = {**config, **config_wandb} - tmp_config_path = 'config/sweep_tmp.yml' - with open(tmp_config_path, 'w') as yaml_file: + tmp_config_path = "config/sweep_tmp.yml" + with open(tmp_config_path, "w") as yaml_file: yaml.dump(config, yaml_file, default_flow_style=False) - if args.model_kind in ['baseline',"cbhg"]: + if args.model_kind in ["baseline", "cbhg"]: trainer = CBHGTrainer(tmp_config_path, args.model_kind) else: raise ValueError("The model kind is not supported") @@ -106,7 +88,6 @@ def train(): trainer.run(config_wandb) - ################################## # MAIN # ################################## diff --git a/python/hebrew/setup.py b/python/hebrew/setup.py index f3e3094..0527c31 100644 --- a/python/hebrew/setup.py +++ b/python/hebrew/setup.py @@ -9,42 +9,42 @@ PKG_VERSION = "0.1.0" GIT_TAG = environ.get("GITHUB_REF", "") -TAG_VERSION = re.match(r'^refs/tags/v([0-9]+\.[0-9a-z]+\.[0-9a-z]+)$', GIT_TAG) +TAG_VERSION = re.match(r"^refs/tags/v([0-9]+\.[0-9a-z]+\.[0-9a-z]+)$", GIT_TAG) if TAG_VERSION: PKG_VERSION = TAG_VERSION.group(1) setuptools.setup( - name='rababa', + name="rababa", version=PKG_VERSION, author="Ribose", author_email="open.source@ribose.com", - license='MIT', - description='Rababa for Arabic diacriticization', + license="MIT", + description="Rababa for Arabic diacriticization", # packages=['rababa'], - url='https://www.interscript.org', - python_requires='>=3.6, <4', + url="https://www.interscript.org", + python_requires=">=3.6, <4", project_urls={ - 'Documentation': 'https://github.com/interscript/rababa', - 'Source': 'https://github.com/interscript/rababa', - 'Tracker': 'https://github.com/interscript/rababa/issues', + "Documentation": "https://github.com/interscript/rababa", + "Source": "https://github.com/interscript/rababa", + "Tracker": "https://github.com/interscript/rababa/issues", }, install_requires=[ - 'torch>=1.9.0', - 'numpy', - 'matplotlib', - 'pandas', - 'ruamel.yaml', - 'tensorboard', - 'diacritization-evaluation', - 'tqdm', - 'onnx', - 'onnxruntime', - 'pyyaml', + "torch>=1.9.0", + "numpy", + "matplotlib", + "pandas", + "ruamel.yaml", + "tensorboard", + "diacritization-evaluation", + "tqdm", + "onnx", + "onnxruntime", + "pyyaml", ], # extras_require={'plotting': ['matplotlib>=2.2.0', 'jupyter']}, - setup_requires=['pytest-runner'], - tests_require=['pytest'], + setup_requires=["pytest-runner"], + tests_require=["pytest"], # entry_points={ # 'console_scripts': ['my-command=exampleproject.example:main'] # }, diff --git a/python/hebrew/tester.py b/python/hebrew/tester.py index 2e8366a..4a794af 100644 --- a/python/hebrew/tester.py +++ b/python/hebrew/tester.py @@ -1,4 +1,3 @@ - import torch from config_manager import ConfigManager from dataset import load_iterators @@ -12,9 +11,7 @@ def __init__(self, config_path: str, model_kind: str) -> None: self.config_path = config_path self.model_kind = model_kind - self.config_manager = ConfigManager( - config_path=config_path, model_kind=model_kind - ) + self.config_manager = ConfigManager(config_path=config_path, model_kind=model_kind) self.config = self.config_manager.config self.pad_idx = 0 self.criterion = nn.CrossEntropyLoss(ignore_index=self.pad_idx) @@ -26,11 +23,10 @@ def __init__(self, config_path: str, model_kind: str) -> None: self.device = "cuda" if torch.cuda.is_available() else "cpu" self.model = self.model.to(self.device) - self.load_model(model_path=self.config["test_model_path"], - load_optimizer=False) - self.model, opt, self.global_step = \ - self.config_manager.load_model(model_path=self.config["test_model_path"], - load_optimizer=False) + self.load_model(model_path=self.config["test_model_path"], load_optimizer=False) + self.model, opt, self.global_step = self.config_manager.load_model( + model_path=self.config["test_model_path"], load_optimizer=False + ) self.model = self.model.to(self.device) self.load_diacritizer() self.diacritizer.set_model(self.model) diff --git a/python/hebrew/train.py b/python/hebrew/train.py index 811640f..dba06c6 100644 --- a/python/hebrew/train.py +++ b/python/hebrew/train.py @@ -1,4 +1,3 @@ - import argparse import random @@ -32,7 +31,7 @@ def train_parser(): args = parser.parse_args() -if args.model_kind in ['baseline',"cbhg"]: +if args.model_kind in ["baseline", "cbhg"]: trainer = CBHGTrainer(args.config, args.model_kind) else: raise ValueError("The model kind is not supported") diff --git a/python/hebrew/trainer.py b/python/hebrew/trainer.py index a8edb79..4e1a3eb 100644 --- a/python/hebrew/trainer.py +++ b/python/hebrew/trainer.py @@ -73,10 +73,7 @@ def print_config(self): def load_diacritizer(self): if self.model_kind in ["cbhg", "baseline"]: - load_model = False # True - self.diacritizer = Diacritizer( - self.config_path, self.model_kind - ) # , load_model) + self.diacritizer = Diacritizer(self.config_path, self.model_kind) # , load_model) else: print("model not found") exit() @@ -87,7 +84,6 @@ def print_losses(self, step_results, tqdm): if len(self.losses) > n_steps: d_losses = process_losses(step_results[-n_steps:]) for k in d_losses.keys(): - for i, k in enumerate(d_losses.keys()): tqdm.display( f"{n_steps}-steps average {k}_loss: {d_losses[k]}", @@ -107,7 +103,6 @@ def get_benchmarks(self, test_data_iterator, dims=["N", "D", "S"]): # tqdm d_scores = {} # Run the model on some test examples with torch.no_grad(): - raw_data, dia_data, losses = self.diacritizer.diacritize_data_iterator( test_data_iterator, self.criterion ) @@ -134,9 +129,7 @@ def evaluate_with_error_rates(self, iterator, tqdm): ) orig_path = os.path.join(self.config_manager.prediction_dir, "original.txt") - predicts_path = os.path.join( - self.config_manager.prediction_dir, "predicted.txt" - ) + predicts_path = os.path.join(self.config_manager.prediction_dir, "predicted.txt") f = open(test_path) all_orig = f.readlines() @@ -172,12 +165,9 @@ def run(self, config_wandb=None): print("--------------------------------------") for batch_inputs in repeater(train_iterator): - tqdm.set_description(f"Global Step {self.global_step}") if self.config["use_decay"]: - self.lr = self.adjust_learning_rate( - self.optimizer, global_step=self.global_step - ) + self.lr = self.adjust_learning_rate(self.optimizer, global_step=self.global_step) self.optimizer.zero_grad() batch_inputs.to_device(self.device) @@ -185,30 +175,24 @@ def run(self, config_wandb=None): if self.device == "cuda" and self.config["use_mixed_precision"]: with autocast(): - for k in step_results.keys(): scaler.scale(step_results[k]).backward(retain_graph=True) scaler.unscale_(self.optimizer) if self.config.get("CLIP"): - torch.nn.utils.clip_grad_norm_( - self.model.parameters(), self.config["CLIP"] - ) + torch.nn.utils.clip_grad_norm_(self.model.parameters(), self.config["CLIP"]) scaler.step(self.optimizer) scaler.update() else: - for k in step_results.keys(): step_results[k].backward(retain_graph=True) if self.config.get("CLIP"): - torch.nn.utils.clip_grad_norm_( - self.model.parameters(), self.config["CLIP"] - ) + torch.nn.utils.clip_grad_norm_(self.model.parameters(), self.config["CLIP"]) self.optimizer.step() - dico = { + { "N": float(step_results["N"]), "S": float(step_results["S"]), "D": float(step_results["D"]), @@ -230,21 +214,16 @@ def run(self, config_wandb=None): ) if self.global_step % n_steps_per_epoch == 0: - self.diacritizer.set_model(self.model) d_scores = self.get_benchmarks(validation_iterator) - scores, _ = self.evaluate_with_error_rates( - validation_iterator, tqdm_error_rates - ) + scores, _ = self.evaluate_with_error_rates(validation_iterator, tqdm_error_rates) if config_wandb is not None: - wandb.log({**d_scores, **scores}) print("scores:: ", scores) else: - tqdm.display( f"Evaluate {self.global_step}: N_accu, {d_scores['N_accu']}, N_loss: {d_scores['N_loss']}", pos=8, @@ -269,7 +248,6 @@ def run(self, config_wandb=None): # print('summray_texts:: ', summary_texts) if scores: - """ self.summary_manager.add_scalar( "error_rates/DEC", DEC, global_step=self.global_step) diff --git a/python/hebrew/util/decorators.py b/python/hebrew/util/decorators.py index 71242be..4a1a46c 100644 --- a/python/hebrew/util/decorators.py +++ b/python/hebrew/util/decorators.py @@ -1,4 +1,3 @@ - import traceback from time import time diff --git a/python/hebrew/util/learning_rates.py b/python/hebrew/util/learning_rates.py index 28e4fae..4078856 100644 --- a/python/hebrew/util/learning_rates.py +++ b/python/hebrew/util/learning_rates.py @@ -12,12 +12,13 @@ def __call__(self, global_step) -> float: step = global_step + 1.0 lr = ( self.lr - * self.warmup_steps ** 0.5 - * np.minimum(step * self.warmup_steps ** -1.5, step ** -0.5) + * self.warmup_steps**0.5 + * np.minimum(step * self.warmup_steps**-1.5, step**-0.5) ) return lr + class SquareRootScheduler: def __init__(self, lr=0.1): self.lr = lr @@ -28,9 +29,7 @@ def __call__(self, global_step): class CosineScheduler: - def __init__( - self, max_update, base_lr=0.02, final_lr=0, warmup_steps=0, warmup_begin_lr=0 - ): + def __init__(self, max_update, base_lr=0.02, final_lr=0, warmup_steps=0, warmup_begin_lr=0): self.base_lr_orig = base_lr self.max_update = max_update self.final_lr = final_lr @@ -53,19 +52,14 @@ def __call__(self, global_step): self.base_lr = ( self.final_lr + (self.base_lr_orig - self.final_lr) - * ( - 1 - + math.cos( - math.pi * (global_step - self.warmup_steps) / self.max_steps - ) - ) + * (1 + math.cos(math.pi * (global_step - self.warmup_steps) / self.max_steps)) / 2 ) return self.base_lr + def adjust_learning_rate(optimizer, global_step): lr = LearningRateDecay()(global_step=global_step) for param_group in optimizer.param_groups: param_group["lr"] = lr return lr - diff --git a/python/hebrew/util/nakdimon_dataset.py b/python/hebrew/util/nakdimon_dataset.py index 7ae42ed..397a80e 100644 --- a/python/hebrew/util/nakdimon_dataset.py +++ b/python/hebrew/util/nakdimon_dataset.py @@ -1,5 +1,4 @@ import random -from typing import List, Tuple import numpy as np import torch @@ -60,15 +59,9 @@ def merge_unconditional(texts, tnss, nss, dss, sss): if tn == 0: break sentence.append(t) - sentence.append( - dagesh_table.indices_char[d] if hebrew.can_dagesh(t) else "\uFEFF" - ) - sentence.append( - sin_table.indices_char[s] if hebrew.can_sin(t) else "\uFEFF" - ) - sentence.append( - niqqud_table.indices_char[n] if hebrew.can_niqqud(t) else "\uFEFF" - ) + sentence.append(dagesh_table.indices_char[d] if hebrew.can_dagesh(t) else "\ufeff") + sentence.append(sin_table.indices_char[s] if hebrew.can_sin(t) else "\ufeff") + sentence.append(niqqud_table.indices_char[n] if hebrew.can_niqqud(t) else "\ufeff") res.append("".join(sentence)) return res @@ -103,12 +96,8 @@ def concatenate(others): sin = np.concatenate([x.sin for x in others]) niqqud = np.concatenate([x.niqqud for x in others]) else: - text = np.concatenate( - [x.text for x in others] - ) # torch.cat([x.text for x in others]) - normalized = torch.cat( - [torch.tensor(x.normalized, device=device) for x in others] - ) + text = np.concatenate([x.text for x in others]) # torch.cat([x.text for x in others]) + normalized = torch.cat([torch.tensor(x.normalized, device=device) for x in others]) dagesh = torch.cat([torch.tensor(x.dagesh, device=device) for x in others]) sin = torch.cat([torch.tensor(x.sin, device=device) for x in others]) niqqud = torch.cat([torch.tensor(x.niqqud, device=device) for x in others]) @@ -128,9 +117,7 @@ def size(self): self.shapes()[0][0] def shuffle(self): - utils.shuffle_in_unison( - self.text, self.normalized, self.dagesh, self.niqqud, self.sin - ) + utils.shuffle_in_unison(self.text, self.normalized, self.dagesh, self.niqqud, self.sin) def to_device(self, device): self.normalized = torch.tensor(self.normalized).to(device) @@ -192,12 +179,9 @@ def read_corpora(base_paths): def load_data( corpora, validation_rate: float, maxlen: int, shuffle=True, subtraining_rate=1 -) -> Tuple[Data, Data]: +) -> tuple[Data, Data]: - corpus = [ - (filename, Data.from_text(heb_items, maxlen)) - for (filename, heb_items) in corpora - ] + corpus = [(filename, Data.from_text(heb_items, maxlen)) for (filename, heb_items) in corpora] validation_data = None if validation_rate > 0: @@ -205,7 +189,7 @@ def load_data( size = sum(len(x) for _, x in corpus) validation_size = size * validation_rate validation = [] - validation_filenames: List[str] = [] + validation_filenames: list[str] = [] total_size = 0 while total_size < validation_size: if abs(total_size - validation_size) < abs( diff --git a/python/hebrew/util/nakdimon_hebrew_model.py b/python/hebrew/util/nakdimon_hebrew_model.py index 2b7a971..8de2a72 100644 --- a/python/hebrew/util/nakdimon_hebrew_model.py +++ b/python/hebrew/util/nakdimon_hebrew_model.py @@ -1,63 +1,73 @@ - from collections.abc import Iterable, Iterator from functools import lru_cache -from typing import List, NamedTuple +from typing import NamedTuple # "rafe" denotes a letter to which it would have been valid to add a diacritic of some category # but instead it is decided not to. This makes the metrics less biased. -RAFE = '\u05BF' +RAFE = "\u05bf" class Niqqud: - SHVA = '\u05B0' - REDUCED_SEGOL = '\u05B1' - REDUCED_PATAKH = '\u05B2' - REDUCED_KAMATZ = '\u05B3' - HIRIK = '\u05B4' - TZEIRE = '\u05B5' - SEGOL = '\u05B6' - PATAKH = '\u05B7' - KAMATZ = '\u05B8' - HOLAM = '\u05B9' - KUBUTZ = '\u05BB' - SHURUK = '\u05BC' - METEG = '\u05BD' + SHVA = "\u05b0" + REDUCED_SEGOL = "\u05b1" + REDUCED_PATAKH = "\u05b2" + REDUCED_KAMATZ = "\u05b3" + HIRIK = "\u05b4" + TZEIRE = "\u05b5" + SEGOL = "\u05b6" + PATAKH = "\u05b7" + KAMATZ = "\u05b8" + HOLAM = "\u05b9" + KUBUTZ = "\u05bb" + SHURUK = "\u05bc" + METEG = "\u05bd" -HEBREW_LETTERS = [chr(c) for c in range(0x05d0, 0x05ea + 1)] +HEBREW_LETTERS = [chr(c) for c in range(0x05D0, 0x05EA + 1)] -NIQQUD = [RAFE] + [chr(c) for c in range(0x05b0, 0x05bc + 1)] + ['\u05b7'] +NIQQUD = [RAFE] + [chr(c) for c in range(0x05B0, 0x05BC + 1)] + ["\u05b7"] HOLAM = Niqqud.HOLAM -SHIN_YEMANIT = '\u05c1' -SHIN_SMALIT = '\u05c2' +SHIN_YEMANIT = "\u05c1" +SHIN_SMALIT = "\u05c2" NIQQUD_SIN = [RAFE, SHIN_YEMANIT, SHIN_SMALIT] # RAFE is for acronyms -DAGESH_LETTER = '\u05bc' +DAGESH_LETTER = "\u05bc" DAGESH = [RAFE, DAGESH_LETTER] # note that DAGESH and SHURUK are one and the same ANY_NIQQUD = [RAFE] + NIQQUD[1:] + NIQQUD_SIN[1:] + DAGESH[1:] -VALID_LETTERS = [' ', '!', '"', "'", '(', ')', ',', '-', '.', ':', ';', '?'] + HEBREW_LETTERS -SPECIAL_TOKENS = ['H', 'O', '5'] +VALID_LETTERS = [" ", "!", '"', "'", "(", ")", ",", "-", ".", ":", ";", "?"] + HEBREW_LETTERS +SPECIAL_TOKENS = ["H", "O", "5"] -ENDINGS_TO_REGULAR = dict(zip('ךםןףץ', 'כמנפצ')) +ENDINGS_TO_REGULAR = dict(zip("ךםןףץ", "כמנפצ")) def normalize(c): - if c in VALID_LETTERS: return c - if c in ENDINGS_TO_REGULAR: return ENDINGS_TO_REGULAR[c] - if c in ['\n', '\t']: return ' ' - if c in ['־', '‒', '–', '—', '―', '−']: return '-' - if c == '[': return '(' - if c == ']': return ')' - if c in ['´', '‘', '’']: return "'" - if c in ['“', '”', '״']: return '"' - if c.isdigit(): return '5' - if c == '…': return ',' - if c in ['ײ', 'װ', 'ױ']: return 'H' - return 'O' + if c in VALID_LETTERS: + return c + if c in ENDINGS_TO_REGULAR: + return ENDINGS_TO_REGULAR[c] + if c in ["\n", "\t"]: + return " " + if c in ["־", "‒", "–", "—", "―", "−"]: + return "-" + if c == "[": + return "(" + if c == "]": + return ")" + if c in ["´", "‘", "’"]: + return "'" + if c in ["“", "”", "״"]: + return '"' + if c.isdigit(): + return "5" + if c == "…": + return "," + if c in ["ײ", "װ", "ױ"]: + return "H" + return "O" class HebrewChar(NamedTuple): @@ -74,19 +84,21 @@ def __repr__(self): return repr((self.letter, bool(self.dagesh), bool(self.sin), ord(self.niqqud or chr(0)))) def vocalize(self): - return self._replace(niqqud=vocalize_niqqud(self.niqqud), - sin=self.sin.replace(RAFE, ''), - dagesh=vocalize_dagesh(self.letter, self.dagesh)) + return self._replace( + niqqud=vocalize_niqqud(self.niqqud), + sin=self.sin.replace(RAFE, ""), + dagesh=vocalize_dagesh(self.letter, self.dagesh), + ) -def items_to_text(items: List[HebrewChar]) -> str: - return ''.join(str(item) for item in items).replace(RAFE, '') +def items_to_text(items: list[HebrewChar]) -> str: + return "".join(str(item) for item in items).replace(RAFE, "") def vocalize_dagesh(letter, dagesh): - if letter not in 'בכפ': - return '' - return dagesh.replace(RAFE, '') + if letter not in "בכפ": + return "" + return dagesh.replace(RAFE, "") def vocalize_niqqud(c): @@ -104,25 +116,25 @@ def vocalize_niqqud(c): return Niqqud.SEGOL if c == Niqqud.SHVA: - return '' + return "" - return c.replace(RAFE, '') + return c.replace(RAFE, "") def is_hebrew_letter(letter: str) -> bool: - return '\u05d0' <= letter <= '\u05ea' + return "\u05d0" <= letter <= "\u05ea" def can_dagesh(letter): - return letter in ('בגדהוזטיכלמנספצקשת' + 'ךף') + return letter in ("בגדהוזטיכלמנספצקשת" + "ךף") def can_sin(letter): - return letter == 'ש' + return letter == "ש" def can_niqqud(letter): - return letter in ('אבגדהוזחטיכלמנסעפצקרשת' + 'ךן') + return letter in ("אבגדהוזחטיכלמנסעפצקרשת" + "ךן") def can_any(letter): @@ -131,20 +143,22 @@ def can_any(letter): def iterate_dotted_text(text: str) -> Iterator[HebrewChar]: n = len(text) - text += ' ' + text += " " i = 0 while i < n: letter = text[i] - dagesh = RAFE if can_dagesh(letter) else '' - sin = RAFE if can_sin(letter) else '' - niqqud = RAFE if can_niqqud(letter) else '' + dagesh = RAFE if can_dagesh(letter) else "" + sin = RAFE if can_sin(letter) else "" + niqqud = RAFE if can_niqqud(letter) else "" normalized = normalize(letter) i += 1 - nbrd = text[i - 15:i + 15].split()[1:-1] + nbrd = text[i - 15 : i + 15].split()[1:-1] - assert letter not in ANY_NIQQUD, f'{i}, {nbrd}, {[name_of(c) for word in nbrd for c in word]}' + assert letter not in ANY_NIQQUD, ( + f"{i}, {nbrd}, {[name_of(c) for word in nbrd for c in word]}" + ) if is_hebrew_letter(normalized): if text[i] == DAGESH_LETTER: @@ -159,7 +173,7 @@ def iterate_dotted_text(text: str) -> Iterator[HebrewChar]: # assert niqqud == RAFE, (text[i-5:i+5]) niqqud = text[i] i += 1 - if letter == 'ו' and dagesh == DAGESH_LETTER and niqqud == RAFE: + if letter == "ו" and dagesh == DAGESH_LETTER and niqqud == RAFE: dagesh = RAFE niqqud = DAGESH_LETTER @@ -175,15 +189,15 @@ def split_by_length(characters: Iterable, maxlen: int): space = len(out) out.append(c) if len(out) == maxlen - 1: - yield out[:space+1] - out = out[space+1:] + yield out[: space + 1] + out = out[space + 1 :] if out: yield out def iterate_file(path): - with open(path, encoding='utf-8') as f: - text = ''.join(s + ' ' for s in f.read().split()) + with open(path, encoding="utf-8") as f: + text = "".join(s + " " for s in f.read().split()) try: yield from iterate_dotted_text(text) except AssertionError as ex: @@ -193,26 +207,26 @@ def iterate_file(path): def is_space(c): if isinstance(c, HebrewChar): - return c.letter == ' ' + return c.letter == " " elif isinstance(c, str): - return c == ' ' + return c == " " assert False class Token: - def __init__(self, items: List[HebrewChar]): + def __init__(self, items: list[HebrewChar]): self.items = items def __str__(self): - return ''.join(str(c) for c in self.items) + return "".join(str(c) for c in self.items) def __repr__(self): - return 'Token(' + repr(self.items) + ')' + return "Token(" + repr(self.items) + ")" - def __lt__(self, other: 'Token'): + def __lt__(self, other: "Token"): return (self.to_undotted(), str(self)) < (other.to_undotted(), str(other)) - def strip_nonhebrew(self) -> 'Token': + def strip_nonhebrew(self) -> "Token": start = 0 end = len(self.items) - 1 while True: @@ -223,7 +237,7 @@ def strip_nonhebrew(self) -> 'Token': start += 1 while self.items[end].letter not in HEBREW_LETTERS + ANY_NIQQUD: end -= 1 - return Token(self.items[start:end+1]) + return Token(self.items[start : end + 1]) def __bool__(self): return bool(self.items) @@ -233,19 +247,25 @@ def __eq__(self, other): @lru_cache def to_undotted(self): - return ''.join(str(c.letter) for c in self.items) + return "".join(str(c.letter) for c in self.items) def is_undotted(self): - return len(self.items) > 1 and all(c.niqqud in [RAFE, ''] for c in self.items) + return len(self.items) > 1 and all(c.niqqud in [RAFE, ""] for c in self.items) def is_definite(self): - return len(self.items) > 2 and self.items[0].niqqud == 'הַ'[-1] and self.items[0].letter in 'כבלה' + return ( + len(self.items) > 2 + and self.items[0].niqqud == "הַ"[-1] + and self.items[0].letter in "כבלה" + ) -def tokenize_into(tokens_list: List[Token], char_iterator: Iterator[HebrewChar]) -> Iterator[HebrewChar]: +def tokenize_into( + tokens_list: list[Token], char_iterator: Iterator[HebrewChar] +) -> Iterator[HebrewChar]: current = [] for c in char_iterator: - if c.letter.isspace() or c.letter == '-': + if c.letter.isspace() or c.letter == "-": if current: tokens_list.append(Token(current).strip_nonhebrew()) current = [] @@ -255,7 +275,8 @@ def tokenize_into(tokens_list: List[Token], char_iterator: Iterator[HebrewChar]) if current: tokens_list.append(Token(current).strip_nonhebrew()) -def tokenize(iterator: Iterator[HebrewChar]) -> List[Token]: + +def tokenize(iterator: Iterator[HebrewChar]) -> list[Token]: tokens = [] _ = list(tokenize_into(tokens, iterator)) return tokens diff --git a/python/hebrew/util/nakdimon_metrics.py b/python/hebrew/util/nakdimon_metrics.py index f94c241..4f735a0 100644 --- a/python/hebrew/util/nakdimon_metrics.py +++ b/python/hebrew/util/nakdimon_metrics.py @@ -1,10 +1,8 @@ - from pathlib import Path -from typing import List, Tuple from util import nakdimon_hebrew_model as hebrew -basepath = Path('tests/validation/expected') +basepath = Path("tests/validation/expected") def metric_cha(actual: str, expected: str, *args, **kwargs) -> float: @@ -12,8 +10,9 @@ def metric_cha(actual: str, expected: str, *args, **kwargs) -> float: Calculate character-level agreement between actual and expected. """ actual_hebrew, expected_hebrew = get_items(actual, expected, *args, **kwargs) - return mean_equal((x, y) for x, y in zip(actual_hebrew, expected_hebrew) - if hebrew.can_any(x.letter)) + return mean_equal( + (x, y) for x, y in zip(actual_hebrew, expected_hebrew) if hebrew.can_any(x.letter) + ) def metric_dec(actual: str, expected: str, *args, **kwargs) -> float: @@ -23,14 +22,21 @@ def metric_dec(actual: str, expected: str, *args, **kwargs) -> float: actual_hebrew, expected_hebrew = get_items(actual, expected, *args, **kwargs) return mean_equal( - ((x.niqqud, y.niqqud) for x, y in zip(actual_hebrew, expected_hebrew) - if hebrew.can_niqqud(x.letter)), - - ((x.dagesh, y.dagesh) for x, y in zip(actual_hebrew, expected_hebrew) - if hebrew.can_dagesh(x.letter)), - - ((x.sin, y.sin) for x, y in zip(actual_hebrew, expected_hebrew) - if hebrew.can_sin(x.letter)), + ( + (x.niqqud, y.niqqud) + for x, y in zip(actual_hebrew, expected_hebrew) + if hebrew.can_niqqud(x.letter) + ), + ( + (x.dagesh, y.dagesh) + for x, y in zip(actual_hebrew, expected_hebrew) + if hebrew.can_dagesh(x.letter) + ), + ( + (x.sin, y.sin) + for x, y in zip(actual_hebrew, expected_hebrew) + if hebrew.can_sin(x.letter) + ), ) @@ -50,8 +56,7 @@ def metric_wor(actual: str, expected: str, *args, **kwargs) -> float: # print('מצוי', token_to_text(x)) # print('רצוי', token_to_text(y)) # print() - return mean_equal((x, y) for x, y in zip(actual_tokens, expected_tokens) - if is_hebrew(x)) + return mean_equal((x, y) for x, y in zip(actual_tokens, expected_tokens) if is_hebrew(x)) def mean_equal(*pair_iterables): @@ -67,45 +72,56 @@ def mean_equal(*pair_iterables): def get_diff(actual, expected): for i, (a, e) in enumerate(zip(actual, expected)): if a != e: - return f'\n{actual[i-15:i+15]}\n!=\n{expected[i-15:i+15]}' - return '' + return f"\n{actual[i - 15 : i + 15]}\n!=\n{expected[i - 15 : i + 15]}" + return "" -def get_items(actual: str, expected: str, vocalize=False) -> Tuple[List[hebrew.HebrewChar], List[hebrew.HebrewChar]]: +def get_items( + actual: str, expected: str, vocalize=False +) -> tuple[list[hebrew.HebrewChar], list[hebrew.HebrewChar]]: expected_hebrew = list(hebrew.iterate_dotted_text(expected)) actual_hebrew = list(hebrew.iterate_dotted_text(actual)) if vocalize: expected_hebrew = [x.vocalize() for x in expected_hebrew] actual_hebrew = [x.vocalize() for x in actual_hebrew] - diff = get_diff(repr(''.join(c.letter for c in actual_hebrew)), - repr(''.join(c.letter for c in expected_hebrew))) + diff = get_diff( + repr("".join(c.letter for c in actual_hebrew)), + repr("".join(c.letter for c in expected_hebrew)), + ) assert not diff, diff return actual_hebrew, expected_hebrew def split_to_sentences(text): - return [sent + '.' for sent in text.split('. ') if len(hebrew.remove_niqqud(sent)) > 15] + return [sent + "." for sent in text.split(". ") if len(hebrew.remove_niqqud(sent)) > 15] def clean_read(filename): - with open(filename, encoding='utf8') as f: + with open(filename, encoding="utf8") as f: return cleanup(f.read()) def all_diffs_for_files(expected_filename, system1, system2): expected_sentences = split_to_sentences(clean_read(expected_filename)) - actual_sentences1 = split_to_sentences(clean_read(expected_filename.replace('expected', system1))) - actual_sentences2 = split_to_sentences(clean_read(expected_filename.replace('expected', system2))) + actual_sentences1 = split_to_sentences( + clean_read(expected_filename.replace("expected", system1)) + ) + actual_sentences2 = split_to_sentences( + clean_read(expected_filename.replace("expected", system2)) + ) assert len(expected_sentences) == len(actual_sentences1) == len(actual_sentences2) - triples = [(e, a1, a2) for (e, a1, a2) in zip(expected_sentences, actual_sentences1, actual_sentences2) - if metric_wor(a1, e) < 0.90 or metric_wor(a2, e) < 0.90] + triples = [ + (e, a1, a2) + for (e, a1, a2) in zip(expected_sentences, actual_sentences1, actual_sentences2) + if metric_wor(a1, e) < 0.90 or metric_wor(a2, e) < 0.90 + ] triples.sort(key=lambda e_a1_a2: metric_cha(e_a1_a2[2], e_a1_a2[0])) - for (e, a1, a2) in triples[:20]: + for e, a1, a2 in triples[:20]: print(f"{system1}: {metric_wor(a1, e):.2%}; {system2}: {metric_wor(a2, e):.2%}") - print('סבבה:', a1) - print('מקור:', e) - print('גרוע:', a2) + print("סבבה:", a1) + print("מקור:", e) + print("גרוע:", a2) print() @@ -120,31 +136,37 @@ def collect_failed_words_for_files(system): for file in folder.iterdir(): expected_filename = str(file) expected_sentences = split_to_sentences(clean_read(expected_filename)) - actual_sentences = split_to_sentences(clean_read(expected_filename.replace('expected', system))) + actual_sentences = split_to_sentences( + clean_read(expected_filename.replace("expected", system)) + ) assert len(expected_sentences) == len(actual_sentences) actual_tokens = [token for sentence in actual_sentences for token in sentence.split()] - expected_tokens = [token for sentence in expected_sentences for token in sentence.split()] + expected_tokens = [ + token for sentence in expected_sentences for token in sentence.split() + ] assert len(actual_tokens) == len(expected_tokens) yield from [(x, y) for x, y in zip(expected_tokens, actual_tokens) if x != y] def all_metrics(actual, expected): - return {'dec': metric_dec(actual, expected), - 'cha': metric_cha(actual, expected), - 'wor': metric_wor(actual, expected), - 'voc': metric_wor(actual, expected, vocalize=True)} + return { + "dec": metric_dec(actual, expected), + "cha": metric_cha(actual, expected), + "wor": metric_wor(actual, expected), + "voc": metric_wor(actual, expected, vocalize=True), + } def cleanup(text): - return ' '.join(text.strip().split()) + return " ".join(text.strip().split()) def all_metrics_for_files(actual_filename, expected_filename): - with open(expected_filename, encoding='utf8') as f: + with open(expected_filename, encoding="utf8") as f: expected = cleanup(f.read()) - with open(actual_filename, encoding='utf8') as f: + with open(actual_filename, encoding="utf8") as f: actual = cleanup(f.read()) try: return all_metrics(actual, expected) diff --git a/python/hebrew/util/nakdimon_utils.py b/python/hebrew/util/nakdimon_utils.py index da750f3..d066354 100644 --- a/python/hebrew/util/nakdimon_utils.py +++ b/python/hebrew/util/nakdimon_utils.py @@ -1,14 +1,12 @@ - import contextlib import os import sys from collections.abc import Iterable -from typing import List import numpy as np -def iterate_files(base_paths: Iterable[str]) -> List[str]: +def iterate_files(base_paths: Iterable[str]) -> list[str]: for name in base_paths: if not os.path.isdir(name): yield name @@ -20,20 +18,20 @@ def iterate_files(base_paths: Iterable[str]) -> List[str]: def read_file(filename): - with open(filename, encoding='utf-8') as f: + with open(filename, encoding="utf-8") as f: return f.read() # from: https://stackoverflow.com/a/45735618/2289509 @contextlib.contextmanager -def smart_open(filename: str, mode: str = 'r', *args, **kwargs): +def smart_open(filename: str, mode: str = "r", *args, **kwargs): """Open files and i/o streams transparently.""" - if filename == '-': - if 'r' in mode: + if filename == "-": + if "r" in mode: stream = sys.stdin else: stream = sys.stdout - if 'b' in mode: + if "b" in mode: fh = stream.buffer else: fh = stream @@ -69,7 +67,7 @@ def pad_sequences(sequences, maxlen, dtype, value) -> np.ndarray: if not len(s): continue # empty list/array was found trunc = s[:maxlen] - x[idx, :len(trunc)] = np.asarray(trunc, dtype=dtype) + x[idx, : len(trunc)] = np.asarray(trunc, dtype=dtype) return x diff --git a/python/hebrew/util/text_encoders.py b/python/hebrew/util/text_encoders.py index 3602bec..2aaf94c 100644 --- a/python/hebrew/util/text_encoders.py +++ b/python/hebrew/util/text_encoders.py @@ -1,4 +1,3 @@ - # from util import text_cleaners from util import nakdimon_dataset as dataset @@ -6,7 +5,8 @@ class TextEncoder: def __init__( - self, config=None, # Dict[str, Any] = None, + self, + config=None, # Dict[str, Any] = None, ): self.config = config diff --git a/python/hebrew/util/utils.py b/python/hebrew/util/utils.py index d42469c..c3b2347 100644 --- a/python/hebrew/util/utils.py +++ b/python/hebrew/util/utils.py @@ -1,4 +1,3 @@ - import os from dataclasses import dataclass from itertools import repeat @@ -62,8 +61,7 @@ def get_mask_from_lengths(memory, memory_lengths): def repeater(data_loader): for loader in repeat(data_loader): - for data in loader: - yield data + yield from loader def count_parameters(model): @@ -90,19 +88,16 @@ def get_decoder_layers_attentions(model): return self_attns, src_attens -def display_attention( - attention, path, global_step: int, name="att", n_heads=4, n_rows=2, n_cols=2 -): +def display_attention(attention, path, global_step: int, name="att", n_heads=4, n_rows=2, n_cols=2): assert n_rows * n_cols == n_heads fig = plt.figure(figsize=(15, 15)) for i in range(n_heads): - ax = fig.add_subplot(n_rows, n_cols, i + 1) _attention = attention.squeeze(0)[i].transpose(0, 1).cpu().detach().numpy() - cax = ax.imshow(_attention, aspect="auto", origin="lower", interpolation="none") + ax.imshow(_attention, aspect="auto", origin="lower", interpolation="none") plot_name = f"{global_step}-{name}.png" plt.savefig(os.path.join(path, plot_name), dpi=300, format="png") @@ -113,17 +108,11 @@ def plot_multi_head(model, path, global_step): encoder_attentions = get_encoder_layers_attentions(model) decoder_attentions, attentions = get_decoder_layers_attentions(model) for i in range(len(attentions)): - display_attention( - attentions[0][0], path, global_step, f"encoder-decoder-layer{i + 1}" - ) + display_attention(attentions[0][0], path, global_step, f"encoder-decoder-layer{i + 1}") for i in range(len(decoder_attentions)): - display_attention( - decoder_attentions[0][0], path, global_step, f"decoder-layer{i + 1}" - ) + display_attention(decoder_attentions[0][0], path, global_step, f"decoder-layer{i + 1}") for i in range(len(encoder_attentions)): - display_attention( - encoder_attentions[0][0], path, global_step, f"encoder-layer {i + 1}" - ) + display_attention(encoder_attentions[0][0], path, global_step, f"encoder-layer {i + 1}") def make_src_mask(src, pad_idx=0): @@ -197,9 +186,7 @@ def categorical_accuracy(preds, y, tag_pad_idx, device="cuda"): """ Returns accuracy per batch, i.e. if you get 8/10 right, this returns 0.8, NOT 8 """ - max_preds = preds.argmax( - dim=1, keepdim=True - ) # get the index of the max probability + max_preds = preds.argmax(dim=1, keepdim=True) # get the index of the max probability non_pad_elements = torch.nonzero(y != tag_pad_idx) correct = max_preds[non_pad_elements].squeeze(1).eq(y[non_pad_elements]) return correct.sum() / torch.FloatTensor([y[non_pad_elements].shape[0]]).to(device) From e3c15aabbb047db15bf847bbdead6f8b84f49b7d Mon Sep 17 00:00:00 2001 From: Ronald Tse Date: Wed, 29 Jul 2026 12:38:08 +0800 Subject: [PATCH 5/8] docs: add CONTRIBUTING.md --- CONTRIBUTING.md | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 CONTRIBUTING.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..388ac39 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,42 @@ +# Contributing + +Thanks for your interest in contributing! + +## Development setup + +```bash +git clone +cd +bundle install # Ruby projects +# or +npm ci # JS projects +``` + +## Workflow + +1. Fork → branch from `main` +2. Make changes with tests +3. Run `bundle exec rspec` (Ruby) or `npm test` (JS) locally +4. Run `bundle exec standardrb` (Ruby) or `npm run lint` (JS) +5. Open a PR with a clear description + +## Code style + +- Ruby: enforced by [StandardRB](https://github.com/standardrb/standard) +- JavaScript: enforced by ESLint + Prettier +- Python: enforced by ruff + +## Commit messages + +Use [Conventional Commits](https://www.conventionalcommits.org/): + +``` +feat: add new transliteration system +fix: correct off-by-one in CALT lookup +chore: bump dependencies +docs: clarify README +``` + +## Releases + +Maintainers tag releases following semver. CI publishes on tag push. From f6dfd073fa09f2194ee364cbd49e14de147abbaa Mon Sep 17 00:00:00 2001 From: Ronald Tse Date: Wed, 29 Jul 2026 12:39:48 +0800 Subject: [PATCH 6/8] docs: add CI badge --- README.adoc | 1 + 1 file changed, 1 insertion(+) diff --git a/README.adoc b/README.adoc index ea9e65f..ca9da92 100644 --- a/README.adoc +++ b/README.adoc @@ -1,3 +1,4 @@ +image:https://github.com/interscript/rababa/actions/workflows/ruby.yml/badge.svg["CI status", link="https://github.com/interscript/rababa/actions/workflows/ruby.yml"] = رُبابَة RABABA the Middle-Eastern Language Diacritization Library Middle-Eastern Language diacritization is useful for several practical business From 6320ec90ffe3ee88854d0d71b62ae680068d207c Mon Sep 17 00:00:00 2001 From: Ronald Tse Date: Wed, 29 Jul 2026 12:40:24 +0800 Subject: [PATCH 7/8] docs: add CHANGELOG.md --- CHANGELOG.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 CHANGELOG.md diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..e78fdd7 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,12 @@ +# Changelog + +All notable changes to this project are documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] + +## [Latest] + +See GitHub releases for detailed release notes: https://github.com/interscript/rababa/releases From b811fe2bd402aa42879f51f40bbc7ddc0d219c3f Mon Sep 17 00:00:00 2001 From: Ronald Tse Date: Wed, 29 Jul 2026 12:41:19 +0800 Subject: [PATCH 8/8] ci: add CodeQL workflow for Ruby --- .github/workflows/codeql.yml | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 .github/workflows/codeql.yml diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 0000000..8be1696 --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,27 @@ +name: codeql + +on: + push: + branches: [main] + pull_request: + schedule: + - cron: "0 0 * * 0" # weekly + +permissions: + actions: read + contents: read + security-events: write + +jobs: + analyze: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + language: [ruby] + steps: + - uses: actions/checkout@v7 + - uses: github/codeql-action/init@v3 + with: + languages: ${{ matrix.language }} + - uses: github/codeql-action/analyze@v3