Use new machine eflomal implementation

machine/corpora/aligned_word_pair.py

@@ -71,7 +71,7 @@ def format_score(score: float) -> str:
         source_index = "NULL" if self.source_index < 0 else str(self.source_index)
         target_index = "NULL" if self.target_index < 0 else str(self.target_index)
         repr = f"{source_index}-{target_index}"
-        if include_scores and self.translation_score >= 0:
+        if include_scores and (self.translation_score >= 0 or self.alignment_score >= 0):
             repr += f":{format_score(self.translation_score)}"
             if self.alignment_score >= 0:
                 repr += f":{format_score(self.alignment_score)}"

machine/corpora/corpora_utils.py

@@ -22,6 +22,7 @@ def alignment_exception(refs: Iterable[str]) -> RuntimeError:
 def batch(iterable: Iterable[T], batch_size: int) -> Iterable[Sequence[T]]:
     if isinstance(iterable, Sequence) and len(iterable) <= batch_size:
         yield iterable
+        return
 
     batch: List[T] = []
     for item in iterable:

machine/corpora/flatten.py

@@ -33,14 +33,13 @@ def flatten(corpora: Iterable[Corpus]) -> Corpus:
     if len(corpus_list) == 1:
         return corpus_list[0]
 
-    if any(type(corpus_list[0]) != type(corpus) for corpus in corpus_list[1:]):  # noqa: E721
-        raise TypeError("All corpora must be of the same type.")
-
-    if isinstance(corpus_list[0], TextCorpus):
+    if all(isinstance(corpus, TextCorpus) for corpus in corpus_list):
         return _FlattenTextCorpus(cast(List[TextCorpus], corpus_list))
-    if isinstance(corpus_list[0], AlignmentCorpus):
+    if all(isinstance(corpus, AlignmentCorpus) for corpus in corpus_list):
         return _FlattenAlignmentCorpus(cast(List[AlignmentCorpus], corpus_list))
-    return _FlattenParallelTextCorpus(cast(List[ParallelTextCorpus], corpus_list))
+    if all(isinstance(corpus, ParallelTextCorpus) for corpus in corpus_list):
+        return _FlattenParallelTextCorpus(cast(List[ParallelTextCorpus], corpus_list))
+    raise TypeError("All corpora must be of the same type.")
 
 
 class _FlattenTextCorpus(TextCorpus):
@@ -100,7 +99,7 @@ def is_target_tokenized(self) -> bool:
     def count(self, include_empty: bool = True, text_ids: Optional[Iterable[str]] = None) -> int:
         return sum(c.count(include_empty, text_ids) for c in self._corpora)
 
-    def _get_rows(self) -> Generator[ParallelTextRow, None, None]:
+    def _get_rows(self, text_ids: Optional[Iterable[str]] = None) -> Generator[ParallelTextRow, None, None]:
         for corpus in self._corpora:
-            with corpus.get_rows() as rows:
+            with corpus.get_rows(text_ids) as rows:
                 yield from rows

machine/jobs/build_nmt_engine.py

@@ -12,6 +12,7 @@
 from .nmt_engine_build_job import NmtEngineBuildJob
 from .nmt_model_factory import NmtModelFactory
 from .shared_file_service_factory import SharedFileServiceType
+from .thot.thot_word_alignment_model_factory import ThotWordAlignmentModelFactory
 from .translation_file_service import TranslationFileService
 
 # Setup logging
@@ -57,7 +58,9 @@ def clearml_progress(status: ProgressStatus) -> None:
         else:
             raise RuntimeError("The model type is invalid.")
 
-        job = NmtEngineBuildJob(SETTINGS, nmt_model_factory, translation_file_service)
+        job = NmtEngineBuildJob(
+            SETTINGS, nmt_model_factory, translation_file_service, ThotWordAlignmentModelFactory(SETTINGS)
+        )
         train_corpus_size, _ = job.run(progress, check_canceled)
         if task is not None:
             task.get_logger().report_single_value(name="train_corpus_size", value=train_corpus_size)