Bug fixes for 2.0.0b10 (MontrealCorpusTools#373)

* Fix bug in parsing lda and fmllr

* Fix evaluation mode of G2P trainer

* Disable phone set detection by default and fix model inspection

* Fix transcription bug

* Fix corpus loading crash

docs/source/changelog/changelog_2.0.rst

@@ -10,6 +10,15 @@
 Beta releases
 =============
 
+2.0.0b10
+--------
+
+- Changed the functionality of validating dictionary phones and acoustic model phones so that the aligner will simply ignore pronunciations containing phones not in the acoustic model (and print a warning).  The validator utility will provide further detail on what was ignored.
+- Fixed a bug where evaluation of training G2P models was not actually triggered
+- Refactored PairNGramAligner into the :class:`~montreal_forced_aligner.g2p.trainer.PyniniTrainer` class to improve logging output
+- Changed the starting index of training blocks with the same name. Old behavior was ``sat``, ``sat1``, ``sat2``, etc.  The new behavior is ``sat``, ``sat2``, ``sat3``, etc.
+- Revert a change with how sets, roots and extra questions are handled
+
 2.0.0b9
 -------
 

docs/source/conf.py

@@ -106,6 +106,10 @@
         "Conda installation",
         "https://docs.conda.io/projects/conda/en/latest/user-guide/install/index.html",
     ),
+    "miniconda": (
+        "Miniconda",
+        "https://docs.conda.io/en/latest/miniconda.html",
+    ),
     "conda_forge": ("Conda Forge", "https://conda-forge.org/"),
     "pydata_sphinx_theme": (
         "Pydata Sphinx Theme",

docs/source/getting_started.rst

@@ -33,7 +33,7 @@ Installation
     .. link-button:: https://docs.conda.io/projects/conda/en/latest/user-guide/install/index.html
             :type: url
             :text: Install Conda
-            :classes: btn-block btn-primary btn-navigation stretched-link
+            :classes: btn-block btn-primary btn-navigation
 
 
     ---

docs/source/installation.rst

@@ -13,7 +13,7 @@ Installation
 All platforms
 =============
 
-1. Install Anaconda/Miniconda (https://docs.conda.io/en/latest/miniconda.html)
+1. Install :xref:`miniconda`/:xref:`conda_installation`
 2. Create new environment and install MFA: :code:`conda create -n aligner -c conda-forge montreal-forced-aligner`
 
    a.  You can enable the :code:`conda-forge` channel by default by running :code:`conda config --add channels conda-forge` in order to omit the :code:`-c conda-forge` from these commands
@@ -49,6 +49,27 @@ If the Conda installation above does not work or the binaries don't work on your
 
    You can also clone the conda forge feedstocks for `OpenFst <https://github.com/conda-forge/openfst-feedstock>`_, `SoX <https://github.com/conda-forge/sox-feedstock>`_, `Kaldi <https://github.com/conda-forge/kaldi-feedstock>`_, and `MFA <https://github.com/conda-forge/montreal-forced-aligner-feedstock>`_ and run them with `conda build <https://docs.conda.io/projects/conda-build/en/latest/>`_ to build for your specific system.
 
+Installing via pip
+------------------
+
+To install with pip and install minimal dependencies from conda:
+
+1. Create a conda environment:
+
+   * :fa:`fab fa-linux` Linux/:fa:`fab fa-apple` MacOSX: ``conda create -n aligner kaldi pynini``
+   * :fa:`fab fa-windows` Windows: ``conda create -n aligner kaldi``
+
+2. Activate environment via ``conda activate aligner``
+3. Install MFA
+
+   * From PyPi: ``pip install montreal-forced-aligner``
+   * From :fa:`fab fa-github` GitHub: ``pip install git+https://github.com/MontrealCorpusTools/Montreal-Forced-Aligner.git``
+   * From inside the MFA repository root directory, you can install a local version via:
+
+     * ``pip install -e .``
+     * ``python setup.py install``
+     * ``python setup.py develop``
+
 MFA temporary files
 ===================
 

docs/source/reference/g2p_modeling/helper.rst

@@ -29,6 +29,5 @@ Helper
 .. autosummary::
    :toctree: generated/
 
-   PairNGramAligner
    RandomStartWorker
    RandomStart

montreal_forced_aligner/abc.py

@@ -155,25 +155,49 @@ def __init__(
         self.use_mp = use_mp
         self.dirty = False
 
-    @abstractmethod
     def log_debug(self, message: str) -> None:
-        """Abstract method for logging debug messages"""
-        ...
+        """
+        Print a debug message
+
+        Parameters
+        ----------
+        message: str
+            Debug message to log
+        """
+        print(message)
+
+    def log_error(self, message: str) -> None:
+        """
+        Print an error message
+
+        Parameters
+        ----------
+        message: str
+            Error message to log
+        """
+        print(message)
 
-    @abstractmethod
     def log_info(self, message: str) -> None:
-        """Abstract method for logging info messages"""
-        ...
+        """
+        Print an info message
+
+        Parameters
+        ----------
+        message: str
+            Info message to log
+        """
+        print(message)
 
-    @abstractmethod
     def log_warning(self, message: str) -> None:
-        """Abstract method for logging warning messages"""
-        ...
+        """
+        Print a warning message
 
-    @abstractmethod
-    def log_error(self, message: str) -> None:
-        """Abstract method for logging error messages"""
-        ...
+        Parameters
+        ----------
+        message: str
+            Warning message to log
+        """
+        print(message)
 
     @classmethod
     def extract_relevant_parameters(cls, config: MetaDict) -> Tuple[MetaDict, List[str]]:
@@ -527,6 +551,13 @@ def setup_logger(self):
         """
         from .utils import CustomFormatter, get_mfa_version
 
+        current_version = get_mfa_version()
+        # Remove previous directory if versions are different
+        if os.path.exists(self.worker_config_path):
+            with open(self.worker_config_path, "r") as f:
+                conf = yaml.load(f, Loader=yaml.SafeLoader)
+            if conf.get("version", current_version) != current_version:
+                self.clean = True
         if self.clean:
             shutil.rmtree(self.output_directory, ignore_errors=True)
         os.makedirs(self.workflow_directory, exist_ok=True)
@@ -548,7 +579,7 @@ def setup_logger(self):
             handler.setLevel(logging.INFO)
         handler.setFormatter(CustomFormatter())
         self.logger.addHandler(handler)
-        self.logger.debug(f"Set up logger for MFA version: {get_mfa_version()}")
+        self.logger.debug(f"Set up logger for MFA version: {current_version}")
         if self.clean:
             self.logger.debug("Cleaned previous run")
 

montreal_forced_aligner/acoustic_modeling/trainer.py

@@ -16,7 +16,7 @@
 from montreal_forced_aligner.alignment.base import CorpusAligner
 from montreal_forced_aligner.exceptions import ConfigError, KaldiProcessingError
 from montreal_forced_aligner.helper import parse_old_features
-from montreal_forced_aligner.models import AcousticModel
+from montreal_forced_aligner.models import AcousticModel, DictionaryModel
 from montreal_forced_aligner.utils import log_kaldi_errors
 
 if TYPE_CHECKING:
@@ -35,6 +35,8 @@ class TrainableAligner(CorpusAligner, TopLevelMfaWorker, ModelExporterMixin):
     ----------
     training_configuration : list[tuple[str, dict[str, Any]]]
         Training identifiers and parameters for training blocks
+    detect_phone_set: bool
+        Flag for auto detecting phone sets for use in building triphone trees
 
     See Also
     --------
@@ -59,7 +61,12 @@ class TrainableAligner(CorpusAligner, TopLevelMfaWorker, ModelExporterMixin):
         Training blocks
     """
 
-    def __init__(self, training_configuration: List[Tuple[str, Dict[str, Any]]] = None, **kwargs):
+    def __init__(
+        self,
+        training_configuration: List[Tuple[str, Dict[str, Any]]] = None,
+        detect_phone_set: bool = True,
+        **kwargs,
+    ):
         self.param_dict = {
             k: v
             for k, v in kwargs.items()
@@ -73,6 +80,10 @@ def __init__(self, training_configuration: List[Tuple[str, Dict[str, Any]]] = No
         self.current_trainer = None
         self.current_acoustic_model: Optional[AcousticModel] = None
         super().__init__(**kwargs)
+        if not detect_phone_set:
+            self.dictionary_model = DictionaryModel(
+                self.dictionary_model.path, detect_phone_set=detect_phone_set
+            )
         os.makedirs(self.output_directory, exist_ok=True)
         self.training_configs: Dict[str, AcousticModelTrainingMixin] = {}
         if training_configuration is None:
@@ -203,7 +214,7 @@ def add_config(self, train_type: str, params: MetaDict) -> None:
         p.update(self.param_dict)
         p.update(params)
         identifier = train_type
-        index = 1
+        index = 2
         while identifier in self.training_configs:
             identifier = f"{train_type}_{index}"
             index += 1

montreal_forced_aligner/alignment/pretrained.py

@@ -142,11 +142,16 @@ def __init__(
         self.acoustic_model = AcousticModel(acoustic_model_path)
         kwargs.update(self.acoustic_model.parameters)
         super().__init__(**kwargs)
-        self.phone_set_type = self.acoustic_model.meta["phone_set_type"]
-        self.base_phone_regex = self.acoustic_model.meta["base_phone_regex"]
-        for d in self.dictionary_mapping.values():
-            d.phone_set_type = self.phone_set_type
-            d.base_phone_regex = self.base_phone_regex
+
+    @property
+    def base_phone_regex(self) -> Optional[str]:
+        """Regex pattern for extracting a base phone for the phone set"""
+        return self.acoustic_model.meta["base_phone_regex"]
+
+    @property
+    def phone_set_type(self) -> str:
+        """Phone set type, defaults to 'UNKNOWN', currently only 'ARPA' is supported"""
+        return self.acoustic_model.meta["phone_set_type"]
 
     @property
     def working_directory(self) -> str:
@@ -167,6 +172,12 @@ def setup(self) -> None:
                     "This may cause issues, run with --clean, if you hit an error."
                 )
             self.load_corpus()
+            if self.excluded_pronunciation_count:
+                self.logger.warning(
+                    f"There were {self.excluded_pronunciation_count} pronunciations in the dictionary that"
+                    f"were ignored for containing one of {len(self.excluded_phones)} phones not present in the"
+                    f"trained acoustic model.  Please run `mfa validate` to get more details."
+                )
             self.acoustic_model.validate(self)
             self.acoustic_model.export_model(self.working_directory)
             self.acoustic_model.log_details(self.logger)

montreal_forced_aligner/command_line/mfa.py

@@ -347,6 +347,12 @@ def add_global_options(subparser: argparse.ArgumentParser, textgrid_output: bool
         default="",
         help="Audio directory root to use for finding audio files",
     )
+    train_parser.add_argument(
+        "--enable_detect_phone_set",
+        dest="detect_phone_set",
+        help="Enable auto-detecting phone sets from the dictionary during training",
+        action="store_true",
+    )
     add_global_options(train_parser, textgrid_output=True)
 
     validate_parser = subparsers.add_parser("validate", help="Validate a corpus for use in MFA")
@@ -776,6 +782,16 @@ def add_global_options(subparser: argparse.ArgumentParser, textgrid_output: bool
         "silences and recombines compound words and clitics",
         action="store_true",
     )
+    config_parser.add_argument(
+        "--disable_detect_phone_set",
+        help="Disable auto-detecting phone sets from the dictionary during training",
+        action="store_true",
+    )
+    config_parser.add_argument(
+        "--enable_detect_phone_set",
+        help="Enable auto-detecting phone sets from the dictionary during training",
+        action="store_true",
+    )
     config_parser.add_argument(
         "--disable_terminal_colors", help="Turn off colored text in output", action="store_true"
     )
@@ -888,7 +904,7 @@ def main() -> None:
             run_train_g2p(args, unknown)
         elif args.subcommand == "validate":
             run_validate_corpus(args, unknown)
-        elif args.subcommand == "model":
+        elif args.subcommand in ["model", "models"]:
             run_model(args)
         elif args.subcommand == "train_lm":
             run_train_lm(args, unknown)

montreal_forced_aligner/command_line/model.py

@@ -67,9 +67,10 @@ def download_model(model_type: str, name: str) -> None:
     name: str
         Name of model
     """
-    if name is None:
+    if not name:
         downloadable = "\n".join(f"  - {x}" for x in list_downloadable_models(model_type))
         print(f"Available models to download for {model_type}:\n\n{downloadable}")
+        return
     try:
         model_class = MODEL_TYPES[model_type]
         extension = model_class.extensions[0]
@@ -189,7 +190,7 @@ def validate_args(args: Namespace) -> None:
             raise ModelTypeNotSupportedError(args.model_type, MODEL_TYPES)
         elif args.model_type:
             args.model_type = args.model_type.lower()
-        if args.name is not None:
+        if args.name:
             available_languages = list_downloadable_models(args.model_type)
             if args.name not in available_languages:
                 raise PretrainedModelNotFoundError(args.name, args.model_type, available_languages)

montreal_forced_aligner/command_line/train_acoustic_model.py

@@ -26,7 +26,6 @@ def train_acoustic_model(args: Namespace, unknown_args: Optional[List[str]] = No
     unknown_args: list[str]
         Optional arguments that will be passed to configuration objects
     """
-
     trainer = TrainableAligner(
         corpus_directory=args.corpus_directory,
         dictionary_path=args.dictionary_path,

montreal_forced_aligner/config.py

@@ -135,6 +135,7 @@ def update_global_config(args: Namespace) -> None:
         "terminal_colors": True,
         "terminal_width": 120,
         "cleanup_textgrids": True,
+        "detect_phone_set": False,
         "num_jobs": 3,
         "blas_num_threads": 1,
         "use_mp": True,
@@ -168,6 +169,10 @@ def update_global_config(args: Namespace) -> None:
         default_config["cleanup_textgrids"] = False
     if args.enable_textgrid_cleanup:
         default_config["cleanup_textgrids"] = True
+    if args.disable_detect_phone_set:
+        default_config["detect_phone_set"] = False
+    if args.enable_detect_phone_set:
+        default_config["detect_phone_set"] = True
     if args.disable_terminal_colors:
         default_config["terminal_colors"] = False
     if args.enable_terminal_colors:
@@ -202,6 +207,7 @@ def load_global_config() -> Dict[str, Any]:
         "terminal_colors": True,
         "terminal_width": 120,
         "cleanup_textgrids": True,
+        "detect_phone_set": False,
         "num_jobs": 3,
         "blas_num_threads": 1,
         "use_mp": True,