Use NullPools

montreal_forced_aligner/abc.py

@@ -74,6 +74,7 @@ def run(self) -> typing.Generator:
             self.db_string,
             poolclass=sqlalchemy.NullPool,
             isolation_level="AUTOCOMMIT",
+            logging_name=f"{type(self).__name__}_engine",
             pool_reset_on_return=None,
         ).execution_options(logging_token=f"{type(self).__name__}_engine")
         try:
@@ -335,8 +336,8 @@ def construct_engine(self, **kwargs) -> sqlalchemy.engine.Engine:
         """
         e = sqlalchemy.create_engine(
             self.db_string,
-            pool_size=GLOBAL_CONFIG.current_profile.num_jobs,
-            max_overflow=10,
+            poolclass=sqlalchemy.NullPool,
+            logging_name="main_process_engine",
             **kwargs,
         ).execution_options(logging_token="main_process_engine")
 

montreal_forced_aligner/alignment/multiprocessing.py

@@ -2385,6 +2385,7 @@ def run(self) -> None:
             self.db_string,
             poolclass=sqlalchemy.NullPool,
             pool_reset_on_return=None,
+            logging_name=f"{type(self).__name__}_engine",
             isolation_level="AUTOCOMMIT",
         ).execution_options(logging_token=f"{type(self).__name__}_engine")
         with mfa_open(self.log_path, "w") as log_file, Session(db_engine) as session:

montreal_forced_aligner/command_line/utils.py

@@ -249,6 +249,7 @@ def check_databases(db_name=None) -> None:
                 f"postgresql+psycopg2://localhost:{GLOBAL_CONFIG.current_profile.database_port}/{db_name}",
                 poolclass=sqlalchemy.NullPool,
                 pool_reset_on_return=None,
+                logging_name="check_databases_engine",
                 isolation_level="AUTOCOMMIT",
             ).execution_options(logging_token="check_databases_engine")
             with engine.connect():
@@ -322,8 +323,8 @@ def cleanup_databases() -> None:
     try:
         subprocess.check_call(
             ["pg_ctl", "-D", db_directory, "stop"],
-            stdout=subprocess.DEVNULL,
-            stderr=subprocess.DEVNULL,
+            stdout=subprocess.DEVNULL if not GLOBAL_CONFIG.current_profile.verbose else None,
+            stderr=subprocess.DEVNULL if not GLOBAL_CONFIG.current_profile.verbose else None,
         )
     except Exception:
         pass

montreal_forced_aligner/corpus/features.py

@@ -2167,6 +2167,7 @@ def _run(self) -> typing.Generator[typing.Tuple[int, int, int]]:
             poolclass=sqlalchemy.NullPool,
             pool_reset_on_return=None,
             isolation_level="AUTOCOMMIT",
+            logging_name=f"{type(self).__name__}_engine",
         ).execution_options(logging_token=f"{type(self).__name__}_engine")
         with sqlalchemy.orm.Session(engine) as session, mfa_open(self.log_path, "w") as log_file:
 

montreal_forced_aligner/diarization/multiprocessing.py

@@ -829,6 +829,7 @@ def run(self) -> None:
             poolclass=sqlalchemy.NullPool,
             pool_reset_on_return=None,
             isolation_level="AUTOCOMMIT",
+            logging_name=f"{type(self).__name__}_engine",
         ).execution_options(logging_token=f"{type(self).__name__}_engine")
         with Session(db_engine) as session:
             try:

montreal_forced_aligner/g2p/phonetisaurus_trainer.py

@@ -154,6 +154,7 @@ def run(self) -> None:
             poolclass=sqlalchemy.NullPool,
             pool_reset_on_return=None,
             isolation_level="AUTOCOMMIT",
+            logging_name=f"{type(self).__name__}_engine",
         ).execution_options(logging_token=f"{type(self).__name__}_engine")
         try:
             symbol_table = pynini.SymbolTable()
@@ -362,6 +363,7 @@ def run(self) -> None:
             poolclass=sqlalchemy.NullPool,
             pool_reset_on_return=None,
             isolation_level="AUTOCOMMIT",
+            logging_name=f"{type(self).__name__}_engine",
         ).execution_options(logging_token=f"{type(self).__name__}_engine")
         Session = scoped_session(sessionmaker(bind=engine, autoflush=False, autocommit=False))
         far_reader = pywrapfst.FarReader.open(self.far_path)
@@ -459,6 +461,7 @@ def run(self) -> None:
             poolclass=sqlalchemy.NullPool,
             pool_reset_on_return=None,
             isolation_level="AUTOCOMMIT",
+            logging_name=f"{type(self).__name__}_engine",
         ).execution_options(logging_token=f"{type(self).__name__}_engine")
         try:
             Session = scoped_session(sessionmaker(bind=engine, autoflush=False, autocommit=False))

montreal_forced_aligner/utils.py

@@ -87,6 +87,7 @@ def inspect_database(name: str) -> DatasetType:
             poolclass=sqlalchemy.NullPool,
             pool_reset_on_return=None,
             isolation_level="AUTOCOMMIT",
+            logging_name="inspect_dataset_engine",
         ).execution_options(logging_token="inspect_dataset_engine")
         with Session(engine) as session:
             corpus = session.query(Corpus).first()

tests/conftest.py

@@ -2,6 +2,7 @@
 
 import os
 import shutil
+import time
 
 import mock
 import pytest
@@ -101,6 +102,7 @@ def db_setup(temp_dir, global_config, request):
     check_databases()
 
     def fin():
+        time.sleep(100)
         cleanup_databases()
         remove_databases()
 

tests/test_alignment_pretrained.py

@@ -1,13 +1,10 @@
 import os
 import shutil
 
-import pytest
-
 from montreal_forced_aligner.alignment import PretrainedAligner
 from montreal_forced_aligner.db import PhoneInterval, Utterance, WordInterval, WorkflowType
 
 
-@pytest.mark.skip("skipping redundant tests")
 def test_align_sick(
     english_dictionary,
     english_acoustic_model,

tests/test_commandline_g2p.py

@@ -38,8 +38,7 @@ def test_generate_pretrained(
     check_databases()
     d = MultispeakerDictionary(output_path)
     d.dictionary_setup()
-
-    assert len(d.word_mapping(1)) > 0
+    assert len(d.word_mapping(list(d.dictionary_lookup.values())[0])) > 0
 
 
 def test_generate_pretrained_threshold(
@@ -72,7 +71,7 @@ def test_generate_pretrained_threshold(
     d = MultispeakerDictionary(output_path)
     d.dictionary_setup()
 
-    assert len(d.word_mapping(1)) > 0
+    assert len(d.word_mapping(list(d.dictionary_lookup.values())[0])) > 0
 
 
 def test_train_g2p(
@@ -171,7 +170,7 @@ def test_generate_dict(
     check_databases()
     d = MultispeakerDictionary(dictionary_path=g2p_basic_output)
     d.dictionary_setup()
-    assert len(d.word_mapping()) > 0
+    assert len(d.word_mapping(list(d.dictionary_lookup.values())[0])) > 0
 
 
 def test_generate_dict_phonetisaurus(
@@ -208,7 +207,7 @@ def test_generate_dict_phonetisaurus(
     check_databases()
     d = MultispeakerDictionary(dictionary_path=g2p_basic_phonetisaurus_output)
     d.dictionary_setup()
-    assert len(d.word_mapping()) > 0
+    assert len(d.word_mapping(list(d.dictionary_lookup.values())[0])) > 0
 
 
 def test_generate_dict_text_only(
@@ -246,7 +245,7 @@ def test_generate_dict_text_only(
     check_databases()
     d = MultispeakerDictionary(dictionary_path=g2p_basic_output)
     d.dictionary_setup()
-    assert len(d.word_mapping()) > 0
+    assert len(d.word_mapping(list(d.dictionary_lookup.values())[0])) > 0
 
 
 def test_generate_dict_textgrid(
@@ -284,4 +283,4 @@ def test_generate_dict_textgrid(
     check_databases()
     d = MultispeakerDictionary(dictionary_path=output_file)
     d.dictionary_setup()
-    assert len(d.word_mapping()) > 0
+    assert len(d.word_mapping(list(d.dictionary_lookup.values())[0])) > 0

tests/test_corpus.py

@@ -1,8 +1,6 @@
 import os
 import shutil
 
-import pytest
-
 from montreal_forced_aligner.corpus.acoustic_corpus import (
     AcousticCorpus,
     AcousticCorpusWithPronunciations,
@@ -28,12 +26,11 @@ def test_opus(opus_test_path):
 
 def test_add(basic_corpus_dir, generated_dir, global_config, db_setup):
     output_directory = os.path.join(generated_dir, "corpus_tests")
-    if os.path.exists(output_directory):
-        shutil.rmtree(output_directory, ignore_errors=True)
     global_config.temporary_directory = output_directory
     corpus = AcousticCorpus(
         corpus_directory=basic_corpus_dir,
     )
+    print(corpus.db_string)
     corpus.load_corpus()
     with corpus.session() as session:
         new_speaker = "new_speaker"
@@ -73,7 +70,6 @@ def test_basic_txt(basic_corpus_txt_dir, basic_dict_path, generated_dir, db_setu
     assert corpus.get_feat_dim() == 45
 
 
-@pytest.mark.xfail
 def test_acoustic_from_temp(
     basic_corpus_txt_dir, basic_dict_path, generated_dir, global_config, db_setup
 ):

tests/test_dict.py

@@ -39,7 +39,9 @@ def test_tabbed(tabbed_dict_path, basic_dict_path, generated_dir, global_config,
     tabbed_dictionary.dictionary_setup()
     basic_dictionary = MultispeakerDictionary(dictionary_path=basic_dict_path)
     basic_dictionary.dictionary_setup()
-    assert tabbed_dictionary.word_mapping(1) == basic_dictionary.word_mapping(1)
+    assert tabbed_dictionary.word_mapping(
+        tabbed_dictionary.dictionary_lookup["test_tabbed_dictionary"]
+    ) == basic_dictionary.word_mapping(basic_dictionary.dictionary_lookup["test_basic"])
 
 
 def test_extra_annotations(extra_annotations_path, generated_dir, global_config, db_setup):