diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index abec5e6..f059faa 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -16,6 +16,7 @@ jobs: strategy: matrix: mongodb-version: [4.0.18] + mongosh-version: [1.10.6] python-version: [3.8, 3.9] java-version: [1.11] nextflow-version: [21.10.5, 22.10.7] @@ -33,8 +34,8 @@ jobs: wget -qO- https://github.com/nextflow-io/nextflow/releases/download/v${{ matrix.nextflow-version }}/nextflow-${{ matrix.nextflow-version }}-all | bash echo "/tmp/nextflow" >> $GITHUB_PATH cd - - - name: Set up MongoDB sharded environment ${{ matrix.mongodb-version }} - run: sudo bash scripts/create_mongo_single_node_sharded_env.sh ${{ matrix.mongodb-version }} + - name: Set up MongoDB ${{ matrix.mongodb-version }} and Mongosh ${{ matrix.mongosh-version }} sharded environment + run: sudo bash scripts/create_mongo_single_node_sharded_env.sh ${{ matrix.mongodb-version }} ${{ matrix.mongosh-version }} - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v2 with: diff --git a/ebi_eva_internal_pyutils/mongodb/mongo_database.py b/ebi_eva_internal_pyutils/mongodb/mongo_database.py index 6023426..e9843a8 100644 --- a/ebi_eva_internal_pyutils/mongodb/mongo_database.py +++ b/ebi_eva_internal_pyutils/mongodb/mongo_database.py @@ -111,9 +111,7 @@ def shard_collections(self, collections_shard_key_map, collections_to_shard): shard_collection_command = f'sh.shardCollection(' \ f'"{self.db_name}.{collection_name}", ' \ f'{shard_key_repr}, {str(shard_key_uniqueness_flag).lower()})' - sharding_command = f"mongo --host {self.uri} " \ - f"--eval " \ - f"'{shard_collection_command}' " + sharding_command = f"mongosh --eval '{shard_collection_command}' {self.uri} " sharding_command += self._get_optional_secrets_file_stdin() run_command_with_output(f"Sharding collection {collection_name} in the database {self.uri_with_db_name} " f"with key {shard_key_repr}...", sharding_command, diff --git a/scripts/create_mongo_single_node_sharded_env.sh b/scripts/create_mongo_single_node_sharded_env.sh index 38e8718..12041f5 100644 --- a/scripts/create_mongo_single_node_sharded_env.sh +++ b/scripts/create_mongo_single_node_sharded_env.sh @@ -5,11 +5,21 @@ systemctl stop mongod || true apt-get remove mongodb-org -y || true apt-get autoremove --purge || true -# Proceed with specific MongoDB version installation +# Proceed with specific MongoDB and Mongosh version installation export mongodb_version=$1 +export mongosh_version=$2 + +BASEDIR=$PWD wget http://fastdl.mongodb.org/linux/mongodb-linux-x86_64-${mongodb_version}.tgz tar xfz mongodb-linux-x86_64-${mongodb_version}.tgz -cd $PWD/mongodb-linux-x86_64-${mongodb_version}/bin/ +cd $BASEDIR/mongodb-linux-x86_64-${mongodb_version}/bin/ +for f in *; + do ln -s $PWD/$f /usr/bin/; chmod a+x /usr/bin/$f; +done +cd $BASEDIR/ +wget https://downloads.mongodb.com/compass/mongosh-${mongosh_version}-linux-x64.tgz +tar xfz mongosh-${mongosh_version}-linux-x64.tgz +cd $BASEDIR/mongosh-${mongosh_version}-linux-x64/bin/ for f in *; do ln -s $PWD/$f /usr/bin/; chmod a+x /usr/bin/$f; done @@ -19,7 +29,7 @@ done function wait_for_mongo() { # Wait until Mongo is ready to accept connections, exit if this does not happen within 2 minutes COUNTER=0 - until mongo --host $1 --eval "printjson(db.serverStatus())" + until mongosh --host $1 --eval "printjson(db.serverStatus())" do sleep 1 COUNTER=$((COUNTER+1)) @@ -36,11 +46,11 @@ rm -rf /data/mongodb mkdir -p /data/mongodb/shard01 /data/mongodb/config mongod --shardsvr --port 27018 --replSet rs0 --dbpath /data/mongodb/shard01 & wait_for_mongo "localhost:27018" -mongo --port 27018 --eval 'rs.initiate({_id:"rs0", members: [{_id: 1, host: "localhost:27018"}]})' +mongosh --port 27018 --eval 'rs.initiate({_id:"rs0", members: [{_id: 1, host: "localhost:27018"}]})' mongod --configsvr --port 27019 --replSet rs1 --dbpath /data/mongodb/config & wait_for_mongo "localhost:27019" -mongo --port 27019 --eval 'rs.initiate()' +mongosh --port 27019 --eval 'rs.initiate()' mongos --configdb rs1/localhost:27019 --port 27017 & wait_for_mongo "localhost:27017" -mongo --eval 'sh.addShard("rs0/localhost:27018")' +mongosh --eval 'sh.addShard("rs0/localhost:27018")' diff --git a/tests/common/test_assembly.py b/tests/common/test_assembly.py index 2429283..633c809 100644 --- a/tests/common/test_assembly.py +++ b/tests/common/test_assembly.py @@ -135,12 +135,8 @@ def test_get_ncbi_genome_folder_url_and_content_multi(self): # GCA_000001405.10, GCA_000001405.11 ... assembly = NCBIAssembly('GCA_000001405.1', 'Homo sapiens', self.genome_folder) url = 'ftp://ftp.ncbi.nlm.nih.gov/genomes/all/GCA/000/001/405/GCA_000001405.1_GRCh37' - content = ['ALT_REF_LOCI_1', 'ALT_REF_LOCI_2', 'ALT_REF_LOCI_3', 'ALT_REF_LOCI_4', 'ALT_REF_LOCI_5', - 'ALT_REF_LOCI_6', 'ALT_REF_LOCI_7', 'ALT_REF_LOCI_8', 'ALT_REF_LOCI_9', 'Primary_Assembly', - 'special_requests', 'genomic_regions_definitions.txt', 'ASSEMBLY_INFO', 'RepeatMasker.run', - 'windowmasker_nmer.oascii.gz', 'par.txt', 'GCA_000001405.1_GRCh37_assembly_report.txt', - 'GCA_000001405.1_GRCh37_assembly_stats.txt', 'GCA_000001405.1_GRCh37_assembly_regions.txt', - 'assembly_status.txt', 'md5checksums.txt'] + content = ['GCA_000001405.1_GRCh37_assembly_regions.txt', 'GCA_000001405.1_GRCh37_assembly_report.txt', + 'GCA_000001405.1_GRCh37_assembly_stats.txt', 'md5checksums.txt', 'assembly_status.txt'] self.assertEqual(assembly._ncbi_genome_folder_url_and_content, (url, content)) diff --git a/tests/internal/test_mongodb.py b/tests/internal/test_mongodb.py index 4bc4893..ffee198 100644 --- a/tests/internal/test_mongodb.py +++ b/tests/internal/test_mongodb.py @@ -21,8 +21,8 @@ class TestMongoDatabase(TestCommon): def setUp(self) -> None: self.test_mongo_db = MongoDatabase(uri=self.uri, db_name=self.dump_db_name) self.dump_dir = os.path.join(self.resources_folder, self.dump_db_name) - run_command_with_output("Drop target test database if it already exists...", f"mongo {self.dump_db_name} " - f"--eval 'db.dropDatabase()'") + run_command_with_output("Drop target test database if it already exists...", + f"mongosh --eval 'db.dropDatabase()' {self.dump_db_name} ") run_command_with_output("Import test database...", f"mongorestore --dir {self.dump_dir}") def tearDown(self) -> None: