Modularity pt. 1: Include module data in InstalledRPM model

repos/system_upgrade/el7toel8/actors/redhatsignedrpmscanner/tests/test_redhatsignedrpmscanner.py

@@ -81,7 +81,7 @@ def test_katello_pkg_goes_to_signed(current_actor_context):
         RPM(name='katello-ca-consumer-vm-098.example.com',
             version='1.0',
             release='1',
-            epoch='(none)',
+            epoch='0',
             packager='None',
             arch='noarch',
             pgpsig=''),

repos/system_upgrade/el7toel8/actors/rpmscanner/actor.py

@@ -1,7 +1,6 @@
 from leapp.actors import Actor
-from leapp.libraries.actor.rpmscanner import get_package_repository_data
-from leapp.libraries.common.rpms import get_installed_rpms
-from leapp.models import InstalledRPM, RPM
+from leapp.libraries.actor import rpmscanner
+from leapp.models import InstalledRPM
 from leapp.tags import IPUWorkflowTag, FactsPhaseTag
 
 
@@ -18,23 +17,4 @@ class RpmScanner(Actor):
     tags = (IPUWorkflowTag, FactsPhaseTag)
 
     def process(self):
-        output = get_installed_rpms()
-        pkg_repos = get_package_repository_data()
-
-        result = InstalledRPM()
-        for entry in output:
-            entry = entry.strip()
-            if not entry:
-                continue
-            name, version, release, epoch, packager, arch, pgpsig = entry.split('|')
-            repository = pkg_repos.get(name, '')
-            result.items.append(RPM(
-                name=name,
-                version=version,
-                epoch=epoch,
-                packager=packager,
-                arch=arch,
-                release=release,
-                pgpsig=pgpsig,
-                repository=repository))
-        self.produce(result)
+        rpmscanner.process()

repos/system_upgrade/el7toel8/actors/rpmscanner/libraries/rpmscanner.py

@@ -1,6 +1,9 @@
 import warnings
 
 from leapp.exceptions import StopActorExecutionError
+from leapp.libraries.stdlib import api
+from leapp.libraries.common import rpms
+from leapp.models import InstalledRPM, RPM
 
 no_yum = False
 no_yum_warning_msg = "package `yum` is unavailable"
@@ -10,16 +13,16 @@
     no_yum = True
     warnings.warn(no_yum_warning_msg, ImportWarning)
 
+no_dnf = False
+no_dnf_warning_msg = "package `dnf` is unavailable"
+try:
+    import dnf
+except ImportError:
+    no_dnf = True
+    warnings.warn(no_dnf_warning_msg, ImportWarning)
 
-def get_package_repository_data():
-    """ Return dictionary mapping package name with repository from which it was installed.
-    Note:
-        There's no yum module for py3. The dnf module could have been used
-        instead but there's a bug in dnf preventing us to do so:
-        https://bugzilla.redhat.com/show_bug.cgi?id=1789840
-    """
-    if no_yum:
-        raise StopActorExecutionError(message=no_yum_warning_msg)
+
+def _get_package_repository_data_yum():
     yum_base = yum.YumBase()
     pkg_repos = {}
 
@@ -37,3 +40,109 @@ def get_package_repository_data():
             })
 
     return pkg_repos
+
+
+def _get_package_repository_data_dnf():
+    dnf_base = dnf.Base()
+    pkg_repos = {}
+
+    try:
+        dnf_base.fill_sack(load_system_repo=True, load_available_repos=False)
+        for pkg in dnf_base.sack.query():
+            pkg_repos[pkg.name] = pkg._from_repo.lstrip('@')
+    except ValueError as e:
+        if 'locale' not in str(e):  # reraise if error is not related to locales
+            raise e
+        raise StopActorExecutionError(
+            message='Failed to get installed RPM packages because of an invalid locale',
+            details={
+                'hint': 'Please run leapp with a valid locale. ' +
+                        'You can get a list of installed locales by running `locale -a`.'
+            })
+
+    return pkg_repos
+
+
+def get_package_repository_data():
+    """ Return dictionary mapping package name with repository from which it was installed.
+    Note:
+        There's no yum module for py3. The dnf module can be used only on RHEL 8,
+        on RHEL 7 there's a bug in dnf preventing us to do so:
+        https://bugzilla.redhat.com/show_bug.cgi?id=1789840
+    """
+    if not no_yum:
+        return _get_package_repository_data_yum()
+    if not no_dnf:
+        return _get_package_repository_data_dnf()
+    raise StopActorExecutionError(message=no_yum_warning_msg)
+
+
+def get_modules():
+    """
+    Return info about all module streams as a list of libdnf.module.ModulePackage objects.
+    """
+    if no_dnf:
+        return []
+
+    base = dnf.Base()
+    base.read_all_repos()
+    base.fill_sack()
+
+    module_base = dnf.module.module_base.ModuleBase(base)
+    # this method is absent on RHEL 7, in which case there are no modules anyway
+    if 'get_modules' not in dir(module_base):
+        return []
+    return module_base.get_modules('*')[0]
+
+
+def map_modular_rpms_to_modules():
+    """
+    Map modular packages to the module streams they come from.
+    """
+    modules = get_modules()
+    # empty on RHEL 7 because of no modules
+    if not modules:
+        return {}
+    # create a reverse mapping from the RPMS to module streams
+    # key: tuple of 4 strings representing a NVRA (name, version, release, arch) of an RPM
+    # value: tuple of 2 strings representing a module and its stream
+    rpm_streams = {}
+    for module in modules:
+        for rpm in module.getArtifacts():
+            # we transform the NEVRA string into a tuple
+            name, epoch_version, release_arch = rpm.rsplit('-', 2)
+            epoch, version = epoch_version.split(':', 1)
+            release, arch = release_arch.rsplit('.', 1)
+            rpm_key = (name, epoch, version, release, arch)
+            # stream could be int or float, convert it to str just in case
+            rpm_streams[rpm_key] = (module.getName(), str(module.getStream()))
+    return rpm_streams
+
+
+# TODO(drehak) unit tests
+def process():
+    output = rpms.get_installed_rpms()
+    pkg_repos = get_package_repository_data()
+    rpm_streams = map_modular_rpms_to_modules()
+
+    result = InstalledRPM()
+    for entry in output:
+        entry = entry.strip()
+        if not entry:
+            continue
+        name, version, release, epoch, packager, arch, pgpsig = entry.split('|')
+        repository = pkg_repos.get(name, '')
+        rpm_key = (name, epoch, version, release, arch)
+        module, stream = rpm_streams.get(rpm_key, (None, None))
+        result.items.append(RPM(
+            name=name,
+            version=version,
+            epoch=epoch,
+            packager=packager,
+            arch=arch,
+            release=release,
+            pgpsig=pgpsig,
+            repository=repository,
+            module=module,
+            stream=stream))
+    api.produce(result)

repos/system_upgrade/el7toel8/actors/rpmscanner/tests/test_rpmscanner.py

@@ -2,26 +2,191 @@
 
 import pytest
 
-from leapp.models import InstalledRPM
+from leapp.models import InstalledRPM, RPM
 from leapp.snactor.fixture import current_actor_context
+from leapp.libraries.actor import rpmscanner
+from leapp.libraries.common import rpms, testutils
+from leapp.libraries.stdlib import api
 
 no_yum = False
 try:
     import yum
 except ImportError:
     no_yum = True
 
+no_dnf = False
+try:
+    import dnf
+except ImportError:
+    no_dnf = True
+
+
+# real module streams taken from Fedora 31
+ARTIFACTS_AFTERBURN = [
+    'afterburn-0:4.2.0-1.module_f31+6825+8330d585.x86_64',
+    'afterburn-debuginfo-0:4.2.0-1.module_f31+6825+8330d585.x86_64',
+    'rust-afterburn-0:4.2.0-1.module_f31+6825+8330d585.src',
+    'rust-afterburn-debugsource-0:4.2.0-1.module_f31+6825+8330d585.x86_64'
+]
+ARTIFACTS_SUBVERSION_110 = [
+    'mod_dav_svn-0:1.10.6-1.module_f31+5204+aeb0fc0d.x86_64',
+    'mod_dav_svn-debuginfo-0:1.10.6-1.module_f31+5204+aeb0fc0d.x86_64',
+    'python2-subversion-0:1.10.6-1.module_f31+5204+aeb0fc0d.x86_64',
+    'python2-subversion-debuginfo-0:1.10.6-1.module_f31+5204+aeb0fc0d.x86_64',
+    'subversion-0:1.10.6-1.module_f31+5204+aeb0fc0d.src',
+    'subversion-0:1.10.6-1.module_f31+5204+aeb0fc0d.x86_64',
+    'subversion-debuginfo-0:1.10.6-1.module_f31+5204+aeb0fc0d.x86_64',
+    'subversion-debugsource-0:1.10.6-1.module_f31+5204+aeb0fc0d.x86_64',
+    'subversion-devel-0:1.10.6-1.module_f31+5204+aeb0fc0d.x86_64',
+    'subversion-devel-debuginfo-0:1.10.6-1.module_f31+5204+aeb0fc0d.x86_64',
+    'subversion-gnome-0:1.10.6-1.module_f31+5204+aeb0fc0d.x86_64',
+    'subversion-gnome-debuginfo-0:1.10.6-1.module_f31+5204+aeb0fc0d.x86_64',
+    'subversion-javahl-0:1.10.6-1.module_f31+5204+aeb0fc0d.noarch',
+    'subversion-kde-0:1.10.6-1.module_f31+5204+aeb0fc0d.x86_64',
+    'subversion-kde-debuginfo-0:1.10.6-1.module_f31+5204+aeb0fc0d.x86_64',
+    'subversion-libs-0:1.10.6-1.module_f31+5204+aeb0fc0d.x86_64',
+    'subversion-libs-debuginfo-0:1.10.6-1.module_f31+5204+aeb0fc0d.x86_64',
+    'subversion-perl-0:1.10.6-1.module_f31+5204+aeb0fc0d.x86_64',
+    'subversion-perl-debuginfo-0:1.10.6-1.module_f31+5204+aeb0fc0d.x86_64',
+    'subversion-tools-0:1.10.6-1.module_f31+5204+aeb0fc0d.x86_64',
+    'subversion-tools-debuginfo-0:1.10.6-1.module_f31+5204+aeb0fc0d.x86_64'
+]
+ARTIFACTS_SUBVERSION_113 = [
+    'mod_dav_svn-0:1.13.0-1.module_f31+6955+7c448939.x86_64',
+    'mod_dav_svn-debuginfo-0:1.13.0-1.module_f31+6955+7c448939.x86_64',
+    'python2-subversion-0:1.13.0-1.module_f31+6955+7c448939.x86_64',
+    'python2-subversion-debuginfo-0:1.13.0-1.module_f31+6955+7c448939.x86_64',
+    'subversion-0:1.13.0-1.module_f31+6955+7c448939.src',
+    'subversion-0:1.13.0-1.module_f31+6955+7c448939.x86_64',
+    'subversion-debuginfo-0:1.13.0-1.module_f31+6955+7c448939.x86_64',
+    'subversion-debugsource-0:1.13.0-1.module_f31+6955+7c448939.x86_64',
+    'subversion-devel-0:1.13.0-1.module_f31+6955+7c448939.x86_64',
+    'subversion-devel-debuginfo-0:1.13.0-1.module_f31+6955+7c448939.x86_64',
+    'subversion-gnome-0:1.13.0-1.module_f31+6955+7c448939.x86_64',
+    'subversion-gnome-debuginfo-0:1.13.0-1.module_f31+6955+7c448939.x86_64',
+    'subversion-javahl-0:1.13.0-1.module_f31+6955+7c448939.noarch',
+    'subversion-kde-0:1.13.0-1.module_f31+6955+7c448939.x86_64',
+    'subversion-kde-debuginfo-0:1.13.0-1.module_f31+6955+7c448939.x86_64',
+    'subversion-libs-0:1.13.0-1.module_f31+6955+7c448939.x86_64',
+    'subversion-libs-debuginfo-0:1.13.0-1.module_f31+6955+7c448939.x86_64',
+    'subversion-perl-0:1.13.0-1.module_f31+6955+7c448939.x86_64',
+    'subversion-perl-debuginfo-0:1.13.0-1.module_f31+6955+7c448939.x86_64',
+    'subversion-tools-0:1.13.0-1.module_f31+6955+7c448939.x86_64',
+    'subversion-tools-debuginfo-0:1.13.0-1.module_f31+6955+7c448939.x86_64'
+]
+
+
+class ModuleMocked(object):
+    def __init__(self, name, stream, artifacts):
+        self.name = name
+        self.stream = stream
+        self.artifacts = artifacts
+
+    def getName(self):
+        return self.name
 
-@pytest.mark.skipif(
-    sys.version_info.major >= 3,
-    reason=(
-        "There's no yum module for py3. The dnf module could have been used "
-        "instead but there's a bug in dnf preventing us to do so: "
-        "https://bugzilla.redhat.com/show_bug.cgi?id=1789840"
-    ),
-)
-@pytest.mark.skipif(no_yum, reason="yum is unavailable")
+    def getStream(self):
+        return self.stream
+
+    def getArtifacts(self):
+        return self.artifacts
+
+
+MODULES = [
+    ModuleMocked('afterburn', 'rolling', ARTIFACTS_AFTERBURN),
+    ModuleMocked('subversion', '1.10', ARTIFACTS_SUBVERSION_110),
+    ModuleMocked('subversion', '1.13', ARTIFACTS_SUBVERSION_113)
+]
+
+
+@pytest.mark.skipif(no_yum and no_dnf, reason='yum/dnf is unavailable')
 def test_actor_execution(current_actor_context):
     current_actor_context.run()
     assert current_actor_context.consume(InstalledRPM)
     assert current_actor_context.consume(InstalledRPM)[0].items
+
+
+def test_map_modular_rpms_to_modules_empty(monkeypatch):
+    monkeypatch.setattr(rpmscanner, 'get_modules', lambda: [])
+    mapping = rpmscanner.map_modular_rpms_to_modules()
+    assert not mapping
+
+
+def test_map_modular_rpms_to_modules(monkeypatch):
+    monkeypatch.setattr(rpmscanner, 'get_modules', lambda: MODULES)
+    mapping = rpmscanner.map_modular_rpms_to_modules()
+    assert mapping[
+        ('afterburn', '0', '4.2.0', '1.module_f31+6825+8330d585', 'x86_64')
+    ] == ('afterburn', 'rolling')
+    assert mapping[
+        ('subversion', '0', '1.10.6', '1.module_f31+5204+aeb0fc0d', 'x86_64')
+    ] == ('subversion', '1.10')
+    assert mapping[
+        ('subversion', '0', '1.13.0', '1.module_f31+6955+7c448939', 'x86_64')
+    ] == ('subversion', '1.13')
+    assert not mapping.get(('subversion', '0', '1.13.0', '1.module_f31+6955+7c448939', 'noarch'))
+    assert not mapping.get(('subversion', '0', '1.13.1', '1.module_f31+6955+7c448939', 'x86_64'))
+    assert not mapping.get(('subversion', '1', '1.13.0', '1.module_f31+6955+7c448939', 'x86_64'))
+
+
+INSTALLED_RPMS = [
+    ('afterburn|4.2.0|1.module_f31+6825+8330d585|0|Fedora Project|x86_64|'
+     'RSA/SHA256, Wed 16 Oct 2019 12:49:08 AM CEST, Key ID 50cb390b3c3359c4'),
+    ('subversion|1.10.6|1.module_f31+5204+aeb0fc0d|0|Fedora Project|x86_64|'
+     'RSA/SHA256, Thu 25 Jul 2019 01:41:52 PM CEST, Key ID 50cb390b3c3359c4'),
+    # non-modular, epoch
+    ('tcpdump|4.9.3|2.fc31|14|Fedora Project|x86_64|'
+     'RSA/SHA256, Wed 22 Jul 2020 12:25:15 PM CEST, Key ID 50cb390b3c3359c4'),
+    # non-modular, no epoch
+    ('passwd|0.80|7.fc31|0|Fedora Project|x86_64|'
+     'RSA/SHA256, Wed 04 Dec 2019 08:48:43 PM CET, Key ID 50cb390b3c3359c4')
+]
+
+
+PACKAGE_REPOS = {
+    'afterburn': 'repo1',
+    'subversion': 'repo2',
+    'tcpdump': 'repo2'
+}
+
+
+def test_process(monkeypatch):
+    monkeypatch.setattr(rpmscanner, 'get_modules', lambda: MODULES)
+    monkeypatch.setattr(rpmscanner, 'get_package_repository_data', lambda: PACKAGE_REPOS)
+    monkeypatch.setattr(rpms, 'get_installed_rpms', lambda: INSTALLED_RPMS)
+    monkeypatch.setattr(api, 'produce', testutils.produce_mocked())
+
+    rpmscanner.process()
+    assert api.produce.called
+    assert len(api.produce.model_instances) == 1
+    assert isinstance(api.produce.model_instances[0], InstalledRPM)
+    items = {i.name: i for i in api.produce.model_instances[0].items}
+    assert len(items) == 4
+
+    assert items['afterburn'].epoch == '0'
+    assert items['afterburn'].version == '4.2.0'
+    assert items['afterburn'].release == '1.module_f31+6825+8330d585'
+    assert items['afterburn'].arch == 'x86_64'
+    assert items['afterburn'].module == 'afterburn'
+    assert items['afterburn'].stream == 'rolling'
+
+    assert items['subversion'].epoch == '0'
+    assert items['subversion'].version == '1.10.6'
+    assert items['subversion'].release == '1.module_f31+5204+aeb0fc0d'
+    assert items['subversion'].arch == 'x86_64'
+    assert items['subversion'].module == 'subversion'
+    assert items['subversion'].stream == '1.10'
+
+    assert items['tcpdump'].epoch == '14'
+    assert items['tcpdump'].version == '4.9.3'
+    assert items['tcpdump'].release == '2.fc31'
+    assert items['tcpdump'].arch == 'x86_64'
+    assert not items['tcpdump'].module
+    assert not items['tcpdump'].stream
+
+    assert items['passwd'].epoch == '0'
+    assert items['passwd'].version == '0.80'
+    assert items['passwd'].release == '7.fc31'
+    assert items['passwd'].arch == 'x86_64'
+    assert not items['passwd'].module
+    assert not items['passwd'].stream