Satellite el7toel8 upgrade actor

repos/system_upgrade/common/actors/createresumeservice/files/leapp_resume.service

@@ -11,3 +11,5 @@ Type=oneshot
 # FIXME: this is temporary workround for Python3
 ExecStart=/root/tmp_leapp_py3/leapp3 upgrade --resume
 StandardOutput=journal+console
+# FIXME: this shouldn't be needed, but Satellite upgrade runs installer, and that's slow
+TimeoutStartSec=infinity

repos/system_upgrade/common/libraries/testutils.py

@@ -1,6 +1,6 @@
-from collections import namedtuple
 import logging
 import os
+from collections import namedtuple
 
 from leapp.libraries.common.config import architecture
 from leapp.models import EnvVar
@@ -20,14 +20,22 @@ def __call__(self, *model_instances):
 class create_report_mocked(object):
     def __init__(self):
         self.called = 0
-        self.report_fields = {}
+        self.reports = []
 
     def __call__(self, report_fields):
         self.called += 1
+        full_report = {}
         # iterate list of report primitives (classes)
         for report in report_fields:
             # last element of path is our field name
-            self.report_fields.update(report.to_dict())
+            full_report.update(report.to_dict())
+        self.reports.append(full_report)
+
+    @property
+    def report_fields(self):
+        if self.reports:
+            return self.reports[-1]
+        return {}
 
 
 class logger_mocked(object):

repos/system_upgrade/el7toel8/actors/satellite_upgrade_check/actor.py

@@ -0,0 +1,22 @@
+from leapp.actors import Actor
+from leapp.libraries.actor.satellite_upgrade_check import satellite_upgrade_check
+from leapp.models import Report, SatelliteFacts
+from leapp.tags import ChecksPhaseTag, IPUWorkflowTag
+
+
+class SatelliteUpgradeCheck(Actor):
+    """
+    Check state of Satellite system before upgrade
+    """
+
+    name = 'satellite_upgrade_check'
+    consumes = (SatelliteFacts,)
+    produces = (Report,)
+    tags = (IPUWorkflowTag, ChecksPhaseTag)
+
+    def process(self):
+        facts = next(self.consume(SatelliteFacts), None)
+        if not facts or not facts.has_foreman:
+            return
+
+        satellite_upgrade_check(facts)

repos/system_upgrade/el7toel8/actors/satellite_upgrade_check/libraries/satellite_upgrade_check.py

@@ -0,0 +1,53 @@
+import textwrap
+
+from leapp import reporting
+
+
+def satellite_upgrade_check(facts):
+    if facts.postgresql.local_postgresql:
+        if facts.postgresql.old_var_lib_pgsql_data:
+            title = "Old PostgreSQL data found in /var/lib/pgsql/data"
+            summary = """
+            The upgrade wants to move PostgreSQL data to /var/lib/pgsql/data,
+            but this directory already exists on your system.
+            Please make sure /var/lib/pgsql/data doesn't exist prior to the upgrade.
+            """
+            reporting.create_report([
+                reporting.Title(title),
+                reporting.Summary(textwrap.dedent(summary).strip()),
+                reporting.Severity(reporting.Severity.HIGH),
+                reporting.Tags([]),
+                reporting.Flags([reporting.Flags.INHIBITOR])
+            ])
+
+        title = "Satellite PostgreSQL data migration"
+        flags = []
+        severity = reporting.Severity.MEDIUM
+
+        if facts.postgresql.same_partition:
+            summary = "Your PostgreSQL data will be automatically migrated."
+        else:
+            scl_psql_path = '/var/opt/rh/rh-postgresql12/lib/pgsql/data/'
+            if facts.postgresql.space_required > facts.postgresql.space_available:
+                storage_message = """You currently don't have enough free storage to move the data.
+                Automatic moving cannot be performed."""
+                flags = [reporting.Flags.INHIBITOR]
+                severity = reporting.Severity.HIGH
+            else:
+                storage_message = """You currently have enough free storage to move the data.
+                This operation can be performed by the upgrade process."""
+            summary = """
+            Your PostgreSQL data in {} is currently on a dedicated volume.
+            PostgreSQL on RHEL8 expects the data to live in /var/lib/pgsql/data.
+            {}
+            However, instead of moving the data over, you might want to consider manually adapting your mounts,
+            so that the contents of {} are available in /var/lib/pgsql/data.
+            """.format(scl_psql_path, storage_message, scl_psql_path)
+
+        reporting.create_report([
+            reporting.Title(title),
+            reporting.Summary(textwrap.dedent(summary).strip()),
+            reporting.Severity(severity),
+            reporting.Tags([]),
+            reporting.Flags(flags)
+        ])

repos/system_upgrade/el7toel8/actors/satellite_upgrade_check/tests/unit_test_satellite_upgrade_check.py

@@ -0,0 +1,79 @@
+from leapp import reporting
+from leapp.libraries.actor.satellite_upgrade_check import satellite_upgrade_check
+from leapp.libraries.common.testutils import create_report_mocked
+from leapp.models import SatelliteFacts, SatellitePostgresqlFacts
+
+
+def test_old_data(monkeypatch):
+    monkeypatch.setattr(reporting, 'create_report', create_report_mocked())
+
+    satellite_upgrade_check(SatelliteFacts(has_foreman=True,
+                            postgresql=SatellitePostgresqlFacts(local_postgresql=True, old_var_lib_pgsql_data=True)))
+
+    assert reporting.create_report.called == 2
+
+    expected_title = 'Old PostgreSQL data found in /var/lib/pgsql/data'
+    assert next((report for report in reporting.create_report.reports if report.get('title') == expected_title), None)
+
+    expected_title = 'Satellite PostgreSQL data migration'
+    assert next((report for report in reporting.create_report.reports if report.get('title') == expected_title), None)
+
+
+def test_no_old_data(monkeypatch):
+    monkeypatch.setattr(reporting, 'create_report', create_report_mocked())
+
+    satellite_upgrade_check(SatelliteFacts(has_foreman=True,
+                            postgresql=SatellitePostgresqlFacts(local_postgresql=True, old_var_lib_pgsql_data=False)))
+
+    assert reporting.create_report.called == 1
+
+    expected_title = 'Satellite PostgreSQL data migration'
+
+    assert expected_title == reporting.create_report.report_fields['title']
+
+
+def test_same_disk(monkeypatch):
+    monkeypatch.setattr(reporting, 'create_report', create_report_mocked())
+
+    satellite_upgrade_check(SatelliteFacts(has_foreman=True,
+                            postgresql=SatellitePostgresqlFacts(local_postgresql=True, same_partition=True)))
+
+    assert reporting.create_report.called == 1
+
+    expected_title = 'Satellite PostgreSQL data migration'
+    expected_summary = 'Your PostgreSQL data will be automatically migrated.'
+
+    assert expected_title == reporting.create_report.report_fields['title']
+    assert expected_summary == reporting.create_report.report_fields['summary']
+
+
+def test_different_disk_sufficient_storage(monkeypatch):
+    monkeypatch.setattr(reporting, 'create_report', create_report_mocked())
+
+    satellite_upgrade_check(SatelliteFacts(has_foreman=True,
+                            postgresql=SatellitePostgresqlFacts(local_postgresql=True, same_partition=False,
+                                                                space_required=5, space_available=10)))
+
+    assert reporting.create_report.called == 1
+
+    expected_title = 'Satellite PostgreSQL data migration'
+    expected_summary = 'You currently have enough free storage to move the data'
+
+    assert expected_title == reporting.create_report.report_fields['title']
+    assert expected_summary in reporting.create_report.report_fields['summary']
+
+
+def test_different_disk_insufficient_storage(monkeypatch):
+    monkeypatch.setattr(reporting, 'create_report', create_report_mocked())
+
+    satellite_upgrade_check(SatelliteFacts(has_foreman=True,
+                            postgresql=SatellitePostgresqlFacts(local_postgresql=True, same_partition=False,
+                                                                space_required=10, space_available=5)))
+
+    assert reporting.create_report.called == 1
+
+    expected_title = 'Satellite PostgreSQL data migration'
+    expected_summary = "You currently don't have enough free storage to move the data"
+
+    assert expected_title == reporting.create_report.report_fields['title']
+    assert expected_summary in reporting.create_report.report_fields['summary']

repos/system_upgrade/el7toel8/actors/satellite_upgrade_data_migration/actor.py

@@ -0,0 +1,43 @@
+import glob
+import os
+import shutil
+
+from leapp.actors import Actor
+from leapp.models import SatelliteFacts
+from leapp.tags import ApplicationsPhaseTag, IPUWorkflowTag
+
+POSTGRESQL_DATA_PATH = '/var/lib/pgsql/data/'
+POSTGRESQL_SCL_DATA_PATH = '/var/opt/rh/rh-postgresql12/lib/pgsql/data/'
+
+SYSTEMD_WANTS_BASE = '/etc/systemd/system/multi-user.target.wants/'
+SERVICES_TO_DISABLE = ['dynflow-sidekiq@*', 'foreman', 'foreman-proxy',
+                       'httpd', 'postgresql', 'pulpcore-api', 'pulpcore-content',
+                       'pulpcore-worker@*', 'tomcat']
+
+
+class SatelliteUpgradeDataMigration(Actor):
+    """
+    Reconfigure Satellite services and migrate PostgreSQL data
+    """
+
+    name = 'satellite_upgrade_data_migration'
+    consumes = (SatelliteFacts,)
+    produces = ()
+    tags = (IPUWorkflowTag, ApplicationsPhaseTag)
+
+    def process(self):
+        facts = next(self.consume(SatelliteFacts), None)
+        if not facts or not facts.has_foreman:
+            return
+
+        # disable services, will be re-enabled by the installer
+        for service_name in SERVICES_TO_DISABLE:
+            for service in glob.glob(os.path.join(SYSTEMD_WANTS_BASE, '{}.service'.format(service_name))):
+                os.unlink(service)
+
+        if facts.postgresql.local_postgresql and os.path.exists(POSTGRESQL_SCL_DATA_PATH):
+            # remove empty PostgreSQL data from the package
+            if os.path.exists(POSTGRESQL_DATA_PATH):
+                os.rmdir(POSTGRESQL_DATA_PATH)
+            # move PostgreSQL data to the new home
+            shutil.move(POSTGRESQL_SCL_DATA_PATH, POSTGRESQL_DATA_PATH)

repos/system_upgrade/el7toel8/actors/satellite_upgrade_facts/actor.py

@@ -0,0 +1,138 @@
+import os
+
+from leapp.actors import Actor
+from leapp.libraries.common.rpms import has_package
+from leapp.libraries.stdlib import run
+from leapp.models import (
+    DNFWorkaround,
+    InstalledRPM,
+    Module,
+    RepositoriesSetupTasks,
+    RpmTransactionTasks,
+    SatelliteFacts,
+    SatellitePostgresqlFacts
+)
+from leapp.tags import FactsPhaseTag, IPUWorkflowTag
+
+POSTGRESQL_SCL_DATA_PATH = '/var/opt/rh/rh-postgresql12/lib/pgsql/data/'
+
+
+class SatelliteUpgradeFacts(Actor):
+    """
+    Report which Satellite packages require updates and how to handle PostgreSQL data
+    """
+
+    name = 'satellite_upgrade_facts'
+    consumes = (InstalledRPM, )
+    produces = (DNFWorkaround, RepositoriesSetupTasks, RpmTransactionTasks, SatelliteFacts)
+    tags = (IPUWorkflowTag, FactsPhaseTag)
+
+    def process(self):
+        has_foreman = has_package(InstalledRPM, 'foreman') or has_package(InstalledRPM, 'foreman-proxy')
+        if not has_foreman:
+            return
+
+        local_postgresql = has_package(InstalledRPM, 'rh-postgresql12-postgresql-server')
+        postgresql_contrib = has_package(InstalledRPM, 'rh-postgresql12-postgresql-contrib')
+        postgresql_evr = has_package(InstalledRPM, 'rh-postgresql12-postgresql-evr')
+
+        to_remove = ['tfm-runtime', 'tfm-pulpcore-runtime', 'rh-redis5-runtime', 'rh-ruby27-runtime',
+                     'rh-python38-runtime']
+        to_install = ['rubygem-foreman_maintain']
+        modules_to_enable = [Module(name='ruby', stream='2.7')]
+
+        if has_package(InstalledRPM, 'katello'):
+            # enable modules that are needed for Candlepin, which is pulled in by Katello
+            modules_to_enable.append(Module(name='pki-core', stream='10.6'))
+            modules_to_enable.append(Module(name='pki-deps', stream='10.6'))
+            # enable modules that are needed for Pulpcore
+            modules_to_enable.append(Module(name='python38', stream='3.8'))
+            to_install.append('katello')
+
+        if has_package(InstalledRPM, 'rh-redis5-redis'):
+            modules_to_enable.append(Module(name='redis', stream='5'))
+            to_install.append('redis')
+
+        for rpm_pkgs in self.consume(InstalledRPM):
+            for pkg in rpm_pkgs.items:
+                if (pkg.name.startswith('tfm-rubygem-hammer') or pkg.name.startswith('tfm-rubygem-foreman')
+                        or pkg.name.startswith('tfm-rubygem-katello')
+                        or pkg.name.startswith('tfm-rubygem-smart_proxy')):
+                    to_install.append(pkg.name.replace('tfm-rubygem-', 'rubygem-'))
+                elif pkg.name.startswith('tfm-pulpcore-python3-pulp'):
+                    to_install.append(pkg.name.replace('tfm-pulpcore-python3-', 'python38-'))
+                elif pkg.name.startswith('foreman-installer') or pkg.name.startswith('satellite-installer'):
+                    to_install.append(pkg.name)
+
+        on_same_partition = True
+        bytes_required = None
+        bytes_available = None
+        old_pgsql_data = False
+
+        if local_postgresql:
+            """
+            Handle migration of the PostgreSQL legacy-actions files.
+            RPM cannot handle replacement of directories by symlinks by default
+            without the %pretrans scriptlet. As PostgreSQL package is packaged wrong,
+            we have to workround that by migration of the PostgreSQL files
+            before the rpm transaction is processed.
+            """
+            self.produce(
+                DNFWorkaround(
+                    display_name='PostgreSQL symlink fix',
+                    script_path=self.get_tool_path('handle-postgresql-legacy-actions'),
+                )
+            )
+
+            old_pgsql_data = bool(os.path.exists('/var/lib/pgsql/data/') and os.listdir('/var/lib/pgsql/data/')
+                                  and os.path.exists(POSTGRESQL_SCL_DATA_PATH)
+                                  and os.listdir(POSTGRESQL_SCL_DATA_PATH))
+            scl_psql_stat = os.stat(POSTGRESQL_SCL_DATA_PATH)
+            for nonscl_path in ['/var/lib/pgsql/data/', '/var/lib/pgsql/', '/var/lib/', '/']:
+                if os.path.exists(nonscl_path):
+                    nonscl_psql_stat = os.stat(nonscl_path)
+                    break
+
+            if scl_psql_stat.st_dev != nonscl_psql_stat.st_dev:
+                on_same_partition = False
+                # get the current disk usage of the PostgreSQL data
+                scl_du_call = run(['du', '--block-size=1', '--summarize', POSTGRESQL_SCL_DATA_PATH])
+                bytes_required = int(scl_du_call['stdout'].split()[0])
+                # get the current free space on the target partition
+                nonscl_stat = os.statvfs(nonscl_path)
+                bytes_available = nonscl_stat.f_bavail * nonscl_stat.f_frsize
+
+            modules_to_enable.append(Module(name='postgresql', stream='12'))
+            to_remove.append('rh-postgresql12-runtime')
+            to_install.extend(['postgresql', 'postgresql-server'])
+            if postgresql_contrib:
+                to_remove.append('rh-postgresql12-postgresql-contrib')
+                to_install.append('postgresql-contrib')
+            if postgresql_evr:
+                to_remove.append('rh-postgresql12-postgresql-evr')
+                to_install.append('postgresql-evr')
+
+        self.produce(SatelliteFacts(
+            has_foreman=has_foreman,
+            postgresql=SatellitePostgresqlFacts(
+                local_postgresql=local_postgresql,
+                old_var_lib_pgsql_data=old_pgsql_data,
+                same_partition=on_same_partition,
+                space_required=bytes_required,
+                space_available=bytes_available,
+            ),
+        ))
+
+        self.produce(RpmTransactionTasks(
+            to_remove=to_remove,
+            to_install=to_install,
+            modules_to_enable=modules_to_enable
+            )
+        )
+        # technically, this needs to be ansible-2.9-for-rhel-8-x86_64-rpms, but my test
+        # system doesn't have it right now - FIXME
+        # there is no repo for satellite yet, but that also will need enablement
+        # probably should find out what's enabled today and map that nicely?
+        repositories_to_enable = ['ansible-2-for-rhel-8-x86_64-rpms', 'satellite-7.0-for-rhel-8-x86_64-rpms',
+                                  'satellite-maintenance-7.0-for-rhel-8-x86_64-rpms']
+        self.produce(RepositoriesSetupTasks(to_enable=repositories_to_enable))