Skip to content

Commit

Permalink
Rewrite RestrictedPCIsScanner using read_or_fetch
Browse files Browse the repository at this point in the history
  • Loading branch information
@drehak @pirat89
drehak authored and pirat89 committed Apr 23, 2021
1 parent e118411 commit a4eb4dd
Show file tree
Hide file tree
Showing 2 changed files with 36 additions and 272 deletions.
157 changes: 17 additions & 140 deletions ...s/system_upgrade/el7toel8/actors/restrictedpcisscanner/libraries/restrictedpcisscanner.py
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,7 @@
import errno
import json
import os

import urllib3

from leapp.exceptions import StopActorExecutionError
from leapp.libraries.common.config import get_env
from leapp.libraries.common import fetch
from leapp.libraries.stdlib import api
from leapp.models import RestrictedPCIDevices
from leapp.models.fields import ModelViolationError
Expand All @@ -18,75 +14,8 @@
# python2
JSONDecodeError = ValueError

REQUEST_TIMEOUT = 0.5 # TODO Consider moving it to the leapp env var
UNSUPPORTED_DRIVER_NAMES_GLOBAL_FILE = os.path.join(
"/etc/leapp/files", "unsupported_driver_names.json"
)
UNSUPPORTED_PCI_IDS_GLOBAL_FILE = os.path.join(
"/etc/leapp/files", "unsupported_pci_ids.json"
)

# TODO Once the microservice for providing the data about removed hardware
# will be provisioned to the Red Hat costumers, then this variable default
# value should be overwritten
PCIS_HOST_DEFAULT = ""
API_VERSION = "v1"


def get_restricted_pcis(http):
"""
Get data about restricted driver names and unsupported pci_ids online.
The function makes API call to the web service, which provides
these data.
:param urllib3.PoolManager http: pool manager instance
:returns: Tuple[dict, dict]
"""
# TODO this constant should be above the scope of this func.
# However due to the way how actor tested with CurrentActorMocked
# we can't do this.
API_URL = "http://{host_port}/api/{api_version}/".format(
host_port=get_env(
"LEAPP_DEVEL_PCIS_HOST",
default=PCIS_HOST_DEFAULT,
),
api_version=API_VERSION,
)
unsupported_driver_names = http.request(
"GET",
API_URL + "unsupported_driver_names/",
timeout=REQUEST_TIMEOUT,
headers={},
)
unsupported_pci_ids = http.request(
"GET",
API_URL + "unsupported_pci_ids/",
timeout=REQUEST_TIMEOUT,
headers={},
)

unsupported_driver_names = json.loads(
unsupported_driver_names.data.decode("utf-8")
)
unsupported_pci_ids = json.loads(unsupported_pci_ids.data.decode("utf-8"))
return unsupported_driver_names, unsupported_pci_ids


def get_restricted_pcis_offline(driver_names_file, pci_ids_file, _open=open):
"""
Get data about restricted driver names and unsupported pci_ids offline.
The function takes the data from locally stored json files in the given
location.
:param _open: made to make the testing easy
"""
with _open(driver_names_file) as f:
unsupported_driver_names = json.load(f)
with _open(pci_ids_file) as f:
unsupported_pci_ids = json.load(f)
return unsupported_driver_names, unsupported_pci_ids
UNSUPPORTED_DRIVER_NAMES_FILE = "unsupported_driver_names.json"
UNSUPPORTED_PCI_IDS_FILE = "unsupported_pci_ids.json"


def produce_restricted_pcis():
Expand All @@ -99,72 +28,20 @@ def produce_restricted_pcis():
"""
unsupported_driver_names = {"devices": {}}
unsupported_pci_ids = {"devices": {}}
# trying get data in leapp files
try:
api.current_logger().info(
"Trying get restricted PCI data from the /etc/leapp/files..."
)
unsupported_driver_names, unsupported_pci_ids = get_restricted_pcis_offline(
driver_names_file=UNSUPPORTED_DRIVER_NAMES_GLOBAL_FILE,
pci_ids_file=UNSUPPORTED_PCI_IDS_GLOBAL_FILE,
)
api.current_logger().info("Data received from /etc/leapp/files.")
# no files
except EnvironmentError as e:
if e.errno == errno.ENOENT and not (
os.path.islink(UNSUPPORTED_DRIVER_NAMES_GLOBAL_FILE)
or os.path.islink(UNSUPPORTED_PCI_IDS_GLOBAL_FILE)
):
# trying to get files online
try:
api.current_logger().info(
"Trying get restricted PCI data online..."
)
http = urllib3.PoolManager()
(
unsupported_driver_names,
unsupported_pci_ids,
) = get_restricted_pcis(http)
api.current_logger().info(
"Data received from the online source."
)
# mcs is not available or return a bad json format
except (urllib3.exceptions.MaxRetryError, JSONDecodeError):
raise StopActorExecutionError(
message=(
"Can't get the data about restricted PCI devices "
"from any available sources."
)
)
else:
raise StopActorExecutionError(
"The required files are presented in the /etc/leapp/files. "
"However, they can't be read by the system (i.e. "
"broken symlink, IO error)."
)
# files exists, but bad json format
unsupported_driver_names = fetch.read_or_fetch(UNSUPPORTED_DRIVER_NAMES_FILE)
unsupported_pci_ids = fetch.read_or_fetch(UNSUPPORTED_PCI_IDS_FILE)
unsupported_driver_names = json.loads(unsupported_driver_names, encoding="utf-8")
unsupported_pci_ids = json.loads(unsupported_pci_ids, encoding="utf-8")
except (JSONDecodeError, UnicodeDecodeError):
raise StopActorExecutionError("The required files have invalid JSON format and can't be decoded.")

# trying to produce the message from received data
try:
api.produce(RestrictedPCIDevices.create({
"driver_names": tuple(unsupported_driver_names["devices"].values()),
"pci_ids": tuple(unsupported_pci_ids["devices"].values())}))
# bad data format
except (KeyError, AttributeError, TypeError, ModelViolationError):
raise StopActorExecutionError(
"The required files are presented in the /etc/leapp/files. "
"However, they have invalid JSON format and can't be decoded."
)
finally:
# trying to produce the message from received data
try:
api.produce(
RestrictedPCIDevices.create(
{
"driver_names": tuple(
unsupported_driver_names["devices"].values()
),
"pci_ids": tuple(
unsupported_pci_ids["devices"].values()
),
},
)
)
# bad data format
except (KeyError, AttributeError, ModelViolationError):
raise StopActorExecutionError(
"Can't produce RestrictedPCIDevices message. The data are incompatible."
)
"Can't produce RestrictedPCIDevices message. The data are incompatible.")
151 changes: 19 additions & 132 deletions ..._upgrade/el7toel8/actors/restrictedpcisscanner/tests/unit_test_restricted_pcis_scanner.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@

from leapp.exceptions import StopActorExecutionError
from leapp.libraries.actor import restrictedpcisscanner
from leapp.libraries.common import fetch
from leapp.libraries.common.testutils import CurrentActorMocked, produce_mocked
from leapp.libraries.stdlib import api
from leapp.models import RestrictedPCIDevice, RestrictedPCIDevices
Expand Down Expand Up @@ -106,82 +107,25 @@ def json_loads_mock_gen(returns_first, returns_second):

@pytest.mark.parametrize(
(
"mcs_socket",
"mcs_available",
"global_files_exists",
"bad_data",
"json_returns_first",
"json_returns_second",
),
[
# Should use global files
# Correct data
(
"not existing host of the mcs",
False,
True,
False,
unsupported_driver_names_example,
unsupported_pci_ids_example,
),
# Should use global files (because it is preferred)
(
"existing host",
True,
True,
False,
unsupported_driver_names_example,
unsupported_pci_ids_example,
),
# Should use mcs (calls are mocked)
(
"existing host",
True,
False,
False,
unsupported_driver_names_example,
unsupported_pci_ids_example,
),
# Should raise StopActorExecutionError
# Bad data. Should raise StopActorExecutionError
(
"not existing host",
False,
False,
True,
unsupported_driver_names_example,
unsupported_pci_ids_example,
),
# Should use global files, which has bad data. Should raise StopActorExecutionError
(
"not existing host of the mcs",
False,
True,
True,
some_very_bad_data,
some_very_bad_data,
),
# Should use global files, which has bad data. Should raise StopActorExecutionError
# Bad data. Should raise StopActorExecutionError
(
"not existing host of the mcs",
False,
True,
True,
some_bad_data,
some_bad_data,
),
# Should use mcs, which has bad data. Should raise StopActorExecutionError
(
"existing host",
True,
False,
True,
some_very_bad_data,
some_very_bad_data,
),
# Should use mcs, which has bad data. Should raise StopActorExecutionError
(
"existing host",
True,
False,
True,
some_bad_data,
some_bad_data,
Expand All @@ -190,88 +134,31 @@ def json_loads_mock_gen(returns_first, returns_second):
)
def test_basic_restricted_pci_scanner(
monkeypatch,
mcs_socket,
mcs_available,
global_files_exists,
bad_data,
json_returns_first,
json_returns_second,
):
"""Test main actor function."""
monkeypatch.setattr(api, "produce", produce_mocked())
json_loads_mock = json_loads_mock_gen(
json_returns_first, json_returns_second
)
monkeypatch.setattr(
api,
"current_actor",
CurrentActorMocked(
envars={
"LEAPP_DEVEL_PCIS_HOST": mcs_socket,
}
),
fetch,
"read_or_fetch",
blank_fn,
)
monkeypatch.setattr(
restrictedpcisscanner.json,
"loads",
value=next(json_loads_mock),
)
monkeypatch.setattr(api, "produce", produce_mocked())
if global_files_exists:
json_loads_mock = json_loads_mock_gen(
json_returns_first, json_returns_second
)
monkeypatch.setattr(
restrictedpcisscanner.json,
"load",
value=next(json_loads_mock),
)
monkeypatch.setattr(
restrictedpcisscanner,
"get_restricted_pcis_offline",
value=partial(
restrictedpcisscanner.get_restricted_pcis_offline,
# no need to provide any data, because we mocked the
# json.loads
_open=mock_open(read_data=""),
),
)
if bad_data:
with pytest.raises(StopActorExecutionError):
restrictedpcisscanner.produce_restricted_pcis()
return
restrictedpcisscanner.produce_restricted_pcis()

if not global_files_exists and mcs_available:
pool_manager_mock = mock.Mock()
get_restricted_pcis_offline = mock.Mock(
side_effect=FileNotFoundError()
)
json_loads_mock = json_loads_mock_gen(
json_returns_first, json_returns_second
)
# even though FileNotFoundError raised as side effect, the error
# number is None (should be 2), so we have to mock the ENOENT
monkeypatch.setattr(errno, "ENOENT", value=None)

monkeypatch.setattr(
restrictedpcisscanner.urllib3,
"PoolManager",
value=pool_manager_mock,
)
monkeypatch.setattr(
restrictedpcisscanner.json,
"loads",
value=next(json_loads_mock),
)
monkeypatch.setattr(
restrictedpcisscanner,
"get_restricted_pcis_offline",
value=get_restricted_pcis_offline,
)

if bad_data:
with pytest.raises(StopActorExecutionError):
restrictedpcisscanner.produce_restricted_pcis()
return
restrictedpcisscanner.produce_restricted_pcis()

pool_manager_mock.assert_called_once()
if not global_files_exists and not mcs_available:
if bad_data:
with pytest.raises(StopActorExecutionError):
restrictedpcisscanner.produce_restricted_pcis()
return

restrictedpcisscanner.produce_restricted_pcis()
assert len(api.produce.model_instances) == 1
assert isinstance(api.produce.model_instances[0], RestrictedPCIDevices)
assert isinstance(
Expand Down

0 comments on commit a4eb4dd

Please sign in to comment.