Merge pull request #1147 from tobias-urdin/py38-sqla1.4

Fix Python 3.8, SQLAlchemy 1.4 and MySQL 8

gnocchi/amqp1d.py

@@ -71,7 +71,7 @@ def _flush(self):
                         self.indexer.get_archive_policy_for_metric(name))
             known_metrics = self.indexer.list_metrics(attribute_filter={
                 "and": [{"=": {"resource_id": resource.id}},
-                        {"in": {"name": names}}]
+                        {"in": {"name": list(names)}}]
             })
             known_names = set((m.name for m in known_metrics))
             already_exists_names = []

gnocchi/indexer/alembic/versions/5c4f93e5bb4_mysql_float_to_timestamp.py

@@ -24,6 +24,7 @@
 """
 
 from alembic import op
+from sqlalchemy.engine.reflection import Inspector
 import sqlalchemy as sa
 from sqlalchemy.sql import func
 
@@ -38,8 +39,21 @@
 
 def upgrade():
     bind = op.get_bind()
+    inspector = Inspector.from_engine(bind)
+
     if bind and bind.engine.name == "mysql":
         op.execute("SET time_zone = '+00:00'")
+
+        previous_cks = {"resource": [], "resource_history": []}
+        for table in ("resource", "resource_history"):
+            existing_cks = [
+                c['name'] for c in inspector.get_check_constraints(table)
+            ]
+            ck_name = "ck_{}_started_before_ended".format(table)
+            if ck_name in existing_cks:
+                op.drop_constraint(ck_name, table, type_="check")
+                previous_cks[table].append(ck_name)
+
         # NOTE(jd) So that crappy engine that is MySQL does not have "ALTER
         # TABLE … USING …". We need to copy everything and convert…
         for table_name, column_name in (("resource", "started_at"),
@@ -75,3 +89,7 @@ def upgrade():
                             existing_nullable=nullable,
                             existing_type=existing_type,
                             new_column_name=column_name)
+
+        for table in ("resource", "resource_history"):
+            for ck_name in previous_cks[table]:
+                op.create_check_constraint(ck_name, table, "started_at <= ended_at")

gnocchi/indexer/sqlalchemy.py

@@ -260,27 +260,46 @@ def _safe_execute(self, connection, works):
 class SQLAlchemyIndexer(indexer.IndexerDriver):
     _RESOURCE_TYPE_MANAGER = ResourceClassMapper()
 
+    @staticmethod
+    def _set_url_database(url, database):
+        if hasattr(url, "set"):
+            return url.set(database=database)
+        else:
+            url.database = database
+            return url
+
+    @staticmethod
+    def _set_url_drivername(url, drivername):
+        if hasattr(url, "set"):
+            return url.set(drivername=drivername)
+        else:
+            url.drivername = drivername
+            return url
+
     @classmethod
     def _create_new_database(cls, url):
         """Used by testing to create a new database."""
         purl = sqlalchemy_url.make_url(
             cls.dress_url(
                 url))
-        purl.database = purl.database + str(uuid.uuid4()).replace('-', '')
+        new_database = purl.database + str(uuid.uuid4()).replace('-', '')
+        purl = cls._set_url_database(purl, new_database)
         new_url = str(purl)
         sqlalchemy_utils.create_database(new_url)
         return new_url
 
-    @staticmethod
-    def dress_url(url):
+    @classmethod
+    def dress_url(cls, url):
         # If no explicit driver has been set, we default to pymysql
         if url.startswith("mysql://"):
             url = sqlalchemy_url.make_url(url)
-            url.drivername = "mysql+pymysql"
+            new_drivername = "mysql+pymysql"
+            url = cls._set_url_drivername(url, new_drivername)
             return str(url)
         if url.startswith("postgresql://"):
             url = sqlalchemy_url.make_url(url)
-            url.drivername = "postgresql+psycopg2"
+            new_drivername = "postgresql+psycopg2"
+            url = cls._set_url_drivername(url, new_drivername)
             return str(url)
         return url
 
@@ -1010,8 +1029,6 @@ def delete_resources(self, resource_type='generic',
             target_cls = self._resource_type_to_mappers(
                 session, resource_type)["resource"]
 
-            q = session.query(target_cls.id)
-
             engine = session.connection()
             try:
                 f = QueryTransformer.build_filter(engine.dialect.name,
@@ -1023,12 +1040,16 @@ def delete_resources(self, resource_type='generic',
                 raise indexer.ResourceAttributeError(resource_type,
                                                      e.attribute)
 
-            q = q.filter(f)
+            q1 = session.query(target_cls.id)
+            q1 = q1.filter(f)
 
             session.query(Metric).filter(
-                Metric.resource_id.in_(q)
+                Metric.resource_id.in_(q1)
             ).update({"status": "delete"},
                      synchronize_session=False)
+
+            q = session.query(target_cls)
+            q = q.filter(f)
             return q.delete(synchronize_session=False)
 
     @retry_on_deadlock

gnocchi/indexer/sqlalchemy_base.py

@@ -51,13 +51,17 @@ def process_bind_param(self, value, dialect):
     def process_result_value(self, value, dialect):
         values = super(ArchivePolicyDefinitionType,
                        self).process_result_value(value, dialect)
+        if values is None:
+            return []
         return [archive_policy.ArchivePolicyItem(**v) for v in values]
 
 
 class SetType(sqlalchemy_utils.JSONType):
     def process_result_value(self, value, dialect):
-        return set(super(SetType,
-                         self).process_result_value(value, dialect))
+        values = super(SetType, self).process_result_value(value, dialect)
+        if values is None:
+            return set()
+        return set(values)
 
 
 class ArchivePolicy(Base, GnocchiBase, archive_policy.ArchivePolicy):

gnocchi/rest/auth_helper.py

@@ -118,8 +118,10 @@ def get_metric_policy_filter(request, rule):
 class BasicAuthHelper(object):
     @staticmethod
     def get_current_user(request):
-        auth = werkzeug.http.parse_authorization_header(
-            request.headers.get("Authorization"))
+        hdr = request.headers.get("Authorization")
+        auth_hdr = (hdr.decode('utf-8') if isinstance(hdr, bytes)
+                    else hdr)
+        auth = werkzeug.http.parse_authorization_header(auth_hdr)
         if auth is None:
             api.abort(401)
         return auth.username

setup.cfg

@@ -57,13 +57,13 @@ keystone =
 mysql =
     pymysql
     oslo.db>=4.29.0
-    sqlalchemy<1.4.0
+    sqlalchemy
     sqlalchemy-utils
     alembic>=0.7.6,!=0.8.1,!=0.9.0
 postgresql =
     psycopg2
     oslo.db>=4.29.0
-    sqlalchemy<1.4.0
+    sqlalchemy
     sqlalchemy-utils
     alembic>=0.7.6,!=0.8.1,!=0.9.0
 s3 =

tox.ini

@@ -91,7 +91,8 @@ commands = pifpaf -g GNOCCHI_INDEXER_URL run postgresql -- python setup.py testr
 exclude = .tox,.eggs,doc,gnocchi/rest/prometheus/remote_pb2.py,gnocchi/indexer/alembic/versions/
 show-source = true
 enable-extensions = H904
-ignore = E501,E731,W503,W504
+# TODO(tobias-urdin): Remove H216 when we use unittest.mock
+ignore = E501,E731,W503,W504,H216
 
 [testenv:docs]
 basepython = python3