v1.5 tests, incremental merge, temp table fix

dbt/adapters/exasol/__version__.py

@@ -1,2 +1,2 @@
-version = "1.4.0"
+version = "1.5.0"
 

dbt/adapters/exasol/impl.py

@@ -7,6 +7,7 @@
 from dbt.adapters.sql import SQLAdapter
 from dbt.exceptions import CompilationError
 from dbt.utils import filter_null_values
+from dbt.adapters.base.meta import available
 
 from dbt.adapters.exasol import (ExasolColumn, ExasolConnectionManager,
                                  ExasolRelation)
@@ -85,4 +86,40 @@ def valid_incremental_strategies(self):
         """The set of standard builtin strategies which this adapter supports out-of-the-box.
         Not used to validate custom strategies defined by end users.
         """
-        return ["append", "delete+insert"]
+        return ["append", "merge", "delete+insert"]
+
+    @staticmethod
+    def is_valid_identifier(identifier) -> bool:
+        # The first character should be alphabetic
+        if not identifier[0].isalpha():
+            return False
+        # Rest of the characters is either alphanumeric or any one of the literals '#', '$', '_'
+        idx = 1
+        while idx < len(identifier):
+            identifier_chr = identifier[idx]
+            if not identifier_chr.isalnum() and identifier_chr not in ('#', '$', '_'):
+                return False
+            idx += 1
+        return True
+
+    @available
+    def should_identifier_be_quoted(self,
+                                    identifier,
+                                    models_column_dict=None) -> bool:
+
+        #Check if the naming is valid
+        if not self.is_valid_identifier(identifier):
+            return True
+        #check if the column is set to be quoted in the model config
+        elif models_column_dict and identifier in models_column_dict:
+            return models_column_dict[identifier].get('quote', False)
+        elif models_column_dict and self.quote(identifier) in models_column_dict:
+            return models_column_dict[self.quote(identifier)].get('quote', False)
+        return False
+
+    @available
+    def check_and_quote_identifier(self, identifier, models_column_dict=None) -> str:
+        if self.should_identifier_be_quoted(identifier, models_column_dict):
+            return self.quote(identifier)
+        else:
+            return identifier

dbt/include/exasol/macros/adapters.sql

@@ -21,9 +21,9 @@ ALTER_COLUMN_TYPE_MACRO_NAME = 'alter_column_type'
       lower(table_schema) as [schema],
   	  lower(table_type) as table_type
     from (
-		select table_name,table_schema,'table' as table_type from sys.exa_all_tables
+		select table_name,table_schema,'table' as table_type from sys.exa_user_tables
 		union
-		select view_name, view_schema,'view' from sys.exa_all_views
+		select view_name, view_schema,'view' from sys.exa_user_views
 	  )
     where upper(table_schema) = '{{ schema |upper }}'
 {% endcall %}  
@@ -64,14 +64,39 @@ ALTER_COLUMN_TYPE_MACRO_NAME = 'alter_column_type'
   {{ return(load_result('check_schema_exists').table) }}
 {% endmacro %}
 
-{% macro exasol__create_view_as(relation, sql) -%}
-CREATE OR REPLACE VIEW {{ relation.schema }}.{{ relation.identifier }} 
-{{- persist_view_column_docs(relation, sql) }}
-AS 
+{% macro get_persist_docs_column_list(model_columns, query_columns, view_comment=false) %}
 (
-    {{ sql | indent(4) }}
+  {% for column_name in query_columns %}
+    {{ get_column_comment_sql(column_name, model_columns, view_comment) }}
+    {{- ", " if not loop.last else "" }}
+  {% endfor %}
 )
-{{ persist_view_relation_docs() }}
+{% endmacro %}
+
+{% macro exasol__create_view_as(relation, sql) -%}
+CREATE OR REPLACE VIEW {{ relation.schema }}.{{ relation.identifier }} 
+  {% if config.persist_column_docs() -%}
+    {% set model_columns = model.columns %}
+    {% set query_columns = get_columns_in_query(sql) %}
+    {{ get_persist_docs_column_list(model_columns, query_columns, view_comment=true) }}
+    {% set model_comment = model.description %}
+  {%- endif %}
+    AS 
+    (
+      {{ sql | indent(4) }}
+    )
+    COMMENT IS '{{ model_comment | replace("'", "''")}}'
+{% endmacro %}
+
+{% macro exasol__create_table_as(temporary, relation, sql) -%}
+  CREATE OR REPLACE TABLE {{ relation.schema }}.{{ relation.identifier }} 
+    {% if config.persist_column_docs() -%}
+      {% set model_comment = model.description %}
+    {%- endif %}
+    as (
+      {{ sql }}
+    )
+    COMMENT IS '{{ model_comment | replace("'", "''")}}';
 {% endmacro %}
 
 {% macro exasol__rename_relation(from_relation, to_relation) -%}
@@ -80,11 +105,6 @@ AS
   {%- endcall %}
 {% endmacro %}
 
-{% macro exasol__create_table_as(temporary, relation, sql) -%}
-    CREATE OR REPLACE TABLE {{ relation.schema }}.{{ relation.identifier }} AS 
-    {{ sql }}
-{% endmacro %}
-
 {% macro exasol__truncate_relation(relation) -%}
   {% call statement('truncate_relation') -%}
     truncate table {{ relation | replace('"', '') }}
@@ -122,56 +142,75 @@ AS
 {% endmacro %}
 
 {% macro exasol__alter_relation_comment(relation, relation_comment) -%}
-  {%- set comment = relation_comment | replace("'", '"') %}
-  COMMENT ON {{ relation.type }} {{ relation }} IS '{{ comment }}';
+  {# Comments on views are not supported outside DDL, see https://docs.exasol.com/db/latest/sql/comment.htm#UsageNotes #}
+  {%- if not relation.is_view %}
+    {%- set comment = relation_comment | replace("'", "''") %}
+      COMMENT ON {{ relation.type }} {{ relation }} IS '{{ comment }}';
+  {%- endif %}
 {% endmacro %}
 
-{% macro get_column_comment_sql(column_name, column_dict, apply_comment=false) -%}
-  {% if (column_name|upper in column_dict) -%}
-    {% set matched_column = column_name|upper -%}
-  {% elif (column_name|lower in column_dict) -%}
-    {% set matched_column = column_name|lower -%}
-  {% elif (column_name in column_dict) -%}
-    {% set matched_column = column_name -%}
-  {% else -%}
-    {% set matched_column = None -%}
-  {% endif -%}
-  {% if matched_column -%}
-    {% set comment = column_dict[matched_column]['description'] | replace("'", '"') -%}
-  {% else -%}
-    {% set comment = "" -%}
-  {% endif -%}
-  {{ adapter.quote(column_name) }} {{ "COMMENT" if apply_comment }} IS '{{ comment }}'
-{%- endmacro %}
 
 {% macro exasol__alter_column_comment(relation, column_dict) -%}
+  {# Comments on views are not supported outside DDL, see https://docs.exasol.com/db/latest/sql/comment.htm#UsageNotes #}
+  {%- if not relation.is_view %}
     {% set query_columns = get_columns_in_query(sql) %} 
     COMMENT ON {{ relation.type }} {{ relation }} (
     {% for column_name in query_columns %}
-        {{ get_column_comment_sql(column_name, column_dict) }} {{- ',' if not loop.last }}
+      {{ get_column_comment_sql(column_name, column_dict) }} {{- ',' if not loop.last }}
     {% endfor %}
     );
-{% endmacro %}
-
-{% macro persist_view_column_docs(relation, sql) %}
-{%- if config.persist_column_docs() %}
-(
-  {% set query_columns = get_columns_in_query(sql) %}
-  {%- for column_name in query_columns %}
-      {{ get_column_comment_sql(column_name, model.columns, true) -}}{{ ',' if not loop.last -}}
-  {%- endfor %}
-)
-{%- endif %}
-{%- endmacro %}
-
-{% macro persist_view_relation_docs() %}
-{%- if config.persist_relation_docs() %}
-COMMENT IS '{{ model.description }}'
-{%- endif %}
+  {%- endif %}
 {% endmacro %}
 
 {% macro exasol__alter_column_type(relation, column_name, new_column_type) -%}
   {% call statement('alter_column_type') %}
     alter table {{ relation }} modify column {{ adapter.quote(column_name) }} {{ new_column_type }};
   {% endcall %}
+{% endmacro %}
+
+{% macro get_column_comment_sql(column_name, column_dict, view_comment=false) -%}
+  {% if (column_name|upper in column_dict) -%}
+    {% set matched_column = column_name|upper -%}
+  {% elif (column_name|lower in column_dict) -%}
+    {% set matched_column = column_name|lower -%}
+  {% elif (column_name in column_dict) -%}
+    {% set matched_column = column_name -%}
+  {% else -%}
+    {% set matched_column = None -%}
+  {% endif -%}
+  {% if not matched_column -%}
+    {{ adapter.quote(column_name) }} {{"COMMENT" if view_comment}} IS ''
+  {%- else -%}
+    {{ adapter.quote(column_name) }} {{"COMMENT" if view_comment}}  IS '{{ column_dict[matched_column]['description'] | replace("'", "''") }}'
+  {%- endif -%}
+{% endmacro %}
+
+{% macro exasol__alter_relation_add_remove_columns(relation, add_columns, remove_columns) %}
+
+  {% if add_columns is none %}
+    {% set add_columns = [] %}
+  {% endif %}
+  {% if remove_columns is none %}
+    {% set remove_columns = [] %}
+  {% endif %}
+
+
+
+            {% for column in add_columns %}
+              {% set sql -%}
+                  alter {{ relation.type }} {{ relation }} add column {{ column.name }} {{ column.data_type }};
+              {%- endset -%}
+              {% do run_query(sql) %}
+            {% endfor %}
+
+            {% for column in remove_columns %}
+              {% set sql -%}
+                 alter {{ relation.type }} {{ relation }} drop column {{ column.name }};
+              {%- endset -%}
+              {% do run_query(sql) %}
+            {% endfor %}
+
+
+
+
 {% endmacro %}

dbt/include/exasol/macros/catalog.sql

@@ -13,7 +13,7 @@
 		  	OBJECT_NAME as table_name,
 		  	ROOT_NAME as table_owner,
 		  	OBJECT_TYPE AS table_type,
-		  	OBJECT_COMMENT
+		  	OBJECT_COMMENT as table_comment
 		  from sys.EXA_USER_OBJECTS 
 		  WHERE OBJECT_TYPE IN('TABLE', 'VIEW')
 
@@ -34,7 +34,8 @@
     )
 
     select tabs.table_owner as [table_owner],
-		   tabs.table_type AS [table_type],
+		       tabs.table_type AS [table_type],
+           tabs.table_comment as [table_comment],
            cols.table_database as [table_database],
            cols.table_schema as [table_schema],
            cols.table_name as [table_name],

dbt/include/exasol/macros/materializations/incremental.sql

@@ -1,82 +1,63 @@
+{% materialization incremental, adapter='exasol', supported_languages=['sql'] %}
 
-{% materialization incremental, adapter='exasol' -%}
-
-  -- relations
-  {%- set existing_relation = load_relation(this) -%}
-
-  {%- set target_relation = this.incorporate(type='table') -%}
-  {%- set temp_relation = make_temp_relation(target_relation)-%}
-  {%- set intermediate_relation = make_intermediate_relation(target_relation)-%}
-  {%- set backup_relation_type = 'table' if existing_relation is none else existing_relation.type -%}
-  {%- set backup_relation = make_backup_relation(target_relation, backup_relation_type) -%}
-
-  -- configs
-  {%- set unique_key = config.get('unique_key') -%}
-  {%- set full_refresh_mode = (should_full_refresh()  or existing_relation.is_view) -%}
-  {%- set on_schema_change = incremental_validate_on_schema_change(config.get('on_schema_change'), default='ignore') -%}
-
-  -- the temp_ and backup_ relations should not already exist in the database; get_relation
-  -- will return None in that case. Otherwise, we get a relation that we can drop
-  -- later, before we try to use this name for the current operation. This has to happen before
-  -- BEGIN, in a separate transaction
-  {%- set preexisting_intermediate_relation = load_cached_relation(intermediate_relation)-%}
-  {%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%}
-   -- grab current tables grants config for comparision later on
-  {% set grant_config = config.get('grants') %}
-  {{ drop_relation_if_exists(preexisting_intermediate_relation) }}
-  {{ drop_relation_if_exists(preexisting_backup_relation) }}
+  {% set unique_key = config.get('unique_key') %}
+  {% set full_refresh_mode = flags.FULL_REFRESH %}
+  {%- set language = model['language'] -%}
+  {% set target_relation = this.incorporate(type='table') %}
+  {% set existing_relation = load_relation(this) %}
+  {% set tmp_relation = make_temp_relation(this) %}
+  {% set on_schema_change = incremental_validate_on_schema_change(config.get('on_schema_change'), default='ignore') %}
+  {% set  grant_config = config.get('grants') %}
 
   {{ run_hooks(pre_hooks, inside_transaction=False) }}
 
   -- `BEGIN` happens here:
   {{ run_hooks(pre_hooks, inside_transaction=True) }}
 
   {% set to_drop = [] %}
-
   {% if existing_relation is none %}
-      {% set build_sql = get_create_table_as_sql(False, target_relation, sql) %}
-  {% elif full_refresh_mode %}
-      {% set build_sql = get_create_table_as_sql(False, intermediate_relation, sql) %}
-      {% set need_swap = true %}
+      {% set build_sql = create_table_as(False, target_relation, sql) %}
+  {% elif existing_relation.is_view or full_refresh_mode %}
+      {#-- Checking if backup relation exists#}
+      {% set backup_identifier = existing_relation.identifier ~ "__dbt_backup" %}
+      {% set backup_relation = existing_relation.incorporate(path={"identifier": backup_identifier}) %}
+      {% do adapter.drop_relation(backup_relation) %}
+      {% if existing_relation.is_view %}
+            {% do adapter.drop_relation(existing_relation) %}
+      {% else %}
+            {% do adapter.rename_relation(existing_relation, backup_relation) %}
+      {% endif %}
+      {% set build_sql = create_table_as(False, target_relation, sql) %}
+      {% do to_drop.append(backup_relation) %}
   {% else %}
-    {% do run_query(get_create_table_as_sql(True, temp_relation, sql)) %}
-    {% do adapter.expand_target_column_types(
-             from_relation=temp_relation,
+      {% set tmp_relation = make_temp_relation(target_relation) %}
+      {% do to_drop.append(tmp_relation) %}
+      {% call statement("make_tmp_relation") %}
+        {{create_table_as(True, tmp_relation, sql)}}
+      {% endcall %}
+      {% do adapter.expand_target_column_types(
+             from_relation=tmp_relation,
              to_relation=target_relation) %}
-    {#-- Process schema changes. Returns dict of changes if successful. Use source columns for upserting/merging --#}
-    {% set dest_columns = process_schema_changes(on_schema_change, temp_relation, existing_relation) %}
-    {% if not dest_columns %}
-      {% set dest_columns = adapter.get_columns_in_relation(existing_relation) %}
-    {% endif %}
-
-    {#-- Get the incremental_strategy, the macro to use for the strategy, and build the sql --#}
-    {% set incremental_strategy = config.get('incremental_strategy') or 'default' %}
-    {% set incremental_predicates = config.get('predicates', none) or config.get('incremental_predicates', none) %}
-    {% set strategy_sql_macro_func = adapter.get_incremental_strategy_macro(context, incremental_strategy) %}
-    {% set strategy_arg_dict = ({'target_relation': target_relation, 'temp_relation': temp_relation, 'unique_key': unique_key, 'dest_columns': dest_columns, 'incremental_predicates': incremental_predicates }) %}
-    {% set build_sql = strategy_sql_macro_func(strategy_arg_dict) %}
+      {% set dest_columns = process_schema_changes(on_schema_change, tmp_relation, existing_relation) %}
+      {% if not dest_columns %}
+        {% set dest_columns = adapter.get_columns_in_relation(existing_relation) %}
+      {% endif %}
+
+      {#-- Get the incremental_strategy, the macro to use for the strategy, and build the sql --#}
+      {% set incremental_strategy = config.get('incremental_strategy') or 'default' %}
+      {% set incremental_predicates = config.get('predicates', none) or config.get('incremental_predicates', none) %}
+      {% set strategy_sql_macro_func = adapter.get_incremental_strategy_macro(context, incremental_strategy) %}
+      {% set strategy_arg_dict = ({'target_relation': target_relation, 'temp_relation': tmp_relation, 'unique_key': unique_key, 'dest_columns': dest_columns, 'incremental_predicates': incremental_predicates }) %}
+      {% set build_sql = strategy_sql_macro_func(strategy_arg_dict) %}
 
   {% endif %}
 
   {% call statement("main") %}
       {{ build_sql }}
   {% endcall %}
 
-  {% if need_swap %}
-      {% do adapter.rename_relation(existing_relation, backup_relation) %}
-      {% do adapter.rename_relation(intermediate_relation, target_relation) %}
-      {% do to_drop.append(backup_relation) %}
-  {% endif %}
-
-  {% set should_revoke = should_revoke(existing_relation, full_refresh_mode) %}
-  {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}
-
   {% do persist_docs(target_relation, model) %}
 
-  {% if existing_relation is none or existing_relation.is_view or should_full_refresh() %}
-    {% do create_indexes(target_relation) %}
-  {% endif %}
-
   {{ run_hooks(post_hooks, inside_transaction=True) }}
 
   -- `COMMIT` happens here
@@ -88,6 +69,9 @@
 
   {{ run_hooks(post_hooks, inside_transaction=False) }}
 
+  {% set should_revoke = should_revoke(existing_relation.is_table, full_refresh_mode) %}
+  {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}
+
   {{ return({'relations': [target_relation]}) }}
 
 {%- endmaterialization %}