diff --git a/dbt/adapters/exasol/impl.py b/dbt/adapters/exasol/impl.py index b775071..3b05079 100644 --- a/dbt/adapters/exasol/impl.py +++ b/dbt/adapters/exasol/impl.py @@ -8,6 +8,7 @@ from dbt.adapters.sql import SQLAdapter from dbt.exceptions import CompilationError from dbt.utils import filter_null_values +from dbt.adapters.base.meta import available from dbt.adapters.base.impl import ConstraintSupport from dbt.contracts.graph.nodes import ConstraintType @@ -99,6 +100,40 @@ def valid_incremental_strategies(self): """The set of standard builtin strategies which this adapter supports out-of-the-box. Not used to validate custom strategies defined by end users. """ - return ["append", "delete+insert"] + return ["append", "merge", "delete+insert"] + + @staticmethod + def is_valid_identifier(identifier) -> bool: + # The first character should be alphabetic + if not identifier[0].isalpha(): + return False + # Rest of the characters is either alphanumeric or any one of the literals '#', '$', '_' + idx = 1 + while idx < len(identifier): + identifier_chr = identifier[idx] + if not identifier_chr.isalnum() and identifier_chr not in ('#', '$', '_'): + return False + idx += 1 + return True - \ No newline at end of file + @available + def should_identifier_be_quoted(self, + identifier, + models_column_dict=None) -> bool: + + #Check if the naming is valid + if not self.is_valid_identifier(identifier): + return True + #check if the column is set to be quoted in the model config + elif models_column_dict and identifier in models_column_dict: + return models_column_dict[identifier].get('quote', False) + elif models_column_dict and self.quote(identifier) in models_column_dict: + return models_column_dict[self.quote(identifier)].get('quote', False) + return False + + @available + def check_and_quote_identifier(self, identifier, models_column_dict=None) -> str: + if self.should_identifier_be_quoted(identifier, models_column_dict): + return self.quote(identifier) + else: + return identifier diff --git a/dbt/include/exasol/macros/adapters.sql b/dbt/include/exasol/macros/adapters.sql index 974a2f7..816fa3b 100644 --- a/dbt/include/exasol/macros/adapters.sql +++ b/dbt/include/exasol/macros/adapters.sql @@ -1,4 +1,3 @@ - /* LIST_RELATIONS_MACRO_NAME = 'list_relations_without_caching' GET_COLUMNS_IN_RELATION_MACRO_NAME = 'get_columns_in_relation' @@ -139,7 +138,7 @@ AS {% macro exasol__alter_relation_comment(relation, relation_comment) -%} {# Comments on views are not supported outside DDL, see https://docs.exasol.com/db/latest/sql/comment.htm#UsageNotes #} {%- if not relation.is_view %} - {%- set comment = relation_comment | replace("'", '"') %} + {%- set comment = relation_comment | replace("'", "''") %} COMMENT ON {{ relation.type }} {{ relation }} IS '{{ comment }}'; {%- endif %} {% endmacro %} @@ -155,7 +154,7 @@ AS {% set matched_column = None -%} {% endif -%} {% if matched_column -%} - {% set comment = column_dict[matched_column]['description'] | replace("'", '"') -%} + {% set comment = column_dict[matched_column]['description'] | replace("'", "''") -%} {% else -%} {% set comment = "" -%} {% endif -%} @@ -187,7 +186,7 @@ AS {% macro persist_view_relation_docs() %} {%- if config.persist_relation_docs() %} -COMMENT IS '{{ model.description }}' +COMMENT IS '{{ model.description | replace("'", "''")}}' {%- endif %} {% endmacro %} @@ -197,8 +196,8 @@ COMMENT IS '{{ model.description }}' {% endcall %} {% endmacro %} -{% macro exasol__get_empty_subquery_sql(select_sql, sql_header=None ) %} +{% macro exasol__get_empty_subquery_sql(select_sql, select_sql_header=None ) %} select * from ( {{ select_sql }} ) dbt_sbq_tmp -{% endmacro %} \ No newline at end of file +{% endmacro %} diff --git a/dbt/include/exasol/macros/catalog.sql b/dbt/include/exasol/macros/catalog.sql index 564d68e..d87ddfb 100644 --- a/dbt/include/exasol/macros/catalog.sql +++ b/dbt/include/exasol/macros/catalog.sql @@ -13,7 +13,7 @@ OBJECT_NAME as table_name, ROOT_NAME as table_owner, OBJECT_TYPE AS table_type, - OBJECT_COMMENT + OBJECT_COMMENT as table_comment from sys.EXA_USER_OBJECTS WHERE OBJECT_TYPE IN('TABLE', 'VIEW') @@ -34,7 +34,8 @@ ) select tabs.table_owner as [table_owner], - tabs.table_type AS [table_type], + tabs.table_type AS [table_type], + tabs.table_comment as [table_comment], cols.table_database as [table_database], cols.table_schema as [table_schema], cols.table_name as [table_name], diff --git a/dbt/include/exasol/macros/materializations/incremental.sql b/dbt/include/exasol/macros/materializations/incremental.sql index c428807..2e02438 100644 --- a/dbt/include/exasol/macros/materializations/incremental.sql +++ b/dbt/include/exasol/macros/materializations/incremental.sql @@ -1,30 +1,13 @@ +{% materialization incremental, adapter='exasol', supported_languages=['sql'] %} -{% materialization incremental, adapter='exasol' -%} - - -- relations - {%- set existing_relation = load_relation(this) -%} - - {%- set target_relation = this.incorporate(type='table') -%} - {%- set temp_relation = make_temp_relation(target_relation)-%} - {%- set intermediate_relation = make_intermediate_relation(target_relation)-%} - {%- set backup_relation_type = 'table' if existing_relation is none else existing_relation.type -%} - {%- set backup_relation = make_backup_relation(target_relation, backup_relation_type) -%} - - -- configs - {%- set unique_key = config.get('unique_key') -%} - {%- set full_refresh_mode = (should_full_refresh() or existing_relation.is_view) -%} - {%- set on_schema_change = incremental_validate_on_schema_change(config.get('on_schema_change'), default='ignore') -%} - - -- the temp_ and backup_ relations should not already exist in the database; get_relation - -- will return None in that case. Otherwise, we get a relation that we can drop - -- later, before we try to use this name for the current operation. This has to happen before - -- BEGIN, in a separate transaction - {%- set preexisting_intermediate_relation = load_cached_relation(intermediate_relation)-%} - {%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%} - -- grab current tables grants config for comparision later on - {% set grant_config = config.get('grants') %} - {{ drop_relation_if_exists(preexisting_intermediate_relation) }} - {{ drop_relation_if_exists(preexisting_backup_relation) }} + {% set unique_key = config.get('unique_key') %} + {% set full_refresh_mode = flags.FULL_REFRESH %} + {%- set language = model['language'] -%} + {% set target_relation = this.incorporate(type='table') %} + {% set existing_relation = load_relation(this) %} + {% set tmp_relation = make_temp_relation(this) %} + {% set on_schema_change = incremental_validate_on_schema_change(config.get('on_schema_change'), default='ignore') %} + {% set grant_config = config.get('grants') %} {{ run_hooks(pre_hooks, inside_transaction=False) }} @@ -32,29 +15,40 @@ {{ run_hooks(pre_hooks, inside_transaction=True) }} {% set to_drop = [] %} - {% if existing_relation is none %} - {% set build_sql = get_create_table_as_sql(False, target_relation, sql) %} - {% elif full_refresh_mode %} - {% set build_sql = get_create_table_as_sql(False, intermediate_relation, sql) %} - {% set need_swap = true %} + {% set build_sql = create_table_as(False, target_relation, sql) %} + {% elif existing_relation.is_view or full_refresh_mode %} + {#-- Checking if backup relation exists#} + {% set backup_identifier = existing_relation.identifier ~ "__dbt_backup" %} + {% set backup_relation = existing_relation.incorporate(path={"identifier": backup_identifier}) %} + {% do adapter.drop_relation(backup_relation) %} + {% if existing_relation.is_view %} + {% do adapter.drop_relation(existing_relation) %} + {% else %} + {% do adapter.rename_relation(existing_relation, backup_relation) %} + {% endif %} + {% set build_sql = create_table_as(False, target_relation, sql) %} + {% do to_drop.append(backup_relation) %} {% else %} - {% do run_query(get_create_table_as_sql(True, temp_relation, sql)) %} - {% do adapter.expand_target_column_types( - from_relation=temp_relation, + {% set tmp_relation = make_temp_relation(target_relation) %} + {% do to_drop.append(tmp_relation) %} + {% call statement("make_tmp_relation") %} + {{create_table_as(True, tmp_relation, sql)}} + {% endcall %} + {% do adapter.expand_target_column_types( + from_relation=tmp_relation, to_relation=target_relation) %} - {#-- Process schema changes. Returns dict of changes if successful. Use source columns for upserting/merging --#} - {% set dest_columns = process_schema_changes(on_schema_change, temp_relation, existing_relation) %} - {% if not dest_columns %} - {% set dest_columns = adapter.get_columns_in_relation(existing_relation) %} - {% endif %} - - {#-- Get the incremental_strategy, the macro to use for the strategy, and build the sql --#} - {% set incremental_strategy = config.get('incremental_strategy') or 'default' %} - {% set incremental_predicates = config.get('predicates', none) or config.get('incremental_predicates', none) %} - {% set strategy_sql_macro_func = adapter.get_incremental_strategy_macro(context, incremental_strategy) %} - {% set strategy_arg_dict = ({'target_relation': target_relation, 'temp_relation': temp_relation, 'unique_key': unique_key, 'dest_columns': dest_columns, 'incremental_predicates': incremental_predicates }) %} - {% set build_sql = strategy_sql_macro_func(strategy_arg_dict) %} + {% set dest_columns = process_schema_changes(on_schema_change, tmp_relation, existing_relation) %} + {% if not dest_columns %} + {% set dest_columns = adapter.get_columns_in_relation(existing_relation) %} + {% endif %} + + {#-- Get the incremental_strategy, the macro to use for the strategy, and build the sql --#} + {% set incremental_strategy = config.get('incremental_strategy') or 'default' %} + {% set incremental_predicates = config.get('predicates', none) or config.get('incremental_predicates', none) %} + {% set strategy_sql_macro_func = adapter.get_incremental_strategy_macro(context, incremental_strategy) %} + {% set strategy_arg_dict = ({'target_relation': target_relation, 'temp_relation': tmp_relation, 'unique_key': unique_key, 'dest_columns': dest_columns, 'incremental_predicates': incremental_predicates }) %} + {% set build_sql = strategy_sql_macro_func(strategy_arg_dict) %} {% endif %} @@ -62,21 +56,8 @@ {{ build_sql }} {% endcall %} - {% if need_swap %} - {% do adapter.rename_relation(existing_relation, backup_relation) %} - {% do adapter.rename_relation(intermediate_relation, target_relation) %} - {% do to_drop.append(backup_relation) %} - {% endif %} - - {% set should_revoke = should_revoke(existing_relation, full_refresh_mode) %} - {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %} - {% do persist_docs(target_relation, model) %} - {% if existing_relation is none or existing_relation.is_view or should_full_refresh() %} - {% do create_indexes(target_relation) %} - {% endif %} - {{ run_hooks(post_hooks, inside_transaction=True) }} -- `COMMIT` happens here @@ -88,6 +69,9 @@ {{ run_hooks(post_hooks, inside_transaction=False) }} + {% set should_revoke = should_revoke(existing_relation.is_table, full_refresh_mode) %} + {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %} + {{ return({'relations': [target_relation]}) }} {%- endmaterialization %} \ No newline at end of file diff --git a/dbt/include/exasol/macros/materializations/merge.sql b/dbt/include/exasol/macros/materializations/merge.sql index 5ee324b..b3cd6bc 100644 --- a/dbt/include/exasol/macros/materializations/merge.sql +++ b/dbt/include/exasol/macros/materializations/merge.sql @@ -1,17 +1,22 @@ -{% macro exasol__get_delete_insert_merge_sql(target, source, unique_key, dest_columns,incremental_predicates=none) -%} +{% macro exasol__get_delete_insert_merge_sql(target, source, unique_key, dest_columns,incremental_predicates) -%} {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute="name")) -%} {% if unique_key %} {% if unique_key is sequence and unique_key is not string %} - delete from {{ target }} + delete from {{target }} where exists ( select 1 from {{ source }} - where + where {% for key in unique_key %} {{ source }}.{{ key }} = {{ target }}.{{ key }} - {{ "and " if not loop.last }} + {{ "and " if not loop.last}} {% endfor %} - ) + {% if incremental_predicates %} + {% for predicate in incremental_predicates %} + and {{ predicate }} + {% endfor %} + {% endif %} + ); {% else %} delete from {{ target }} where ( @@ -19,6 +24,11 @@ select ({{ unique_key }}) from {{ source }} ) + {%- if incremental_predicates %} + {% for predicate in incremental_predicates %} + and {{ predicate }} + {% endfor %} + {%- endif -%}; {% endif %} {% endif %} @@ -32,3 +42,102 @@ ) {%- endmacro %} + + +{% macro exasol_check_and_quote_unique_key_for_incremental_merge(unique_key, incremental_predicates=none) %} + {%- set unique_key_list = [] -%} + {%- set unique_key_merge_predicates = [] if incremental_predicates is none else [] + incremental_predicates -%} + {% if unique_key is sequence and unique_key is not mapping and unique_key is not string %} + {% for key in unique_key | unique %} + {% if adapter.should_identifier_be_quoted(key, model.columns) == true %} + {% do unique_key_list.append('"' ~ key ~ '"') %} + {% else %} + {% do unique_key_list.append(key.upper()) %} + {% endif %} + {% endfor %} + {% else %} + {% if adapter.should_identifier_be_quoted(unique_key, model.columns) == true %} + {% do unique_key_list.append('"' ~ unique_key ~ '"') %} + {% else %} + {% do unique_key_list.append(unique_key.upper()) %} + {% endif %} + {% endif %} + {% for key in unique_key_list %} + {% set this_key_match %} + DBT_INTERNAL_SOURCE.{{ key }} = DBT_INTERNAL_DEST.{{ key }} + {% endset %} + {% do unique_key_merge_predicates.append(this_key_match) %} + {% endfor %} + {%- set unique_key_result = {'unique_key_list': unique_key_list, + 'unique_key_merge_predicates': unique_key_merge_predicates} -%} + {{ return(unique_key_result)}} +{% endmacro %} + + +{% macro exasol__get_merge_update_columns(merge_update_columns, merge_exclude_columns, dest_columns) %} + {%- set default_cols = dest_columns | map(attribute='name') | list -%} + + {%- if merge_update_columns and merge_exclude_columns -%} + {{ exceptions.raise_compiler_error( + 'Model cannot specify merge_update_columns and merge_exclude_columns. Please update model to use only one config' + )}} + {%- elif merge_update_columns -%} + {%- set update_columns = merge_update_columns -%} + {%- elif merge_exclude_columns -%} + {%- set update_columns = [] -%} + {%- for column in dest_columns -%} + {% if column.column | lower not in merge_exclude_columns | map("lower") | list %} + {%- do update_columns.append(column.name) -%} + {% endif %} + {%- endfor -%} + {%- else -%} + {%- set update_columns = default_cols -%} + {%- endif -%} + + {%- set quoted_update_columns = [] -%} + {% for col in update_columns %} + {% do quoted_update_columns.append(adapter.check_and_quote_identifier(col, model.columns)) %} + {% endfor %} + {{ return(quoted_update_columns)}} +{% endmacro %} + + +{% macro exasol__get_incremental_merge_sql(args_dict) %} + {%- set dest_columns = args_dict["dest_columns"] -%} + {%- set temp_relation = args_dict["temp_relation"] -%} + {%- set target_relation = args_dict["target_relation"] -%} + {%- set unique_key = args_dict["unique_key"] -%} + {%- set dest_column_names = dest_columns | map(attribute='name') | list -%} + {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute="name")) -%} + {%- set merge_update_columns = config.get('merge_update_columns') -%} + {%- set merge_exclude_columns = config.get('merge_exclude_columns') -%} + {%- set incremental_predicates = args_dict["incremental_predicates"] -%} + {%- set update_columns = get_merge_update_columns(merge_update_columns, merge_exclude_columns, dest_columns) -%} + {%- if unique_key -%} + {%- set unique_key_result = exasol_check_and_quote_unique_key_for_incremental_merge(unique_key, incremental_predicates) -%} + {%- set unique_key_list = unique_key_result['unique_key_list'] -%} + {%- set unique_key_merge_predicates = unique_key_result['unique_key_merge_predicates'] -%} + merge into {{ target_relation }} DBT_INTERNAL_DEST + using {{ temp_relation }} DBT_INTERNAL_SOURCE + on ({{ unique_key_merge_predicates | join(' AND ') }}) + when matched then + update set + {% for col in update_columns if (col.upper() not in unique_key_list and col not in unique_key_list) -%} + DBT_INTERNAL_DEST.{{ col }} = DBT_INTERNAL_SOURCE.{{ col }}{% if not loop.last %}, {% endif %} + {% endfor -%} + when not matched then + insert({{ dest_cols_csv }}) + values( + {% for col in dest_columns -%} + DBT_INTERNAL_SOURCE.{{ adapter.check_and_quote_identifier(col.name, model.columns) }}{% if not loop.last %}, {% endif %} + {% endfor -%} + ) + {%- else -%} + insert into {{ target_relation }} ({{ dest_cols_csv }}) + ( + select {{ dest_cols_csv }} + from {{ temp_relation }} + ) + {%- endif -%} +{% endmacro %} + diff --git a/dbt/include/exasol/macros/materializations/strategies.sql b/dbt/include/exasol/macros/materializations/strategies.sql index cb9bc4e..bb8489f 100644 --- a/dbt/include/exasol/macros/materializations/strategies.sql +++ b/dbt/include/exasol/macros/materializations/strategies.sql @@ -36,7 +36,7 @@ select {{ snapshot_get_time() }} as snapshot_start {%- endset %} - {#-- don't access the column by name, to avoid dealing with casing issues on snowflake #} + {#-- don't access the column by name, to avoid dealing with casing issues on exasol #} {%- set now = run_query(select_current_time)[0][0] -%} {% if now is none or now is undefined -%} {%- do exceptions.raise_compiler_error('Could not get a snapshot start time from the database') -%} diff --git a/dbt/include/exasol/macros/materializations/table.sql b/dbt/include/exasol/macros/materializations/table.sql index 92ac86f..8c51428 100644 --- a/dbt/include/exasol/macros/materializations/table.sql +++ b/dbt/include/exasol/macros/materializations/table.sql @@ -49,7 +49,7 @@ -- cleanup {% if old_relation is not none %} {% if old_relation.type == 'view' %} - {#-- This is the primary difference between Snowflake and Redshift. Renaming this view + {#-- This is the primary difference between Exasol and Redshift. Renaming this view -- would cause an error if the view has become invalid due to upstream schema changes #} {{ log("Dropping relation " ~ old_relation ~ " because it is a view and this model is a table.") }} {{ drop_relation_if_exists(old_relation) }} diff --git a/poetry.lock b/poetry.lock index 10a6ab3..b69d5e7 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,9 +1,10 @@ -# This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.4.2 and should not be changed by hand. [[package]] name = "agate" version = "1.7.0" description = "A data analysis library that is optimized for humans instead of machines." +category = "main" optional = false python-versions = "*" files = [ @@ -27,6 +28,7 @@ test = ["PyICU (>=2.4.2)", "coverage (>=3.7.1)", "cssselect (>=0.9.1)", "lxml (> name = "appdirs" version = "1.4.4" description = "A small Python module for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +category = "main" optional = false python-versions = "*" files = [ @@ -38,6 +40,7 @@ files = [ name = "appnope" version = "0.1.3" description = "Disable App Nap on macOS >= 10.9" +category = "dev" optional = false python-versions = "*" files = [ @@ -49,6 +52,7 @@ files = [ name = "astroid" version = "2.15.6" description = "An abstract syntax tree for Python with inference support." +category = "dev" optional = false python-versions = ">=3.7.2" files = [ @@ -68,6 +72,7 @@ wrapt = [ name = "asttokens" version = "2.2.1" description = "Annotate AST trees with source code positions" +category = "dev" optional = false python-versions = "*" files = [ @@ -85,6 +90,7 @@ test = ["astroid", "pytest"] name = "attrs" version = "23.1.0" description = "Classes Without Boilerplate" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -103,6 +109,7 @@ tests-no-zope = ["cloudpickle", "hypothesis", "mypy (>=1.1.1)", "pympler", "pyte name = "babel" version = "2.12.1" description = "Internationalization utilities" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -117,6 +124,7 @@ pytz = {version = ">=2015.7", markers = "python_version < \"3.9\""} name = "backcall" version = "0.2.0" description = "Specifications for callback functions passed in to an API" +category = "dev" optional = false python-versions = "*" files = [ @@ -128,6 +136,7 @@ files = [ name = "black" version = "22.12.0" description = "The uncompromising code formatter." +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -163,6 +172,7 @@ uvloop = ["uvloop (>=0.15.2)"] name = "certifi" version = "2023.7.22" description = "Python package for providing Mozilla's CA Bundle." +category = "main" optional = false python-versions = ">=3.6" files = [ @@ -174,6 +184,7 @@ files = [ name = "cffi" version = "1.15.1" description = "Foreign Function Interface for Python calling C code." +category = "main" optional = false python-versions = "*" files = [ @@ -250,6 +261,7 @@ pycparser = "*" name = "chardet" version = "5.2.0" description = "Universal encoding detector for Python 3" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -261,6 +273,7 @@ files = [ name = "charset-normalizer" version = "3.2.0" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +category = "main" optional = false python-versions = ">=3.7.0" files = [ @@ -345,6 +358,7 @@ files = [ name = "click" version = "8.1.3" description = "Composable command line interface toolkit" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -359,6 +373,7 @@ colorama = {version = "*", markers = "platform_system == \"Windows\""} name = "colorama" version = "0.4.6" description = "Cross-platform colored terminal text." +category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" files = [ @@ -370,6 +385,7 @@ files = [ name = "comm" version = "0.1.4" description = "Jupyter Python Comm implementation, for usage in ipykernel, xeus-python etc." +category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -389,6 +405,7 @@ typing = ["mypy (>=0.990)"] name = "cryptography" version = "41.0.3" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -434,6 +451,7 @@ test-randomorder = ["pytest-randomly"] name = "dbt-core" version = "1.5.4" description = "With dbt, data analysts and engineers can build analytics the way engineers build applications." +category = "main" optional = false python-versions = ">=3.7.2" files = [ @@ -469,6 +487,7 @@ werkzeug = ">=1,<3" name = "dbt-extractor" version = "0.4.1" description = "A tool to analyze and extract information from Jinja used in dbt projects." +category = "main" optional = false python-versions = ">=3.6.1" files = [ @@ -494,6 +513,7 @@ files = [ name = "dbt-tests-adapter" version = "1.5.4" description = "The dbt adapter tests for adapter plugins" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -509,6 +529,7 @@ pytest = ">=7.0.0" name = "debugpy" version = "1.6.7.post1" description = "An implementation of the Debug Adapter Protocol for Python" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -536,6 +557,7 @@ files = [ name = "decorator" version = "5.1.1" description = "Decorators for Humans" +category = "dev" optional = false python-versions = ">=3.5" files = [ @@ -547,6 +569,7 @@ files = [ name = "diff-cover" version = "7.7.0" description = "Run coverage and linting reports on diffs" +category = "main" optional = false python-versions = ">=3.7.2,<4.0.0" files = [ @@ -567,6 +590,7 @@ toml = ["tomli (>=1.2.1)"] name = "dill" version = "0.3.7" description = "serialize all of Python" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -581,6 +605,7 @@ graph = ["objgraph (>=1.7.2)"] name = "distlib" version = "0.3.7" description = "Distribution utilities" +category = "dev" optional = false python-versions = "*" files = [ @@ -592,6 +617,7 @@ files = [ name = "exceptiongroup" version = "1.1.3" description = "Backport of PEP 654 (exception groups)" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -606,6 +632,7 @@ test = ["pytest (>=6)"] name = "executing" version = "1.2.0" description = "Get the currently executing AST node of a frame, and other information" +category = "dev" optional = false python-versions = "*" files = [ @@ -620,6 +647,7 @@ tests = ["asttokens", "littleutils", "pytest", "rich"] name = "filelock" version = "3.12.2" description = "A platform independent file lock." +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -635,6 +663,7 @@ testing = ["covdefaults (>=2.3)", "coverage (>=7.2.7)", "diff-cover (>=7.5)", "p name = "future" version = "0.18.3" description = "Clean single-source support for Python 3 and 2" +category = "main" optional = false python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" files = [ @@ -645,6 +674,7 @@ files = [ name = "hologram" version = "0.0.16" description = "JSON schema generation from dataclasses" +category = "main" optional = false python-versions = "*" files = [ @@ -660,6 +690,7 @@ python-dateutil = ">=2.8,<2.9" name = "idna" version = "3.4" description = "Internationalized Domain Names in Applications (IDNA)" +category = "main" optional = false python-versions = ">=3.5" files = [ @@ -671,6 +702,7 @@ files = [ name = "importlib-metadata" version = "6.8.0" description = "Read metadata from Python packages" +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -690,6 +722,7 @@ testing = ["flufl.flake8", "importlib-resources (>=1.3)", "packaging", "pyfakefs name = "importlib-resources" version = "6.0.1" description = "Read resources from Python packages" +category = "main" optional = false python-versions = ">=3.8" files = [ @@ -708,6 +741,7 @@ testing = ["pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", name = "iniconfig" version = "2.0.0" description = "brain-dead simple config-ini parsing" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -719,6 +753,7 @@ files = [ name = "ipykernel" version = "6.25.1" description = "IPython Kernel for Jupyter" +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -732,7 +767,7 @@ comm = ">=0.1.1" debugpy = ">=1.6.5" ipython = ">=7.23.1" jupyter-client = ">=6.1.12" -jupyter-core = ">=4.12,<5.0.dev0 || >=5.1.dev0" +jupyter-core = ">=4.12,<5.0.0 || >=5.1.0" matplotlib-inline = ">=0.1" nest-asyncio = "*" packaging = "*" @@ -752,6 +787,7 @@ test = ["flaky", "ipyparallel", "pre-commit", "pytest (>=7.0)", "pytest-asyncio" name = "ipython" version = "8.12.2" description = "IPython: Productive Interactive Computing" +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -791,6 +827,7 @@ test-extra = ["curio", "matplotlib (!=3.2.0)", "nbformat", "numpy (>=1.21)", "pa name = "isodate" version = "0.6.1" description = "An ISO 8601 date/time/duration parser and formatter" +category = "main" optional = false python-versions = "*" files = [ @@ -805,6 +842,7 @@ six = "*" name = "isort" version = "5.12.0" description = "A Python utility / library to sort Python imports." +category = "dev" optional = false python-versions = ">=3.8.0" files = [ @@ -822,6 +860,7 @@ requirements-deprecated-finder = ["pip-api", "pipreqs"] name = "jedi" version = "0.19.0" description = "An autocompletion tool for Python that can be used for text editors." +category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -841,6 +880,7 @@ testing = ["Django (<3.1)", "attrs", "colorama", "docopt", "pytest (<7.0.0)"] name = "jinja2" version = "3.1.2" description = "A very fast and expressive template engine." +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -858,6 +898,7 @@ i18n = ["Babel (>=2.7)"] name = "jsonschema" version = "4.19.0" description = "An implementation of JSON Schema validation for Python" +category = "main" optional = false python-versions = ">=3.8" files = [ @@ -881,6 +922,7 @@ format-nongpl = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339- name = "jsonschema-specifications" version = "2023.7.1" description = "The JSON Schema meta-schemas and vocabularies, exposed as a Registry" +category = "main" optional = false python-versions = ">=3.8" files = [ @@ -896,6 +938,7 @@ referencing = ">=0.28.0" name = "jupyter-client" version = "8.3.0" description = "Jupyter protocol implementation and client libraries" +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -905,7 +948,7 @@ files = [ [package.dependencies] importlib-metadata = {version = ">=4.8.3", markers = "python_version < \"3.10\""} -jupyter-core = ">=4.12,<5.0.dev0 || >=5.1.dev0" +jupyter-core = ">=4.12,<5.0.0 || >=5.1.0" python-dateutil = ">=2.8.2" pyzmq = ">=23.0" tornado = ">=6.2" @@ -919,6 +962,7 @@ test = ["coverage", "ipykernel (>=6.14)", "mypy", "paramiko", "pre-commit", "pyt name = "jupyter-core" version = "5.3.1" description = "Jupyter core package. A base package on which Jupyter projects rely." +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -939,6 +983,7 @@ test = ["ipykernel", "pre-commit", "pytest", "pytest-cov", "pytest-timeout"] name = "lazy-object-proxy" version = "1.9.0" description = "A fast and thorough lazy object proxy." +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -984,6 +1029,7 @@ files = [ name = "leather" version = "0.3.4" description = "Python charting for 80% of humans." +category = "main" optional = false python-versions = "*" files = [ @@ -998,6 +1044,7 @@ six = ">=1.6.1" name = "logbook" version = "1.5.3" description = "A logging replacement for Python" +category = "main" optional = false python-versions = "*" files = [ @@ -1027,6 +1074,7 @@ zmq = ["pyzmq"] name = "markupsafe" version = "2.1.3" description = "Safely add untrusted strings to HTML/XML markup." +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1086,6 +1134,7 @@ files = [ name = "mashumaro" version = "3.6" description = "Fast serialization library on top of dataclasses" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1107,6 +1156,7 @@ yaml = ["pyyaml (>=3.13)"] name = "matplotlib-inline" version = "0.1.6" description = "Inline Matplotlib backend for Jupyter" +category = "dev" optional = false python-versions = ">=3.5" files = [ @@ -1121,6 +1171,7 @@ traitlets = "*" name = "mccabe" version = "0.7.0" description = "McCabe checker, plugin for flake8" +category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -1132,6 +1183,7 @@ files = [ name = "minimal-snowplow-tracker" version = "0.0.2" description = "A minimal snowplow event tracker for Python. Add analytics to your Python and Django apps, webapps and games" +category = "main" optional = false python-versions = "*" files = [ @@ -1146,6 +1198,7 @@ six = ">=1.9.0,<2.0" name = "msgpack" version = "1.0.5" description = "MessagePack serializer" +category = "main" optional = false python-versions = "*" files = [ @@ -1218,6 +1271,7 @@ files = [ name = "mypy-extensions" version = "1.0.0" description = "Type system extensions for programs checked with the mypy type checker." +category = "dev" optional = false python-versions = ">=3.5" files = [ @@ -1229,6 +1283,7 @@ files = [ name = "nest-asyncio" version = "1.5.7" description = "Patch asyncio to allow nested event loops" +category = "dev" optional = false python-versions = ">=3.5" files = [ @@ -1240,6 +1295,7 @@ files = [ name = "networkx" version = "2.8.8" description = "Python package for creating and manipulating graphs and networks" +category = "main" optional = false python-versions = ">=3.8" files = [ @@ -1258,6 +1314,7 @@ test = ["codecov (>=2.1)", "pytest (>=7.2)", "pytest-cov (>=4.0)"] name = "packaging" version = "23.1" description = "Core utilities for Python packages" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1269,6 +1326,7 @@ files = [ name = "parsedatetime" version = "2.4" description = "Parse human-readable date/time text." +category = "main" optional = false python-versions = "*" files = [ @@ -1283,6 +1341,7 @@ future = "*" name = "parso" version = "0.8.3" description = "A Python Parser" +category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -1298,6 +1357,7 @@ testing = ["docopt", "pytest (<6.0.0)"] name = "pathspec" version = "0.11.2" description = "Utility library for gitignore style pattern matching of file paths." +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1309,6 +1369,7 @@ files = [ name = "pexpect" version = "4.8.0" description = "Pexpect allows easy control of interactive console applications." +category = "dev" optional = false python-versions = "*" files = [ @@ -1323,6 +1384,7 @@ ptyprocess = ">=0.5" name = "pickleshare" version = "0.7.5" description = "Tiny 'shelve'-like database with concurrency support" +category = "dev" optional = false python-versions = "*" files = [ @@ -1334,6 +1396,7 @@ files = [ name = "pkgutil-resolve-name" version = "1.3.10" description = "Resolve a name to an object." +category = "main" optional = false python-versions = ">=3.6" files = [ @@ -1345,6 +1408,7 @@ files = [ name = "platformdirs" version = "3.10.0" description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1360,6 +1424,7 @@ test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4)", "pytest-co name = "pluggy" version = "1.2.0" description = "plugin and hook calling mechanisms for python" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1375,6 +1440,7 @@ testing = ["pytest", "pytest-benchmark"] name = "prompt-toolkit" version = "3.0.39" description = "Library for building powerful interactive command lines in Python" +category = "dev" optional = false python-versions = ">=3.7.0" files = [ @@ -1389,6 +1455,7 @@ wcwidth = "*" name = "protobuf" version = "4.24.0" description = "" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1411,6 +1478,7 @@ files = [ name = "psutil" version = "5.9.5" description = "Cross-platform lib for process and system monitoring in Python." +category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ @@ -1437,6 +1505,7 @@ test = ["enum34", "ipaddress", "mock", "pywin32", "wmi"] name = "ptyprocess" version = "0.7.0" description = "Run a subprocess in a pseudo terminal" +category = "dev" optional = false python-versions = "*" files = [ @@ -1448,6 +1517,7 @@ files = [ name = "pure-eval" version = "0.2.2" description = "Safely evaluate AST nodes without side effects" +category = "dev" optional = false python-versions = "*" files = [ @@ -1462,6 +1532,7 @@ tests = ["pytest"] name = "py" version = "1.11.0" description = "library with cross-python path, ini-parsing, io, code, log facilities" +category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" files = [ @@ -1473,6 +1544,7 @@ files = [ name = "pyasn1" version = "0.5.0" description = "Pure-Python implementation of ASN.1 types and DER/BER/CER codecs (X.208)" +category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" files = [ @@ -1484,6 +1556,7 @@ files = [ name = "pycparser" version = "2.21" description = "C parser in Python" +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ @@ -1495,6 +1568,7 @@ files = [ name = "pyexasol" version = "0.25.2" description = "Exasol python driver with extra features" +category = "main" optional = false python-versions = ">=3.6" files = [ @@ -1519,6 +1593,7 @@ ujson = ["ujson"] name = "pygments" version = "2.16.1" description = "Pygments is a syntax highlighting package written in Python." +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1533,6 +1608,7 @@ plugins = ["importlib-metadata"] name = "pylint" version = "2.17.5" description = "python code static checker" +category = "dev" optional = false python-versions = ">=3.7.2" files = [ @@ -1562,6 +1638,7 @@ testutils = ["gitpython (>3)"] name = "pyopenssl" version = "23.2.0" description = "Python wrapper module around the OpenSSL library" +category = "main" optional = false python-versions = ">=3.6" files = [ @@ -1580,6 +1657,7 @@ test = ["flaky", "pretend", "pytest (>=3.0.1)"] name = "pytest" version = "7.4.0" description = "pytest: simple powerful testing with Python" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1602,6 +1680,7 @@ testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "no name = "pytest-dotenv" version = "0.5.2" description = "A py.test plugin that parses environment files before running tests" +category = "dev" optional = false python-versions = "*" files = [ @@ -1617,6 +1696,7 @@ python-dotenv = ">=0.9.1" name = "pytest-parallel" version = "0.1.1" description = "a pytest plugin for parallel and concurrent testing" +category = "dev" optional = false python-versions = "*" files = [ @@ -1632,6 +1712,7 @@ tblib = "*" name = "python-dateutil" version = "2.8.2" description = "Extensions to the standard Python datetime module" +category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" files = [ @@ -1646,6 +1727,7 @@ six = ">=1.5" name = "python-dotenv" version = "1.0.0" description = "Read key-value pairs from a .env file and set them as environment variables" +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -1660,6 +1742,7 @@ cli = ["click (>=5.0)"] name = "python-slugify" version = "8.0.1" description = "A Python slugify application that also handles Unicode" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1677,6 +1760,7 @@ unidecode = ["Unidecode (>=1.1.1)"] name = "pytimeparse" version = "1.1.8" description = "Time expression parser" +category = "main" optional = false python-versions = "*" files = [ @@ -1688,6 +1772,7 @@ files = [ name = "pytz" version = "2023.3" description = "World timezone definitions, modern and historical" +category = "main" optional = false python-versions = "*" files = [ @@ -1699,6 +1784,7 @@ files = [ name = "pywin32" version = "306" description = "Python for Window Extensions" +category = "dev" optional = false python-versions = "*" files = [ @@ -1722,6 +1808,7 @@ files = [ name = "pyyaml" version = "6.0.1" description = "YAML parser and emitter for Python" +category = "main" optional = false python-versions = ">=3.6" files = [ @@ -1771,6 +1858,7 @@ files = [ name = "pyzmq" version = "25.1.1" description = "Python bindings for 0MQ" +category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -1876,6 +1964,7 @@ cffi = {version = "*", markers = "implementation_name == \"pypy\""} name = "referencing" version = "0.30.2" description = "JSON Referencing + Python" +category = "main" optional = false python-versions = ">=3.8" files = [ @@ -1891,6 +1980,7 @@ rpds-py = ">=0.7.0" name = "regex" version = "2023.8.8" description = "Alternative regular expression module, to replace re." +category = "main" optional = false python-versions = ">=3.6" files = [ @@ -1988,6 +2078,7 @@ files = [ name = "requests" version = "2.31.0" description = "Python HTTP for Humans." +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -2009,6 +2100,7 @@ use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] name = "rpds-py" version = "0.9.2" description = "Python bindings to Rust's persistent data structures (rpds)" +category = "main" optional = false python-versions = ">=3.8" files = [ @@ -2115,6 +2207,7 @@ files = [ name = "rsa" version = "4.9" description = "Pure-Python RSA implementation" +category = "main" optional = false python-versions = ">=3.6,<4" files = [ @@ -2129,6 +2222,7 @@ pyasn1 = ">=0.1.3" name = "six" version = "1.16.0" description = "Python 2 and 3 compatibility utilities" +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" files = [ @@ -2140,6 +2234,7 @@ files = [ name = "sqlfluff" version = "2.3.0" description = "The SQL Linter for Humans" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -2168,6 +2263,7 @@ typing-extensions = "*" name = "sqlparse" version = "0.4.4" description = "A non-validating SQL parser." +category = "main" optional = false python-versions = ">=3.5" files = [ @@ -2184,6 +2280,7 @@ test = ["pytest", "pytest-cov"] name = "stack-data" version = "0.6.2" description = "Extract data from python stack frames and tracebacks for informative displays" +category = "dev" optional = false python-versions = "*" files = [ @@ -2203,6 +2300,7 @@ tests = ["cython", "littleutils", "pygments", "pytest", "typeguard"] name = "tblib" version = "2.0.0" description = "Traceback serialization library." +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -2214,6 +2312,7 @@ files = [ name = "text-unidecode" version = "1.3" description = "The most basic Text::Unidecode port" +category = "main" optional = false python-versions = "*" files = [ @@ -2225,6 +2324,7 @@ files = [ name = "toml" version = "0.10.2" description = "Python Library for Tom's Obvious, Minimal Language" +category = "main" optional = false python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" files = [ @@ -2236,6 +2336,7 @@ files = [ name = "tomli" version = "2.0.1" description = "A lil' TOML parser" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -2247,6 +2348,7 @@ files = [ name = "tomlkit" version = "0.12.1" description = "Style preserving TOML library" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -2258,6 +2360,7 @@ files = [ name = "tornado" version = "6.3.3" description = "Tornado is a Python web framework and asynchronous networking library, originally developed at FriendFeed." +category = "dev" optional = false python-versions = ">= 3.8" files = [ @@ -2278,6 +2381,7 @@ files = [ name = "tox" version = "3.28.0" description = "tox is a generic virtualenv management and test command line tool" +category = "dev" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" files = [ @@ -2303,6 +2407,7 @@ testing = ["flaky (>=3.4.0)", "freezegun (>=0.3.11)", "pathlib2 (>=2.3.3)", "psu name = "tqdm" version = "4.66.1" description = "Fast, Extensible Progress Meter" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -2323,6 +2428,7 @@ telegram = ["requests"] name = "traitlets" version = "5.9.0" description = "Traitlets Python configuration system" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -2338,6 +2444,7 @@ test = ["argcomplete (>=2.0)", "pre-commit", "pytest", "pytest-mock"] name = "typing-extensions" version = "4.7.1" description = "Backported and Experimental Type Hints for Python 3.7+" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -2349,6 +2456,7 @@ files = [ name = "urllib3" version = "2.0.4" description = "HTTP library with thread-safe connection pooling, file post, and more." +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -2366,6 +2474,7 @@ zstd = ["zstandard (>=0.18.0)"] name = "virtualenv" version = "20.24.3" description = "Virtual Python Environment builder" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -2386,6 +2495,7 @@ test = ["covdefaults (>=2.3)", "coverage (>=7.2.7)", "coverage-enable-subprocess name = "wcwidth" version = "0.2.6" description = "Measures the displayed width of unicode strings in a terminal" +category = "dev" optional = false python-versions = "*" files = [ @@ -2397,6 +2507,7 @@ files = [ name = "websocket-client" version = "1.6.1" description = "WebSocket client for Python with low level API options" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -2413,6 +2524,7 @@ test = ["websockets"] name = "werkzeug" version = "2.3.7" description = "The comprehensive WSGI web application library." +category = "main" optional = false python-versions = ">=3.8" files = [ @@ -2430,6 +2542,7 @@ watchdog = ["watchdog (>=2.3)"] name = "wrapt" version = "1.15.0" description = "Module for decorators, wrappers and monkey patching." +category = "dev" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" files = [ @@ -2514,6 +2627,7 @@ files = [ name = "zipp" version = "3.16.2" description = "Backport of pathlib-compatible object wrapper for zip files" +category = "main" optional = false python-versions = ">=3.8" files = [ diff --git a/pyproject.toml b/pyproject.toml index fc589f8..6b75f7d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,6 +18,7 @@ python = "^3.8, <3.12" dbt-core = "^1.5 <1.6" pyexasol = "^0.25.0" sqlfluff = "^2.0" +pyyaml = "6.0.1" [tool.poetry.group.dev.dependencies] pylint = "^2.15.8" diff --git a/tests/functional/adapter/aliases/test_aliases.py b/tests/functional/adapter/aliases/test_aliases.py new file mode 100644 index 0000000..9412751 --- /dev/null +++ b/tests/functional/adapter/aliases/test_aliases.py @@ -0,0 +1,29 @@ +import pytest +from dbt.tests.adapter.aliases.test_aliases import BaseAliases + +MACROS__EXASOL_CAST_SQL = """ +{% macro exasol__string_literal(s) %} + TO_CHAR('{{ s }}') +{% endmacro %} +""" + +MACROS__EXPECT_VALUE_SQL = """ +-- cross-db compatible test, similar to accepted_values + +{% test expect_value(model, field, value) %} + +select * +from {{ model }} +where {{ field }} != '{{ value }}' + +{% endtest %} +""" + + +class TestAliasesExasol(BaseAliases): + @pytest.fixture(scope="class") + def macros(self): + return { + "exasol_cast.sql": MACROS__EXASOL_CAST_SQL, + "expect_value.sql": MACROS__EXPECT_VALUE_SQL, + } \ No newline at end of file diff --git a/tests/functional/adapter/test_concurrency.py b/tests/functional/adapter/concurrency/test_concurrency.py similarity index 95% rename from tests/functional/adapter/test_concurrency.py rename to tests/functional/adapter/concurrency/test_concurrency.py index 61f2298..fb115f0 100644 --- a/tests/functional/adapter/test_concurrency.py +++ b/tests/functional/adapter/concurrency/test_concurrency.py @@ -12,7 +12,7 @@ ) class TestConcurrencyExasol(BaseConcurrency): - def test_conncurrency_snowflake(self, project): + def test_conncurrency_exasol(self, project): run_dbt(["seed", "--select", "seed"]) results = run_dbt(["run"], expect_pass=False) assert len(results) == 7 diff --git a/tests/functional/adapter/ephemeral/test_ephemeral.py b/tests/functional/adapter/ephemeral/test_ephemeral.py new file mode 100644 index 0000000..99cfd7f --- /dev/null +++ b/tests/functional/adapter/ephemeral/test_ephemeral.py @@ -0,0 +1,37 @@ +import pytest +from dbt.tests.adapter.ephemeral.test_ephemeral import ( + BaseEphemeralMulti, + BaseEphemeral, + ephemeral_errors__dependent_sql, + ephemeral_errors__base__base_sql, + ephemeral_errors__base__base_copy_sql +) +from dbt.tests.util import run_dbt, check_relations_equal + +class TestEphemeralMultiExasol(BaseEphemeralMulti): + + def test_ephemeral_multi_exasol(self, project): + run_dbt(["seed"]) + results = run_dbt(["run"]) + assert len(results) == 3 + check_relations_equal(project.adapter, ["SEED", "DEPENDENT", "DOUBLE_DEPENDENT", "SUPER_DEPENDENT"]) + +class TestEphemeralNestedExasol(BaseEphemeral): + pass + +class TestEphemeralErrorHandling(BaseEphemeral): + @pytest.fixture(scope="class") + def models(self): + return { + "dependent.sql": ephemeral_errors__dependent_sql, + "base": { + "base.sql": ephemeral_errors__base__base_sql, + "base_copy.sql": ephemeral_errors__base__base_copy_sql, + }, + } + + def test_ephemeral_error_handling(self, project): + results = run_dbt(["run"], expect_pass=False) + assert len(results) == 1 + assert results[0].status == "skipped" + assert "Compilation Error" in results[0].message \ No newline at end of file diff --git a/tests/functional/test_grants.py b/tests/functional/adapter/grants/test_grants.py similarity index 100% rename from tests/functional/test_grants.py rename to tests/functional/adapter/grants/test_grants.py diff --git a/tests/functional/adapter/incremental/test_incremental_merge_exclude_columns.py b/tests/functional/adapter/incremental/test_incremental_merge_exclude_columns.py new file mode 100644 index 0000000..3bf888e --- /dev/null +++ b/tests/functional/adapter/incremental/test_incremental_merge_exclude_columns.py @@ -0,0 +1,7 @@ +from dbt.tests.adapter.incremental.test_incremental_merge_exclude_columns import ( + BaseMergeExcludeColumns, +) + + +class TestMergeExcludeColumns(BaseMergeExcludeColumns): + pass \ No newline at end of file diff --git a/tests/functional/adapter/incremental/test_incremental_on_schema_change.py b/tests/functional/adapter/incremental/test_incremental_on_schema_change.py new file mode 100644 index 0000000..101e39e --- /dev/null +++ b/tests/functional/adapter/incremental/test_incremental_on_schema_change.py @@ -0,0 +1,7 @@ +from dbt.tests.adapter.incremental.test_incremental_on_schema_change import ( + BaseIncrementalOnSchemaChange, +) + + +class TestIncrementalOnSchemaChange(BaseIncrementalOnSchemaChange): + pass \ No newline at end of file diff --git a/tests/functional/adapter/incremental/test_incremental_predicates.py b/tests/functional/adapter/incremental/test_incremental_predicates.py new file mode 100644 index 0000000..afab314 --- /dev/null +++ b/tests/functional/adapter/incremental/test_incremental_predicates.py @@ -0,0 +1,34 @@ +import pytest +from dbt.tests.adapter.incremental.test_incremental_predicates import BaseIncrementalPredicates + + +class TestIncrementalPredicatesDeleteInsertExasol(BaseIncrementalPredicates): + pass + + +class TestPredicatesDeleteInsertExasol(BaseIncrementalPredicates): + @pytest.fixture(scope="class") + def project_config_update(self): + return {"models": {"+predicates": ["id != 2"], "+incremental_strategy": "delete+insert"}} + + +class TestIncrementalPredicatesMergeExasol(BaseIncrementalPredicates): + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "models": { + "+incremental_predicates": ["dbt_internal_dest.id != 2"], + "+incremental_strategy": "merge", + } + } + + +class TestPredicatesMergeExasol(BaseIncrementalPredicates): + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "models": { + "+predicates": ["dbt_internal_dest.id != 2"], + "+incremental_strategy": "merge", + } + } \ No newline at end of file diff --git a/tests/functional/adapter/incremental/test_incremental_run_result.py b/tests/functional/adapter/incremental/test_incremental_run_result.py new file mode 100644 index 0000000..1fbc4d0 --- /dev/null +++ b/tests/functional/adapter/incremental/test_incremental_run_result.py @@ -0,0 +1,29 @@ +from dbt.tests.util import run_dbt +from dbt.tests.adapter.basic.test_incremental import ( + BaseIncremental, + BaseIncrementalNotSchemaChange, +) + + +class TestBaseIncrementalNotSchemaChange(BaseIncrementalNotSchemaChange): + pass + + +class TestIncrementalRunResultExasol(BaseIncremental): + """Bonus test to verify that incremental models return the number of rows affected""" + + def test_incremental(self, project): + # seed command + results = run_dbt(["seed"]) + assert len(results) == 2 + + # run with initial seed + results = run_dbt(["run", "--vars", "seed_name: base"]) + assert len(results) == 1 + + # run with additions + results = run_dbt(["run", "--vars", "seed_name: added"]) + assert len(results) == 1 + # verify that run_result is correct + rows_affected = results[0].adapter_response["rows_affected"] + assert rows_affected == 10, f"Expected 10 rows changed, found {rows_affected}" \ No newline at end of file diff --git a/tests/functional/test_incremental.py b/tests/functional/adapter/incremental/test_incremental_unique_id.py similarity index 100% rename from tests/functional/test_incremental.py rename to tests/functional/adapter/incremental/test_incremental_unique_id.py diff --git a/tests/functional/adapter/persist_docs/test_persist_docs.py b/tests/functional/adapter/persist_docs/test_persist_docs.py new file mode 100644 index 0000000..a5feb0b --- /dev/null +++ b/tests/functional/adapter/persist_docs/test_persist_docs.py @@ -0,0 +1,61 @@ +import json +import os + +from dbt.tests.util import run_dbt + +from dbt.tests.adapter.persist_docs.test_persist_docs import ( + BasePersistDocs, + BasePersistDocsColumnMissing, + BasePersistDocsCommentOnQuotedColumn, +) + + +class TestPersistDocsExasol(BasePersistDocs): + + def _assert_has_table_comments(self, table_node): + table_comment = table_node["metadata"]["comment"] + assert table_comment.startswith("Table model description") + + table_id_comment = table_node["columns"]["ID"]["comment"] + assert table_id_comment.startswith("id Column description") + + table_name_comment = table_node["columns"]["NAME"]["comment"] + assert table_name_comment.startswith("Some stuff here and then a call to") + + self._assert_common_comments(table_comment, table_id_comment, table_name_comment) + + def _assert_has_view_comments( + self, view_node, has_node_comments=True, has_column_comments=True + ): + view_comment = view_node["metadata"]["comment"] + if has_node_comments: + assert view_comment.startswith("View model description") + self._assert_common_comments(view_comment) + else: + assert not view_comment + + view_id_comment = view_node["columns"]["ID"]["comment"] + if has_column_comments: + assert view_id_comment.startswith("id Column description") + self._assert_common_comments(view_id_comment) + else: + assert not view_id_comment + + view_name_comment = view_node["columns"]["NAME"]["comment"] + assert not view_name_comment + + +class TestPersistDocsColumnMissingExasol(BasePersistDocsColumnMissing): + def test_missing_column(self, project): + run_dbt(["docs", "generate"]) + with open("target/catalog.json") as fp: + catalog_data = json.load(fp) + assert "nodes" in catalog_data + + table_node = catalog_data["nodes"]["model.test.missing_column"] + table_id_comment = table_node["columns"]["ID"]["comment"] + assert table_id_comment.startswith("test id column description") + + +class TestPersistDocsCommentOnQuotedColumnExasol(BasePersistDocsCommentOnQuotedColumn): + pass \ No newline at end of file diff --git a/tests/functional/adapter/simple_copy/test_simple_copy.py b/tests/functional/adapter/simple_copy/test_simple_copy.py new file mode 100644 index 0000000..deb2635 --- /dev/null +++ b/tests/functional/adapter/simple_copy/test_simple_copy.py @@ -0,0 +1,46 @@ +import pytest + +from pathlib import Path + +from dbt.tests.util import run_dbt, rm_file, write_file, check_relations_equal + +from dbt.tests.adapter.simple_copy.test_simple_copy import ( + SimpleCopySetup, + SimpleCopyBase, + EmptyModelsArentRunBase, +) + + + +class TestSimpleCopyBaseExasol(SimpleCopyBase): + @pytest.mark.xfail + def test_simple_copy_with_materialized_views(self, project): + pass + +# This return a dictionary of table names to 'view' or 'table' values. +def exasol_get_tables_in_schema(self): + sql = """ + select object_name, + case when object_type = 'TABLE' then 'table' + when object_type = 'VIEW' then 'view' + else object_type + end as materialization + from SYS.EXA_USER_OBJECTS + where {} + order by object_name + """ + sql = sql.format("{} like '{}'".format("ROOT_NAME", self.test_schema)) + result = self.run_sql(sql, fetch="all") + return {model_name: materialization for (model_name, materialization) in result} + +class TestEmptyModelsArentRunExasol(EmptyModelsArentRunBase): + def test_dbt_doesnt_run_empty_models(self, project): + results = run_dbt(["seed"]) + assert len(results) == 1 + results = run_dbt() + assert len(results) == 7 + + tables = exasol_get_tables_in_schema(self=project) + + assert "empty" not in tables.keys() + assert "disabled" not in tables.keys() \ No newline at end of file diff --git a/tests/functional/adapter/simple_seed/test_simple_seed.py b/tests/functional/adapter/simple_seed/test_simple_seed.py new file mode 100644 index 0000000..db5a841 --- /dev/null +++ b/tests/functional/adapter/simple_seed/test_simple_seed.py @@ -0,0 +1,16 @@ +import pytest + +from dbt.tests.adapter.simple_seed.test_seed import SeedConfigBase +from dbt.tests.util import run_dbt + + +class TestSimpleBigSeedBatchedExasol(SeedConfigBase): + @pytest.fixture(scope="class") + def seeds(self): + seed_data = ["seed_id"] + seed_data.extend([str(i) for i in range(20_000)]) + return {"big_batched_seed.csv": "\n".join(seed_data)} + + def test_big_batched_seed(self, project): + seed_results = run_dbt(["seed"]) + assert len(seed_results) == 1 \ No newline at end of file diff --git a/tests/functional/adapter/simple_seed/test_simple_seed_override.py b/tests/functional/adapter/simple_seed/test_simple_seed_override.py new file mode 100644 index 0000000..897a581 --- /dev/null +++ b/tests/functional/adapter/simple_seed/test_simple_seed_override.py @@ -0,0 +1,95 @@ +import pytest, os +from dbt.tests.adapter.simple_seed.test_seed_type_override import BaseSimpleSeedColumnOverride +from dbt.tests.adapter.utils.base_utils import run_dbt + +_SCHEMA_YML = """ +version: 2 +seeds: +- name: seed_enabled + columns: + - name: birthday + tests: + - column_type: + type: VARCHAR(2000000) + - name: seed_id + tests: + - column_type: + type: DOUBLE + +- name: seed_tricky + columns: + - name: seed_id + tests: + - column_type: + type: DECIMAL(18,0) + - name: seed_id_str + tests: + - column_type: + type: VARCHAR(2000000) + - name: a_bool + tests: + - column_type: + type: BOOLEAN + - name: looks_like_a_bool + tests: + - column_type: + type: VARCHAR(2000000) + - name: a_date + tests: + - column_type: + type: TIMESTAMP + - name: looks_like_a_date + tests: + - column_type: + type: VARCHAR(2000000) + - name: relative + tests: + - column_type: + type: VARCHAR(2000000) + - name: weekday + tests: + - column_type: + type: VARCHAR(2000000) +""".lstrip() + + +class TestSimpleSeedColumnOverride(BaseSimpleSeedColumnOverride): + @pytest.fixture(scope="class") + def dbt_profile_target(self): + return { + "type": "exasol", + "threads": 1, + "dsn": os.getenv("DBT_DSN", "localhost:8563"), + "user": os.getenv("DBT_USER", "sys"), + "pass": os.getenv("DBT_PASS", "exasol"), + "dbname": "DB", + "timestamp_format": "YYYY-MM-DD HH:MI:SS.FF6", + } + @pytest.fixture(scope="class") + def schema(self): + return "simple_seed" + + @pytest.fixture(scope="class") + def models(self): + return {"models-exasol.yml": _SCHEMA_YML} + + @staticmethod + def seed_enabled_types(): + return { + "seed_id": "DOUBLE PRECISION", + "birthday": "VARCHAR(2000000)", + } + + @staticmethod + def seed_tricky_types(): + return { + "seed_id_str": "VARCHAR(2000000)", + "looks_like_a_bool": "VARCHAR(2000000)", + "looks_like_a_date": "VARCHAR(2000000)", + } + + def test_exasol_simple_seed_with_column_override_exasol(self, project): + seed_results = run_dbt(["seed"]) + assert len(seed_results) == 2 + test_results = run_dbt(["test"]) + assert len(test_results) == 10 \ No newline at end of file diff --git a/tests/functional/adapter/simple_snapshot/test_simple_snapshot.py b/tests/functional/adapter/simple_snapshot/test_simple_snapshot.py new file mode 100644 index 0000000..ddcf505 --- /dev/null +++ b/tests/functional/adapter/simple_snapshot/test_simple_snapshot.py @@ -0,0 +1,43 @@ +from dbt.tests.adapter.simple_snapshot.test_snapshot import BaseSnapshotCheck, BaseSimpleSnapshot +from dbt.tests.util import run_dbt + + +class TestSnapshot(BaseSimpleSnapshot): + + def test_updates_are_captured_by_snapshot(self, project): + """ + Update the last 5 records. Show that all ids are current, but the last 5 reflect updates. + """ + self.update_fact_records( + {"updated_at": "updated_at + interval '1' day"}, "id between 16 and 20" + ) + run_dbt(["snapshot"]) + self._assert_results( + ids_with_current_snapshot_records=range(1, 21), + ids_with_closed_out_snapshot_records=range(16, 21), + ) + + def test_new_column_captured_by_snapshot(self, project): + """ + Add a column to `fact` and populate the last 10 records with a non-null value. + Show that all ids are current, but the last 10 reflect updates and the first 10 don't + i.e. if the column is added, but not updated, the record doesn't reflect that it's updated + """ + self.add_fact_column("full_name", "varchar(200) default null") + self.update_fact_records( + { + "full_name": "first_name || ' ' || last_name", + "updated_at": "updated_at + interval '1' day", + }, + "id between 11 and 20", + ) + run_dbt(["snapshot"]) + self._assert_results( + ids_with_current_snapshot_records=range(1, 21), + ids_with_closed_out_snapshot_records=range(11, 21), + ) + + + +class TestSnapshotCheck(BaseSnapshotCheck): + pass diff --git a/tests/functional/adapter/store_test_failures_tests/test_store_test_failures.py b/tests/functional/adapter/store_test_failures_tests/test_store_test_failures.py new file mode 100644 index 0000000..fbfb5dc --- /dev/null +++ b/tests/functional/adapter/store_test_failures_tests/test_store_test_failures.py @@ -0,0 +1,7 @@ +from dbt.tests.adapter.store_test_failures_tests.test_store_test_failures import ( + TestStoreTestFailures, +) + + +class TestStoreTestFailuresExasol(TestStoreTestFailures): + pass \ No newline at end of file diff --git a/tests/functional/adapter/test_ephemeral.py b/tests/functional/adapter/test_ephemeral.py deleted file mode 100644 index 4a14aa9..0000000 --- a/tests/functional/adapter/test_ephemeral.py +++ /dev/null @@ -1,11 +0,0 @@ -import pytest -from dbt.tests.adapter.ephemeral.test_ephemeral import BaseEphemeralMulti -from dbt.tests.util import run_dbt, check_relations_equal - -class TestEphemeralMultiExasol(BaseEphemeralMulti): - - def test_ephemeral_multi_snowflake(self, project): - run_dbt(["seed"]) - results = run_dbt(["run"]) - assert len(results) == 3 - check_relations_equal(project.adapter, ["SEED", "DEPENDENT", "DOUBLE_DEPENDENT", "SUPER_DEPENDENT"]) \ No newline at end of file