From aaf7a632680e177f07f05e4ee32353254e974bb7 Mon Sep 17 00:00:00 2001 From: Jay Rosenthal Date: Fri, 31 May 2024 10:48:22 -0700 Subject: [PATCH 01/44] add where functions go --- tableau_utilities/scripts/compare_config.py | 229 ++++++++++++++++++ .../tableau_file/tableau_file.py | 8 + 2 files changed, 237 insertions(+) create mode 100644 tableau_utilities/scripts/compare_config.py diff --git a/tableau_utilities/scripts/compare_config.py b/tableau_utilities/scripts/compare_config.py new file mode 100644 index 0000000..a1f6638 --- /dev/null +++ b/tableau_utilities/scripts/compare_config.py @@ -0,0 +1,229 @@ +from typing import Dict, Any + +# Compare 2 configs and generate a list of adjustments +# do not include the data source piece + + +UPDATE_ACTIONS = [ + 'delete_metadata', + 'modify_metadata', + 'add_metadata', + 'add_column', + 'modify_column', + 'add_folder', + 'delete_folder' +] + +class CompareConfigs(): + """ Compares 2 config files and returns a list of changes to make + + Keyword Args: + snowflake_conn_id (str): The connection ID for Snowflake, used for the datasource connection info + tableau_conn_id (str): The connection ID for Tableau + github_conn_id (str): The connection ID for GitHub + + Returns: A dict of tasks to be updated for the datasource. + """ + def __init__(self, current_config: Dict[str, Any], new_config: Dict[str, Any]) -> None: + self.current_config = current_config + self.new_config = new_config + + def __add_task(self, datasource_id, action, action_attrs, log_compare_attrs=None): + """ Add a task to the dictionary of tasks: + add_column, modify_column, add_folder, delete_folder, or update_connection + + Sample: { + "abc123def456": { + "datasource_name": "Datasource Name", + "project": "Project Name", + "add_column": [attrib, attrib], + "modify_column": [attrib, attrib] + "add_folder": [attrib, attrib] + "delete_folder": [attrib, attrib] + "update_connection": [attrib, attrib] + } + } + Args: + datasource_id (str): The ID of the datasource + action (str): The name of action to do. + action_attrs (dict): Dict of attributes for the action to use, from the config. + log_compare_attrs (dict): (Optional) Dict of the attributes to be logged for comparison. + """ + if action and action not in UPDATE_ACTIONS: + raise Exception(f'Invalid action {action}') + + if action: + self.tasks[datasource_id][action].append(action_attrs) + datasource_name = self.tasks[datasource_id]['datasource_name'] + logging.info( + ' > (Adding task) %s: %s %s\nAttributes:\n\t%s\n\t%s', + action, datasource_id, datasource_name, action_attrs, log_compare_attrs + ) + + @staticmethod + def __get_column_diffs(tds_col, cfg_column): + """ Compare the column from the tds to attributes we expect. + + Args: + tds_col (Column): The Tableau Column object from the datasource. + cfg_column (cfg.CFGColumn): The column from the Config. + + Returns: A dict of differences + """ + different_value_attrs = dict() + # If there is no column, either in the Datasource.columns or the config, then return False + if not tds_col or not cfg_column: + return different_value_attrs + # Get a list of attributes that have different values in the Datasource Column vs the config + cfg_attrs = cfg_column.dict() + cfg_attrs.pop('folder_name', None) + cfg_attrs.pop('remote_name', None) + for attr, value in cfg_attrs.items(): + tds_value = getattr(tds_col, attr) + if tds_value != value: + different_value_attrs[attr] = tds_value + # Return the different attributes + if different_value_attrs: + logging.info(' > (Column diffs) %s: %s', cfg_column.caption, different_value_attrs) + return different_value_attrs + + def __compare_column_metadata(self, datasource_id: str, tds: Datasource, column: cfg.CFGColumn): + """ Compares the metadata of the Datasource to the config, + and adds tasks for metadata that needs to be added, modified, or deleted. + + Returns: True if metadata needs to be updated + """ + metadata_update = False + if not column.remote_name or column.calculation: + return metadata_update + + # Add task to delete metadata if the opposite casing version of it exists + is_upper = column.remote_name.upper() == column.remote_name + lower_metadata: MetadataRecord = tds.connection.metadata_records.get(column.remote_name.lower()) + if lower_metadata and is_upper: + self.__add_task( + datasource_id=datasource_id, + action='delete_metadata', + action_attrs={'remote_name': lower_metadata.remote_name}, + log_compare_attrs={'remote_name': column.remote_name} + ) + is_lower = column.remote_name.lower() == column.remote_name + upper_metadata: MetadataRecord = tds.connection.metadata_records.get(column.remote_name.upper()) + if upper_metadata and is_lower: + self.__add_task( + datasource_id=datasource_id, + action='delete_metadata', + action_attrs={'remote_name': upper_metadata.remote_name}, + log_compare_attrs={'remote_name': column.remote_name} + ) + + # Add task to modify the metadata if the local_name does not match + metadata: MetadataRecord = tds.connection.metadata_records.get(column.remote_name) + if metadata and metadata.local_name != column.name: + metadata_update = True + self.__add_task( + datasource_id=datasource_id, + action='modify_metadata', + action_attrs={'remote_name': metadata.remote_name, 'local_name': column.name}, + log_compare_attrs={'local_name': metadata.local_name} + ) + + # Add task to add the metadata if it doesn't exist + if not metadata: + metadata_update = True + logging.warning('Column metadata does not exist - may be missing in the SQL: %s', + column.remote_name) + metadata_attrs = { + 'conn': { + 'parent_name': f'[{tds.connection.relation.name}]', + 'ordinal': len(tds.connection.metadata_records) + + len(self.tasks[datasource_id]['add_metadata']), + }, + } + metadata_attrs['conn'].update(column.metadata) + # Set extract attributes if the datasource has an extract + if tds.extract: + metadata_attrs['extract'] = { + 'parent_name': f'[{tds.extract.connection.relation.name}]', + 'ordinal': len(tds.extract.connection.metadata_records) + + len(self.tasks[datasource_id]['add_metadata']), + 'family': tds.connection.relation.name + } + metadata_attrs['extract'].update(column.metadata) + self.__add_task(datasource_id, 'add_metadata', metadata_attrs) + + return metadata_update + + @staticmethod + def __compare_column_mapping(tds: Datasource, column: cfg.CFGColumn): + """ Compares the expected column mapping to the mapping in the Datasource """ + + # Mapping is not required when the column is a calculation or remote_name is not provided + if column.calculation or not column.remote_name: + return False + + # Mapping is not required when there is no cols section and the local_name is the same as the remote_name + if not tds.connection.cols and column.name[1:-1] == column.remote_name: + return False + + parent_name = f'[{tds.connection.relation.name}]' + # Return True If the column is not already mapped the cols section + if {'key': column.name, 'value': f'{parent_name}.[{column.remote_name}]'} not in tds.connection.cols: + return True + + # Return True If the column is mapped in opposite case of the expected key / column name + if column.name.upper() != column.name: + opposite_case = column.name.upper() + else: + opposite_case = column.name.lower() + if tds.connection.cols.get(opposite_case): + return True + + return False + + def __compare_connection(self, dsid, ds_name, tds_connection, expected_attrs): + """ Compare the connection from the Datasource to attributes we expect. + If there is a difference, add a task to update the connection. + + Args: + dsid (str): The Datasource ID. + ds_name (str): The Datasource name. + tds_connection (Datasource.connection): The Datasource.connection object. + expected_attrs (dict): The dict of expected connection attributes. + """ + named_conn = tds_connection.named_connections[expected_attrs['class_name']] + tds_conn = tds_connection[expected_attrs['class_name']] + if not tds_conn: + logging.warning('Datasource does not have a %s connection: %s', + expected_attrs['class_name'], ds_name) + # Check for a difference between the Datasource connection and the expected connection information + connection_diff = False + if expected_attrs['server'] != named_conn.caption: + connection_diff = True + for attr, value in expected_attrs.items(): + tds_attr_value = getattr(tds_conn, attr) + if tds_attr_value and tds_attr_value.lower() != value.lower(): + connection_diff = True + # Add a task if there is a difference + if connection_diff: + self.__add_task(dsid, 'update_connection', expected_attrs, tds_conn.dict()) + else: + logging.info(' > (No changes needed) Connection: %s', ds_name) + + def __compare_folders(self, datasource_id, tds_folders, cfg_folders): + """ Compares folders found in the datasource and in the config. + - If there are folders in the source that are not in the config, + a task will be added to delete the folder. + - If there are folders in the config that are not in the datasource, + a task will be added to add the folder. + + Args: + tds_folders (Datasource.folders_common): The dict of folders from the Datasource + cfg_folders (cfg.CFGList[cfg.CFGFolder]): The dict of folders from the Config + """ + for tds_folder in tds_folders: + if not cfg_folders.get(tds_folder): + self.__add_task(datasource_id, 'delete_folder', {'name': tds_folder.name}) + for cfg_folder in cfg_folders: + if not tds_folders.get(cfg_folder): + self.__add_task(datasource_id, 'add_folder', {'name': cfg_folder.name}) diff --git a/tableau_utilities/tableau_file/tableau_file.py b/tableau_utilities/tableau_file/tableau_file.py index 8d55d56..f2efb47 100644 --- a/tableau_utilities/tableau_file/tableau_file.py +++ b/tableau_utilities/tableau_file/tableau_file.py @@ -301,6 +301,14 @@ def enforce_column(self, column, folder_name=None, remote_name=None): if not found: self.extract.connection.cols.append(extract_col) + + def update_metadata(self): + pass + + def remove_empty_folders(self): + pass + + def save(self): """ Save all changes made to each section of the Datasource """ parent = self._root.find('.') From 6478902ad256b55e78418c592b050064e3ab460f Mon Sep 17 00:00:00 2001 From: Jay Rosenthal Date: Sun, 2 Jun 2024 04:17:53 -0700 Subject: [PATCH 02/44] bump version to include in 2.2.12 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index eac0708..7380dd4 100644 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ long_description=readme, long_description_content_type='text/markdown', name="tableau_utilities", - version="2.2.11", + version="2.2.12", requires_python=">=3.8", packages=[ 'tableau_utilities', From 1645ec23ac64fae1e789d49ec0c58ded53679d49 Mon Sep 17 00:00:00 2001 From: Jay Rosenthal Date: Sun, 2 Jun 2024 05:31:31 -0700 Subject: [PATCH 03/44] removing empty folder works --- .gitignore | 1 + tableau_utilities/scripts/cli.py | 1 + tableau_utilities/scripts/datasource.py | 8 ++++- .../tableau_file/tableau_file.py | 35 ++++++++++++++++++- 4 files changed, 43 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index bcbe97d..e6ca506 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ sheets.googleapis.com-python.json .idea .DS_Store tmp_tdsx_and_config/ +development_test_files/ # Byte-compiled / optimized / DLL files __pycache__/ diff --git a/tableau_utilities/scripts/cli.py b/tableau_utilities/scripts/cli.py index e02651d..8a2d221 100644 --- a/tableau_utilities/scripts/cli.py +++ b/tableau_utilities/scripts/cli.py @@ -163,6 +163,7 @@ help='Deletes data from the extract based on the condition string provided. ' """E.g. "CREATED_AT" < '1/1/2024'""") parser_datasource.add_argument('-ci', '--column_init', action='store_true', help="Adds Columns from all Metadata Records, if they don't already exist.") +parser_datasource.add_argument('-cf', '--clean_folders', action='store_true', help="Removes any empty folders without columns") parser_datasource.set_defaults(func=datasource) # GENERATE CONFIG diff --git a/tableau_utilities/scripts/datasource.py b/tableau_utilities/scripts/datasource.py index adc4d8b..7be9826 100644 --- a/tableau_utilities/scripts/datasource.py +++ b/tableau_utilities/scripts/datasource.py @@ -67,6 +67,7 @@ def datasource(args, server=None): remote_name = args.remote_name list_objects = args.list.title() if args.list else None column_init = args.column_init + clean_folders = args.clean_folders # Datasource Connection Args conn_type = args.conn_type @@ -213,6 +214,11 @@ def datasource(args, server=None): if delete == 'folder': ds.folders_common.folder.delete(folder_name) + # Clean folders + if clean_folders: + cleaned = ds.remove_empty_folders() + print(f'Removed this list of folders: {color.fg_cyan}{cleaned}{color.reset}') + # Enforce Connection if enforce_connection: if debugging_logs: @@ -231,7 +237,7 @@ def datasource(args, server=None): ds.connection.update(connection) # Save the datasource if an edit may have happened - if column_name or folder_name or delete or enforce_connection or empty_extract or column_init: + if column_name or folder_name or delete or enforce_connection or empty_extract or column_init or clean_folders: start = time() print(f'{color.fg_cyan}...Saving datasource changes...{color.reset}') ds.save() diff --git a/tableau_utilities/tableau_file/tableau_file.py b/tableau_utilities/tableau_file/tableau_file.py index f2efb47..bff40bb 100644 --- a/tableau_utilities/tableau_file/tableau_file.py +++ b/tableau_utilities/tableau_file/tableau_file.py @@ -306,7 +306,40 @@ def update_metadata(self): pass def remove_empty_folders(self): - pass + """ Removes any folder without a column in it + + Example: + The "Folder - 2 columns" will be unchanged and the xml line for "Folder - Empty" will be removed + + <_.fcp.SchemaViewerObjectModel.true...folders-common> + + + + + + + + + Returns: + The list of folders that were removed + + """ + + # Identify empty folders + empty_folder_list = [] + + for folder in self.folders_common.folder: + number_columns_in_folder = len(folder.folder_item) + + if number_columns_in_folder == 0: + print(folder.name) + empty_folder_list.append(folder.name) + + # Remove Empty Folders + for empty_folder in empty_folder_list: + self.folders_common.folder.delete(empty_folder) + + return empty_folder_list def save(self): From d1f664d53467423b9558064ce47e58ae90cb5021 Mon Sep 17 00:00:00 2001 From: Jay Rosenthal Date: Sun, 2 Jun 2024 06:01:31 -0700 Subject: [PATCH 04/44] mock test works --- .../tableau_file/tableau_file.py | 1 - .../test_tableau_file_objects.py | 54 +++++++++++++++++++ 2 files changed, 54 insertions(+), 1 deletion(-) create mode 100644 tableau_utilities/test_tableau_file_objects.py diff --git a/tableau_utilities/tableau_file/tableau_file.py b/tableau_utilities/tableau_file/tableau_file.py index bff40bb..c239c02 100644 --- a/tableau_utilities/tableau_file/tableau_file.py +++ b/tableau_utilities/tableau_file/tableau_file.py @@ -332,7 +332,6 @@ def remove_empty_folders(self): number_columns_in_folder = len(folder.folder_item) if number_columns_in_folder == 0: - print(folder.name) empty_folder_list.append(folder.name) # Remove Empty Folders diff --git a/tableau_utilities/test_tableau_file_objects.py b/tableau_utilities/test_tableau_file_objects.py new file mode 100644 index 0000000..930f9f2 --- /dev/null +++ b/tableau_utilities/test_tableau_file_objects.py @@ -0,0 +1,54 @@ +# test_remove_empty_folders.py +import pytest +from unittest.mock import patch +from tableau_utilities.tableau_file.tableau_file_objects import FoldersCommon, Folder, FolderItem +from tableau_utilities.tableau_file.tableau_file import Datasource + +@pytest.fixture +def mock_datasource(): + with patch('tableau_utilities.tableau_file.tableau_file.Datasource.__init__', lambda x, file_path: None): + datasource = Datasource(file_path='dummy_path') + + # Create the mock data + mock_folders = [ + Folder( + name='Folder - 2 columns', + tag='folder', + role=None, + folder_item=[ + FolderItem(name='[COLUMN_1]', type='field', tag='folder-item'), + FolderItem(name='[COLUMN_2]', type='field', tag='folder-item') + ] + ), + Folder( + name='Folder - Empty', + tag='folder', + role=None, + folder_item=[] + ), + Folder( + name='People', + tag='folder', + role=None, + folder_item=[ + FolderItem(name='[COLUMN_2+3]', type='field', tag='folder-item') + ] + ) + ] + + # Assign the mock folders to the folders_common attribute + folders_common = FoldersCommon(folder=mock_folders) + datasource.folders_common = folders_common + + return datasource + +def test_remove_empty_folders(mock_datasource): + # Run the function and verify the result + removed_folders = mock_datasource.remove_empty_folders() + assert removed_folders == ['Folder - Empty'] + assert len(mock_datasource.folders_common.folder) == 2 + folder_names = [folder.name for folder in mock_datasource.folders_common.folder] + assert 'Folder - Empty' not in folder_names + +if __name__ == '__main__': + pytest.main() From 8de989c100b74b6a5076f101af13d28f8dbd5ce0 Mon Sep 17 00:00:00 2001 From: Jay Rosenthal Date: Sun, 2 Jun 2024 06:05:19 -0700 Subject: [PATCH 05/44] 3 tests passed --- tableau_utilities/test_tableau_file_objects.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tableau_utilities/test_tableau_file_objects.py b/tableau_utilities/test_tableau_file_objects.py index 930f9f2..0e653c3 100644 --- a/tableau_utilities/test_tableau_file_objects.py +++ b/tableau_utilities/test_tableau_file_objects.py @@ -42,11 +42,16 @@ def mock_datasource(): return datasource -def test_remove_empty_folders(mock_datasource): - # Run the function and verify the result +def test_remove_empty_folders_removed_folders(mock_datasource): removed_folders = mock_datasource.remove_empty_folders() assert removed_folders == ['Folder - Empty'] + +def test_remove_empty_folders_folder_count(mock_datasource): + mock_datasource.remove_empty_folders() assert len(mock_datasource.folders_common.folder) == 2 + +def test_remove_empty_folders_folder_names(mock_datasource): + mock_datasource.remove_empty_folders() folder_names = [folder.name for folder in mock_datasource.folders_common.folder] assert 'Folder - Empty' not in folder_names From a747c19197317c60acb87736b1f7dd666d0a8616 Mon Sep 17 00:00:00 2001 From: Jay Rosenthal Date: Sun, 2 Jun 2024 06:07:15 -0700 Subject: [PATCH 06/44] rename --- ..._tableau_file_objects.py => test_datasource_remove_folders.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tableau_utilities/{test_tableau_file_objects.py => test_datasource_remove_folders.py} (100%) diff --git a/tableau_utilities/test_tableau_file_objects.py b/tableau_utilities/test_datasource_remove_folders.py similarity index 100% rename from tableau_utilities/test_tableau_file_objects.py rename to tableau_utilities/test_datasource_remove_folders.py From de1ade3de6494a459c20589c39442c5c528289e4 Mon Sep 17 00:00:00 2001 From: Jay Rosenthal Date: Sun, 2 Jun 2024 06:18:18 -0700 Subject: [PATCH 07/44] rename --- ...remove_folders.py => test_datasource_remove_empty_folders.py} | 1 - 1 file changed, 1 deletion(-) rename tableau_utilities/{test_datasource_remove_folders.py => test_datasource_remove_empty_folders.py} (98%) diff --git a/tableau_utilities/test_datasource_remove_folders.py b/tableau_utilities/test_datasource_remove_empty_folders.py similarity index 98% rename from tableau_utilities/test_datasource_remove_folders.py rename to tableau_utilities/test_datasource_remove_empty_folders.py index 0e653c3..bce4a20 100644 --- a/tableau_utilities/test_datasource_remove_folders.py +++ b/tableau_utilities/test_datasource_remove_empty_folders.py @@ -1,4 +1,3 @@ -# test_remove_empty_folders.py import pytest from unittest.mock import patch from tableau_utilities.tableau_file.tableau_file_objects import FoldersCommon, Folder, FolderItem From 803aeeda2709cf865afee6a61450d3a98be030e6 Mon Sep 17 00:00:00 2001 From: Jay Rosenthal Date: Sun, 2 Jun 2024 06:25:33 -0700 Subject: [PATCH 08/44] add new testing to the ci --- .github/workflows/python-package.yml | 4 ++++ .../test_datasource_remove_empty_folders.py | 0 2 files changed, 4 insertions(+) rename {tableau_utilities => tests}/test_datasource_remove_empty_folders.py (100%) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 4df1778..4755bea 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -32,6 +32,10 @@ jobs: flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + - name: Run Pytest mock tests + run: + pytest tests/test_datasource_remove_empty_folders.py -p no:warnings - name: Run pytest on tableau-utilities run: | cd tableau_utilities && pytest -v + diff --git a/tableau_utilities/test_datasource_remove_empty_folders.py b/tests/test_datasource_remove_empty_folders.py similarity index 100% rename from tableau_utilities/test_datasource_remove_empty_folders.py rename to tests/test_datasource_remove_empty_folders.py From acc1d2725d69d9c79b427043124b1a0fd4c0141e Mon Sep 17 00:00:00 2001 From: Jay Rosenthal Date: Sun, 2 Jun 2024 14:34:29 -0700 Subject: [PATCH 09/44] move generate config dics to their own function to make it resuavble --- tableau_utilities/scripts/apply_config.py | 53 ++++ tableau_utilities/scripts/compare_config.py | 229 ------------------ tableau_utilities/scripts/gen_config.py | 100 ++++++-- .../tableau_file/tableau_file.py | 4 - 4 files changed, 127 insertions(+), 259 deletions(-) create mode 100644 tableau_utilities/scripts/apply_config.py delete mode 100644 tableau_utilities/scripts/compare_config.py diff --git a/tableau_utilities/scripts/apply_config.py b/tableau_utilities/scripts/apply_config.py new file mode 100644 index 0000000..d5685f3 --- /dev/null +++ b/tableau_utilities/scripts/apply_config.py @@ -0,0 +1,53 @@ + + + +def compare_config_and_datasource(config, datasource_file, datasource_name: + """ Compares the config to a datasource. Generates a list of changes to make the datasource match the config + + Returns: + dict: a dictionary with the columns that need updating + + """ + pass + + +def execute_changes(column_config, calculated_field_config, datasource): + """ Applies changes to make + + Args: + config: + datasource: + + Returns: + + """ + pass + +def apply_config(column_config, calculated_field_config, datasource): + """ Applies changes to make + + Args: + config: + datasource: + + Returns: + + """ + + # Generate the column_config from the datasource + # Generate the calculation config from the datasource + + # Get the changes to make for the column config + # Get the changes to make for the calculation config + + # Apply the changes for the column config + # Apply the changes for the calc config + + # Clean up the empty folders + + # Save the file + + + + + pass diff --git a/tableau_utilities/scripts/compare_config.py b/tableau_utilities/scripts/compare_config.py deleted file mode 100644 index a1f6638..0000000 --- a/tableau_utilities/scripts/compare_config.py +++ /dev/null @@ -1,229 +0,0 @@ -from typing import Dict, Any - -# Compare 2 configs and generate a list of adjustments -# do not include the data source piece - - -UPDATE_ACTIONS = [ - 'delete_metadata', - 'modify_metadata', - 'add_metadata', - 'add_column', - 'modify_column', - 'add_folder', - 'delete_folder' -] - -class CompareConfigs(): - """ Compares 2 config files and returns a list of changes to make - - Keyword Args: - snowflake_conn_id (str): The connection ID for Snowflake, used for the datasource connection info - tableau_conn_id (str): The connection ID for Tableau - github_conn_id (str): The connection ID for GitHub - - Returns: A dict of tasks to be updated for the datasource. - """ - def __init__(self, current_config: Dict[str, Any], new_config: Dict[str, Any]) -> None: - self.current_config = current_config - self.new_config = new_config - - def __add_task(self, datasource_id, action, action_attrs, log_compare_attrs=None): - """ Add a task to the dictionary of tasks: - add_column, modify_column, add_folder, delete_folder, or update_connection - - Sample: { - "abc123def456": { - "datasource_name": "Datasource Name", - "project": "Project Name", - "add_column": [attrib, attrib], - "modify_column": [attrib, attrib] - "add_folder": [attrib, attrib] - "delete_folder": [attrib, attrib] - "update_connection": [attrib, attrib] - } - } - Args: - datasource_id (str): The ID of the datasource - action (str): The name of action to do. - action_attrs (dict): Dict of attributes for the action to use, from the config. - log_compare_attrs (dict): (Optional) Dict of the attributes to be logged for comparison. - """ - if action and action not in UPDATE_ACTIONS: - raise Exception(f'Invalid action {action}') - - if action: - self.tasks[datasource_id][action].append(action_attrs) - datasource_name = self.tasks[datasource_id]['datasource_name'] - logging.info( - ' > (Adding task) %s: %s %s\nAttributes:\n\t%s\n\t%s', - action, datasource_id, datasource_name, action_attrs, log_compare_attrs - ) - - @staticmethod - def __get_column_diffs(tds_col, cfg_column): - """ Compare the column from the tds to attributes we expect. - - Args: - tds_col (Column): The Tableau Column object from the datasource. - cfg_column (cfg.CFGColumn): The column from the Config. - - Returns: A dict of differences - """ - different_value_attrs = dict() - # If there is no column, either in the Datasource.columns or the config, then return False - if not tds_col or not cfg_column: - return different_value_attrs - # Get a list of attributes that have different values in the Datasource Column vs the config - cfg_attrs = cfg_column.dict() - cfg_attrs.pop('folder_name', None) - cfg_attrs.pop('remote_name', None) - for attr, value in cfg_attrs.items(): - tds_value = getattr(tds_col, attr) - if tds_value != value: - different_value_attrs[attr] = tds_value - # Return the different attributes - if different_value_attrs: - logging.info(' > (Column diffs) %s: %s', cfg_column.caption, different_value_attrs) - return different_value_attrs - - def __compare_column_metadata(self, datasource_id: str, tds: Datasource, column: cfg.CFGColumn): - """ Compares the metadata of the Datasource to the config, - and adds tasks for metadata that needs to be added, modified, or deleted. - - Returns: True if metadata needs to be updated - """ - metadata_update = False - if not column.remote_name or column.calculation: - return metadata_update - - # Add task to delete metadata if the opposite casing version of it exists - is_upper = column.remote_name.upper() == column.remote_name - lower_metadata: MetadataRecord = tds.connection.metadata_records.get(column.remote_name.lower()) - if lower_metadata and is_upper: - self.__add_task( - datasource_id=datasource_id, - action='delete_metadata', - action_attrs={'remote_name': lower_metadata.remote_name}, - log_compare_attrs={'remote_name': column.remote_name} - ) - is_lower = column.remote_name.lower() == column.remote_name - upper_metadata: MetadataRecord = tds.connection.metadata_records.get(column.remote_name.upper()) - if upper_metadata and is_lower: - self.__add_task( - datasource_id=datasource_id, - action='delete_metadata', - action_attrs={'remote_name': upper_metadata.remote_name}, - log_compare_attrs={'remote_name': column.remote_name} - ) - - # Add task to modify the metadata if the local_name does not match - metadata: MetadataRecord = tds.connection.metadata_records.get(column.remote_name) - if metadata and metadata.local_name != column.name: - metadata_update = True - self.__add_task( - datasource_id=datasource_id, - action='modify_metadata', - action_attrs={'remote_name': metadata.remote_name, 'local_name': column.name}, - log_compare_attrs={'local_name': metadata.local_name} - ) - - # Add task to add the metadata if it doesn't exist - if not metadata: - metadata_update = True - logging.warning('Column metadata does not exist - may be missing in the SQL: %s', - column.remote_name) - metadata_attrs = { - 'conn': { - 'parent_name': f'[{tds.connection.relation.name}]', - 'ordinal': len(tds.connection.metadata_records) - + len(self.tasks[datasource_id]['add_metadata']), - }, - } - metadata_attrs['conn'].update(column.metadata) - # Set extract attributes if the datasource has an extract - if tds.extract: - metadata_attrs['extract'] = { - 'parent_name': f'[{tds.extract.connection.relation.name}]', - 'ordinal': len(tds.extract.connection.metadata_records) - + len(self.tasks[datasource_id]['add_metadata']), - 'family': tds.connection.relation.name - } - metadata_attrs['extract'].update(column.metadata) - self.__add_task(datasource_id, 'add_metadata', metadata_attrs) - - return metadata_update - - @staticmethod - def __compare_column_mapping(tds: Datasource, column: cfg.CFGColumn): - """ Compares the expected column mapping to the mapping in the Datasource """ - - # Mapping is not required when the column is a calculation or remote_name is not provided - if column.calculation or not column.remote_name: - return False - - # Mapping is not required when there is no cols section and the local_name is the same as the remote_name - if not tds.connection.cols and column.name[1:-1] == column.remote_name: - return False - - parent_name = f'[{tds.connection.relation.name}]' - # Return True If the column is not already mapped the cols section - if {'key': column.name, 'value': f'{parent_name}.[{column.remote_name}]'} not in tds.connection.cols: - return True - - # Return True If the column is mapped in opposite case of the expected key / column name - if column.name.upper() != column.name: - opposite_case = column.name.upper() - else: - opposite_case = column.name.lower() - if tds.connection.cols.get(opposite_case): - return True - - return False - - def __compare_connection(self, dsid, ds_name, tds_connection, expected_attrs): - """ Compare the connection from the Datasource to attributes we expect. - If there is a difference, add a task to update the connection. - - Args: - dsid (str): The Datasource ID. - ds_name (str): The Datasource name. - tds_connection (Datasource.connection): The Datasource.connection object. - expected_attrs (dict): The dict of expected connection attributes. - """ - named_conn = tds_connection.named_connections[expected_attrs['class_name']] - tds_conn = tds_connection[expected_attrs['class_name']] - if not tds_conn: - logging.warning('Datasource does not have a %s connection: %s', - expected_attrs['class_name'], ds_name) - # Check for a difference between the Datasource connection and the expected connection information - connection_diff = False - if expected_attrs['server'] != named_conn.caption: - connection_diff = True - for attr, value in expected_attrs.items(): - tds_attr_value = getattr(tds_conn, attr) - if tds_attr_value and tds_attr_value.lower() != value.lower(): - connection_diff = True - # Add a task if there is a difference - if connection_diff: - self.__add_task(dsid, 'update_connection', expected_attrs, tds_conn.dict()) - else: - logging.info(' > (No changes needed) Connection: %s', ds_name) - - def __compare_folders(self, datasource_id, tds_folders, cfg_folders): - """ Compares folders found in the datasource and in the config. - - If there are folders in the source that are not in the config, - a task will be added to delete the folder. - - If there are folders in the config that are not in the datasource, - a task will be added to add the folder. - - Args: - tds_folders (Datasource.folders_common): The dict of folders from the Datasource - cfg_folders (cfg.CFGList[cfg.CFGFolder]): The dict of folders from the Config - """ - for tds_folder in tds_folders: - if not cfg_folders.get(tds_folder): - self.__add_task(datasource_id, 'delete_folder', {'name': tds_folder.name}) - for cfg_folder in cfg_folders: - if not tds_folders.get(cfg_folder): - self.__add_task(datasource_id, 'add_folder', {'name': cfg_folder.name}) diff --git a/tableau_utilities/scripts/gen_config.py b/tableau_utilities/scripts/gen_config.py index ab8c367..f6a55c8 100644 --- a/tableau_utilities/scripts/gen_config.py +++ b/tableau_utilities/scripts/gen_config.py @@ -253,6 +253,50 @@ def build_folder_mapping(folders): return mappings +def build_configs(datasource_path, datasource_name, debugging_logs=False, definitions_csv_path=None): + """ + + Args: + datasource_path: + datasource_name: + debugging_logs: + definitions_csv_path: + + Returns: + + """ + + datasource = Datasource(datasource_path) + # Get column information from the metadata records + metadata_record_config = get_metadata_record_config( + datasource.connection.metadata_records, + datasource_name, + debugging_logs + ) + + # Get the mapping of definitions from the csv + definitions_mapping = dict() + if definitions_csv_path is not None: + definitions_mapping = load_csv_with_definitions(file=definitions_csv_path) + + # Extract the columns and folders. Build the new config + folder_mapping = build_folder_mapping(datasource.folders_common) + column_configs, calculated_column_configs = create_column_config( + columns=datasource.columns, + datasource_name=datasource_name, + folder_mapping=folder_mapping, + metadata_record_columns=metadata_record_config, + definitions_mapping=definitions_mapping, + debugging_logs=debugging_logs + ) + + # Sort configs + column_configs = dict(sorted(column_configs.items())) + calculated_column_configs = dict(sorted(calculated_column_configs.items())) + + return column_configs, calculated_column_configs + + def generate_config(args, server: TableauServer = None): """ Downloads a datasource and saves configs for that datasource @@ -293,33 +337,37 @@ def generate_config(args, server: TableauServer = None): print(f'{color.fg_yellow}BUILDING CONFIG {symbol.arrow_r} ' f'{color.fg_grey}{datasource_name} {symbol.sep} {datasource_path}{color.reset}') - datasource = Datasource(datasource_path) - # Get column information from the metadata records - metadata_record_config = get_metadata_record_config( - datasource.connection.metadata_records, - datasource_name, - debugging_logs - ) - - # Get the mapping of definitions from the csv - definitions_mapping = dict() - if definitions_csv_path is not None: - definitions_mapping = load_csv_with_definitions(file=definitions_csv_path) - - # Extract the columns and folders. Build the new config - folder_mapping = build_folder_mapping(datasource.folders_common) - column_configs, calculated_column_configs = create_column_config( - columns=datasource.columns, - datasource_name=datasource_name, - folder_mapping=folder_mapping, - metadata_record_columns=metadata_record_config, - definitions_mapping=definitions_mapping, - debugging_logs=debugging_logs - ) - # Sort configs - column_configs = dict(sorted(column_configs.items())) - calculated_column_configs = dict(sorted(calculated_column_configs.items())) + column_configs, calculated_column_configs = build_configs(datasource_path, debugging_logs, datasource_name, + definitions_csv_path) + + # datasource = Datasource(datasource_path) + # # Get column information from the metadata records + # metadata_record_config = get_metadata_record_config( + # datasource.connection.metadata_records, + # datasource_name, + # debugging_logs + # ) + # + # # Get the mapping of definitions from the csv + # definitions_mapping = dict() + # if definitions_csv_path is not None: + # definitions_mapping = load_csv_with_definitions(file=definitions_csv_path) + # + # # Extract the columns and folders. Build the new config + # folder_mapping = build_folder_mapping(datasource.folders_common) + # column_configs, calculated_column_configs = create_column_config( + # columns=datasource.columns, + # datasource_name=datasource_name, + # folder_mapping=folder_mapping, + # metadata_record_columns=metadata_record_config, + # definitions_mapping=definitions_mapping, + # debugging_logs=debugging_logs + # ) + # + # # Sort configs + # column_configs = dict(sorted(column_configs.items())) + # calculated_column_configs = dict(sorted(calculated_column_configs.items())) datasource_name_snake = convert_to_snake_case(datasource_name) output_file_column_config = 'column_config.json' diff --git a/tableau_utilities/tableau_file/tableau_file.py b/tableau_utilities/tableau_file/tableau_file.py index c239c02..63f6f56 100644 --- a/tableau_utilities/tableau_file/tableau_file.py +++ b/tableau_utilities/tableau_file/tableau_file.py @@ -301,10 +301,6 @@ def enforce_column(self, column, folder_name=None, remote_name=None): if not found: self.extract.connection.cols.append(extract_col) - - def update_metadata(self): - pass - def remove_empty_folders(self): """ Removes any folder without a column in it From 20daaf7801c2caede4a7c2a6669c4b37d0df14b8 Mon Sep 17 00:00:00 2001 From: Jay Rosenthal Date: Sun, 2 Jun 2024 14:39:48 -0700 Subject: [PATCH 10/44] add comments and delete duplicate code --- tableau_utilities/scripts/apply_config.py | 6 ++-- tableau_utilities/scripts/gen_config.py | 40 +++++------------------ 2 files changed, 11 insertions(+), 35 deletions(-) diff --git a/tableau_utilities/scripts/apply_config.py b/tableau_utilities/scripts/apply_config.py index d5685f3..22f9927 100644 --- a/tableau_utilities/scripts/apply_config.py +++ b/tableau_utilities/scripts/apply_config.py @@ -1,4 +1,4 @@ - +from gen_config import build_configs def compare_config_and_datasource(config, datasource_file, datasource_name: @@ -34,8 +34,8 @@ def apply_config(column_config, calculated_field_config, datasource): """ - # Generate the column_config from the datasource - # Generate the calculation config from the datasource + # Generate the column_configs from the datasource + # Get the changes to make for the column config # Get the changes to make for the calculation config diff --git a/tableau_utilities/scripts/gen_config.py b/tableau_utilities/scripts/gen_config.py index f6a55c8..91eb142 100644 --- a/tableau_utilities/scripts/gen_config.py +++ b/tableau_utilities/scripts/gen_config.py @@ -257,12 +257,14 @@ def build_configs(datasource_path, datasource_name, debugging_logs=False, defini """ Args: - datasource_path: - datasource_name: - debugging_logs: - definitions_csv_path: + datasource_path: The path to a datasource file + datasource_name: The name of the datasource + debugging_logs: True to print debugging logs to the console + definitions_csv_path: The path to a .csv with data definitions Returns: + column_configs: A dictionary with the column configs + calculated_column_configs = A dictionary with the calculated field configs """ @@ -338,37 +340,11 @@ def generate_config(args, server: TableauServer = None): print(f'{color.fg_yellow}BUILDING CONFIG {symbol.arrow_r} ' f'{color.fg_grey}{datasource_name} {symbol.sep} {datasource_path}{color.reset}') + # Build the config dictionaries column_configs, calculated_column_configs = build_configs(datasource_path, debugging_logs, datasource_name, definitions_csv_path) - # datasource = Datasource(datasource_path) - # # Get column information from the metadata records - # metadata_record_config = get_metadata_record_config( - # datasource.connection.metadata_records, - # datasource_name, - # debugging_logs - # ) - # - # # Get the mapping of definitions from the csv - # definitions_mapping = dict() - # if definitions_csv_path is not None: - # definitions_mapping = load_csv_with_definitions(file=definitions_csv_path) - # - # # Extract the columns and folders. Build the new config - # folder_mapping = build_folder_mapping(datasource.folders_common) - # column_configs, calculated_column_configs = create_column_config( - # columns=datasource.columns, - # datasource_name=datasource_name, - # folder_mapping=folder_mapping, - # metadata_record_columns=metadata_record_config, - # definitions_mapping=definitions_mapping, - # debugging_logs=debugging_logs - # ) - # - # # Sort configs - # column_configs = dict(sorted(column_configs.items())) - # calculated_column_configs = dict(sorted(calculated_column_configs.items())) - + # Output the configs to files datasource_name_snake = convert_to_snake_case(datasource_name) output_file_column_config = 'column_config.json' output_file_calculated_column_config = 'tableau_calc_config.json' From 292546c48b406b9a79a312d008d17fe1831d7e36 Mon Sep 17 00:00:00 2001 From: Jay Rosenthal Date: Sun, 2 Jun 2024 15:29:23 -0700 Subject: [PATCH 11/44] getting args validation error as expected --- tableau_utilities/scripts/apply_config.py | 53 ------------------ tableau_utilities/scripts/apply_configs.py | 63 ++++++++++++++++++++++ tableau_utilities/scripts/cli.py | 18 +++++++ tableau_utilities/scripts/gen_config.py | 2 +- 4 files changed, 82 insertions(+), 54 deletions(-) delete mode 100644 tableau_utilities/scripts/apply_config.py create mode 100644 tableau_utilities/scripts/apply_configs.py diff --git a/tableau_utilities/scripts/apply_config.py b/tableau_utilities/scripts/apply_config.py deleted file mode 100644 index 22f9927..0000000 --- a/tableau_utilities/scripts/apply_config.py +++ /dev/null @@ -1,53 +0,0 @@ -from gen_config import build_configs - - -def compare_config_and_datasource(config, datasource_file, datasource_name: - """ Compares the config to a datasource. Generates a list of changes to make the datasource match the config - - Returns: - dict: a dictionary with the columns that need updating - - """ - pass - - -def execute_changes(column_config, calculated_field_config, datasource): - """ Applies changes to make - - Args: - config: - datasource: - - Returns: - - """ - pass - -def apply_config(column_config, calculated_field_config, datasource): - """ Applies changes to make - - Args: - config: - datasource: - - Returns: - - """ - - # Generate the column_configs from the datasource - - - # Get the changes to make for the column config - # Get the changes to make for the calculation config - - # Apply the changes for the column config - # Apply the changes for the calc config - - # Clean up the empty folders - - # Save the file - - - - - pass diff --git a/tableau_utilities/scripts/apply_configs.py b/tableau_utilities/scripts/apply_configs.py new file mode 100644 index 0000000..6762050 --- /dev/null +++ b/tableau_utilities/scripts/apply_configs.py @@ -0,0 +1,63 @@ +from tableau_utilities.scripts.gen_config import build_configs + + +def compare_configs(config, datasource_cureent_config, datasource_name): + """ Compares the config to a datasource. Generates a list of changes to make the datasource match the config + + Returns: + dict: a dictionary with the columns that need updating + + """ + pass + + +def execute_changes(column_config, calculated_field_config, datasource): + """ Applies changes to make + + Args: + config: + datasource: + + Returns: + + """ + pass + +def apply_config_to_datasource(column_config, calculated_field_config, datasource_path, datasource_name): + """ Applies changes to make + + Args: + column_config: + calculated_field_config: + datasource_path: + datasource_name: + + Returns: + None + + """ + + # Build the config dictionaries from the datasource + datasource_current_column_config, datasource_current_calculated_column_config = build_configs(datasource_path, + datasource_name) + + # Get the changes to make for the column config + # Get the changes to make for the calculation config + + # Apply the changes for the column config + # Apply the changes for the calc config + + # Clean up the empty folders + + # Save the file + pass + + +def apply_configs(args): + # Set variables from the args + debugging_logs = args.debugging_logs + datasource_name = args.name + datasource_path = args.file_path + project_name = args.project_name + + apply_config_to_datasource() diff --git a/tableau_utilities/scripts/cli.py b/tableau_utilities/scripts/cli.py index 8a2d221..55c7b93 100644 --- a/tableau_utilities/scripts/cli.py +++ b/tableau_utilities/scripts/cli.py @@ -15,6 +15,7 @@ from tableau_utilities.scripts.server_operate import server_operate from tableau_utilities.scripts.datasource import datasource from tableau_utilities.scripts.csv_config import csv_config +from tableau_utilities.scripts.apply_configs import apply_configs __version__ = importlib.metadata.version('tableau_utilities') @@ -164,6 +165,8 @@ """E.g. "CREATED_AT" < '1/1/2024'""") parser_datasource.add_argument('-ci', '--column_init', action='store_true', help="Adds Columns from all Metadata Records, if they don't already exist.") parser_datasource.add_argument('-cf', '--clean_folders', action='store_true', help="Removes any empty folders without columns") +# parser_datasource.add_argument('-cc', '--column_config', help='The path to the column configs file') +# parser_datasource.add_argument('-cac', '--calculated_column_config', help='The path to the calculated field config file.') parser_datasource.set_defaults(func=datasource) # GENERATE CONFIG @@ -201,6 +204,14 @@ 'Use with --merge_with generate_merge_all') parser_config_merge.set_defaults(func=merge_configs) +# APPLY CONFIGS +parser_config_apply = subparsers.add_parser( + 'apply_configs', help='Applies a config to a datasource. Writes over any datasource attributes to make it ' + 'conform to the config.', formatter_class=RawTextHelpFormatter) +parser_config_apply.add_argument('-cc', '--column_config', help='The path to the column configs file') +parser_config_apply.add_argument('-cac', '--calculated_column_config', help='The path to the calculated field config file.') +parser_config_apply.set_defaults(func=apply_configs) + def validate_args_server_operate(args): """ Validate that combinations of args are present """ @@ -264,6 +275,11 @@ def validate_args_command_merge_config(args): parser.error(f'--merge_with {args.merge_with} requires --target_directory') +def validate_args_command_apply_configs(args): + if args.file_path is None or args.name is None or args.column_config is None or args.calculated_column_config is None: + parser.error(f'{args.command} requires --name and --file_path for a datasource and --column_config and --calculated_column_config') + + def validate_subpackage_hyper(): """ Checks that the hyper subpackage is installed for functions that use it """ @@ -456,6 +472,8 @@ def main(): validate_args_command_datasource(args) if args.command == 'merge_config': validate_args_command_merge_config(args) + if args.command == 'apply_configs': + validate_args_command_apply_configs(args) # Set/Reset the directory tmp_folder = args.output_dir diff --git a/tableau_utilities/scripts/gen_config.py b/tableau_utilities/scripts/gen_config.py index 91eb142..fc87bd8 100644 --- a/tableau_utilities/scripts/gen_config.py +++ b/tableau_utilities/scripts/gen_config.py @@ -341,7 +341,7 @@ def generate_config(args, server: TableauServer = None): f'{color.fg_grey}{datasource_name} {symbol.sep} {datasource_path}{color.reset}') # Build the config dictionaries - column_configs, calculated_column_configs = build_configs(datasource_path, debugging_logs, datasource_name, + column_configs, calculated_column_configs = build_configs(datasource_path, datasource_name, debugging_logs, definitions_csv_path) # Output the configs to files From 57e63a8686c79d0a8ba4215a612962a825fc14f7 Mon Sep 17 00:00:00 2001 From: Jay Rosenthal Date: Sun, 2 Jun 2024 15:35:09 -0700 Subject: [PATCH 12/44] wip --- tableau_utilities/scripts/apply_configs.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/tableau_utilities/scripts/apply_configs.py b/tableau_utilities/scripts/apply_configs.py index 6762050..5fd7741 100644 --- a/tableau_utilities/scripts/apply_configs.py +++ b/tableau_utilities/scripts/apply_configs.py @@ -23,14 +23,17 @@ def execute_changes(column_config, calculated_field_config, datasource): """ pass -def apply_config_to_datasource(column_config, calculated_field_config, datasource_path, datasource_name): +def apply_config_to_datasource(datasource_name, datasource_path, column_config, calculated_field_config, debugging_logs): """ Applies changes to make Args: + datasource_name: + datasource_path: column_config: calculated_field_config: - datasource_path: - datasource_name: + debugging_logs: + + Returns: None @@ -41,6 +44,8 @@ def apply_config_to_datasource(column_config, calculated_field_config, datasourc datasource_current_column_config, datasource_current_calculated_column_config = build_configs(datasource_path, datasource_name) + + # Get the changes to make for the column config # Get the changes to make for the calculation config @@ -58,6 +63,7 @@ def apply_configs(args): debugging_logs = args.debugging_logs datasource_name = args.name datasource_path = args.file_path - project_name = args.project_name + column_config = args.column_config + calculated_column_config = args.calculated_column_config apply_config_to_datasource() From 835402bde84ff82a6bfdcb48ff9d379d212418ce Mon Sep 17 00:00:00 2001 From: Jay Rosenthal Date: Mon, 3 Jun 2024 05:46:21 -0700 Subject: [PATCH 13/44] Move column_init to finction --- tableau_utilities/scripts/datasource.py | 110 +++++++++++++++++++----- 1 file changed, 89 insertions(+), 21 deletions(-) diff --git a/tableau_utilities/scripts/datasource.py b/tableau_utilities/scripts/datasource.py index 7be9826..de1c7d3 100644 --- a/tableau_utilities/scripts/datasource.py +++ b/tableau_utilities/scripts/datasource.py @@ -32,6 +32,73 @@ def create_column(name: str, persona: dict): return column + +def add_metadata_records_as_columns(ds, color=None, debugging_logs=False): + """ Adds records when they are only present in the + + When you create your Tableau extract the first time all columns will be present in Metadata records like this: + + + MY_COLUMN + 129 + [MY_COLUMN] + [Custom SQL Query] + MY_COLUMN + 1 + string + Count + 16777216 + true + + + "SQL_VARCHAR" + "SQL_C_CHAR" + "true" + + <_.fcp.ObjectModelEncapsulateLegacy.true...object-id>[_62A667B34C534415B10B2075B0DC36DC] + + + Separately some columns may have a column like this: + + + Manipulating Tableau columns requires a record. + + Args: + ds: A Datasource object + color: The cli color styling class + debugging_logs: True to print debugging information to the console + + Returns: + ds: An altered datasource. You'll still need to save this ds to apply the changes. + + """ + + # Create the list of columns to add + columns_to_add = [ + m for m in ds.connection.metadata_records + if m.local_name not in [c.name for c in ds.columns] + ] + print(f'{color.fg_yellow}Adding missing columns from Metadata Records:{color.reset} ' + f'{[m.local_name for m in columns_to_add]}') + + # Add the columns making the best guess of the proper persona + for m in columns_to_add: + if debugging_logs: + print(f'{color.fg_magenta}Metadata Record -> {m.local_name}:{color.reset} {m}') + + persona = get_persona_by_metadata_local_type(m.local_type) + persona_dict = personas.get(persona, {}) + if debugging_logs: + print(f' - {color.fg_blue}Persona -> {persona}:{color.reset} {persona_dict}') + + column = create_column(m.local_name, persona_dict) + + if debugging_logs: + print(f' - {color.fg_cyan}Creating Column -> {column.name}:{color.reset} {column.dict()}') + ds.enforce_column(column, remote_name=m.remote_name) + + return ds + def datasource(args, server=None): """ Updates a Tableau Datasource locally @@ -147,27 +214,28 @@ def datasource(args, server=None): # Column Init - Add columns for any column in Metadata records but not in columns if column_init: - columns_to_add = [ - m for m in ds.connection.metadata_records - if m.local_name not in [c.name for c in ds.columns] - ] - print(f'{color.fg_yellow}Adding missing columns from Metadata Records:{color.reset} ' - f'{[m.local_name for m in columns_to_add]}') - - for m in columns_to_add: - if debugging_logs: - print(f'{color.fg_magenta}Metadata Record -> {m.local_name}:{color.reset} {m}') - - persona = get_persona_by_metadata_local_type(m.local_type) - persona_dict = personas.get(persona, {}) - if debugging_logs: - print(f' - {color.fg_blue}Persona -> {persona}:{color.reset} {persona_dict}') - - column = create_column(m.local_name, persona_dict) - - if debugging_logs: - print(f' - {color.fg_cyan}Creating Column -> {column.name}:{color.reset} {column.dict()}') - ds.enforce_column(column, remote_name=m.remote_name) + ds = add_metadata_records_as_columns(ds, color, debugging_logs) + # columns_to_add = [ + # m for m in ds.connection.metadata_records + # if m.local_name not in [c.name for c in ds.columns] + # ] + # print(f'{color.fg_yellow}Adding missing columns from Metadata Records:{color.reset} ' + # f'{[m.local_name for m in columns_to_add]}') + # + # for m in columns_to_add: + # if debugging_logs: + # print(f'{color.fg_magenta}Metadata Record -> {m.local_name}:{color.reset} {m}') + # + # persona = get_persona_by_metadata_local_type(m.local_type) + # persona_dict = personas.get(persona, {}) + # if debugging_logs: + # print(f' - {color.fg_blue}Persona -> {persona}:{color.reset} {persona_dict}') + # + # column = create_column(m.local_name, persona_dict) + # + # if debugging_logs: + # print(f' - {color.fg_cyan}Creating Column -> {column.name}:{color.reset} {column.dict()}') + # ds.enforce_column(column, remote_name=m.remote_name) # Add / modify a specified column From 71559be323ccfab6356f957fe10f3e66bc0cce2a Mon Sep 17 00:00:00 2001 From: Jay Rosenthal Date: Mon, 3 Jun 2024 06:35:14 -0700 Subject: [PATCH 14/44] clean comments and docs --- tableau_utilities/scripts/datasource.py | 46 +++++++------------------ 1 file changed, 12 insertions(+), 34 deletions(-) diff --git a/tableau_utilities/scripts/datasource.py b/tableau_utilities/scripts/datasource.py index de1c7d3..a634836 100644 --- a/tableau_utilities/scripts/datasource.py +++ b/tableau_utilities/scripts/datasource.py @@ -38,28 +38,28 @@ def add_metadata_records_as_columns(ds, color=None, debugging_logs=False): When you create your Tableau extract the first time all columns will be present in Metadata records like this: - + MY_COLUMN - 129 + 131 [MY_COLUMN] [Custom SQL Query] MY_COLUMN - 1 - string - Count - 16777216 + 5 + integer + Sum + 38 + 0 true - - "SQL_VARCHAR" - "SQL_C_CHAR" - "true" + "SQL_DECIMAL" + "SQL_C_NUMERIC" <_.fcp.ObjectModelEncapsulateLegacy.true...object-id>[_62A667B34C534415B10B2075B0DC36DC] - + Separately some columns may have a column like this: - + + Manipulating Tableau columns requires a record. @@ -215,28 +215,6 @@ def datasource(args, server=None): # Column Init - Add columns for any column in Metadata records but not in columns if column_init: ds = add_metadata_records_as_columns(ds, color, debugging_logs) - # columns_to_add = [ - # m for m in ds.connection.metadata_records - # if m.local_name not in [c.name for c in ds.columns] - # ] - # print(f'{color.fg_yellow}Adding missing columns from Metadata Records:{color.reset} ' - # f'{[m.local_name for m in columns_to_add]}') - # - # for m in columns_to_add: - # if debugging_logs: - # print(f'{color.fg_magenta}Metadata Record -> {m.local_name}:{color.reset} {m}') - # - # persona = get_persona_by_metadata_local_type(m.local_type) - # persona_dict = personas.get(persona, {}) - # if debugging_logs: - # print(f' - {color.fg_blue}Persona -> {persona}:{color.reset} {persona_dict}') - # - # column = create_column(m.local_name, persona_dict) - # - # if debugging_logs: - # print(f' - {color.fg_cyan}Creating Column -> {column.name}:{color.reset} {column.dict()}') - # ds.enforce_column(column, remote_name=m.remote_name) - # Add / modify a specified column if column_name and not delete: From 0924b79f77017eb5d49a34b4289654e9bfd31b5d Mon Sep 17 00:00:00 2001 From: Jay Rosenthal Date: Mon, 3 Jun 2024 09:03:25 -0700 Subject: [PATCH 15/44] define color and symbols as globals so it will all work in functions --- tableau_utilities/scripts/apply_configs.py | 24 ++++++++++++++++++++++ tableau_utilities/scripts/datasource.py | 7 ++++--- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/tableau_utilities/scripts/apply_configs.py b/tableau_utilities/scripts/apply_configs.py index 5fd7741..7efe8e5 100644 --- a/tableau_utilities/scripts/apply_configs.py +++ b/tableau_utilities/scripts/apply_configs.py @@ -1,5 +1,29 @@ +from copy import deepcopy + from tableau_utilities.scripts.gen_config import build_configs +def invert_config(config): + """ Helper function to invert the column config and calc config. + Output -> {datasource: {column: info}} + + Args: + iterator (dict): The iterator to append invert data to. + config (dict): The config to invert. + """ + + temp_config = {} + + for column, i in config.items(): + for datasource in i['datasources']: + new_info = deepcopy(i) + del new_info['datasources'] + new_info['local-name'] = datasource['local-name'] + new_info['remote_name'] = datasource['sql_alias'] if 'sql_alias' in datasource else None + iterator.setdefault(datasource['name'], {column: new_info}) + iterator[datasource['name']].setdefault(column, new_info) + +def combine_configs(): + pass def compare_configs(config, datasource_cureent_config, datasource_name): """ Compares the config to a datasource. Generates a list of changes to make the datasource match the config diff --git a/tableau_utilities/scripts/datasource.py b/tableau_utilities/scripts/datasource.py index a634836..6da3777 100644 --- a/tableau_utilities/scripts/datasource.py +++ b/tableau_utilities/scripts/datasource.py @@ -11,6 +11,10 @@ from tableau_utilities.tableau_server.tableau_server import TableauServer +# Define color and symbol as globals +color = Color() +symbol = Symbol() + def create_column(name: str, persona: dict): """ Creates the tfo column object with the minimum required fields to add a column @@ -145,9 +149,6 @@ def datasource(args, server=None): conn_schema = args.conn_schema conn_warehouse = args.conn_warehouse - # Print Styling - color = Color() - symbol = Symbol() # Downloads the datasource from Tableau Server if the datasource is not local if location == 'online': From 1b774f9060f06aebda62cbcc02eeb0f5be5284cc Mon Sep 17 00:00:00 2001 From: Jay Rosenthal Date: Mon, 3 Jun 2024 09:03:50 -0700 Subject: [PATCH 16/44] define color and symbols as globals so it will all work in functions that are used anywhere --- tableau_utilities/scripts/datasource.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tableau_utilities/scripts/datasource.py b/tableau_utilities/scripts/datasource.py index 6da3777..0e1e1f5 100644 --- a/tableau_utilities/scripts/datasource.py +++ b/tableau_utilities/scripts/datasource.py @@ -37,7 +37,7 @@ def create_column(name: str, persona: dict): return column -def add_metadata_records_as_columns(ds, color=None, debugging_logs=False): +def add_metadata_records_as_columns(ds, debugging_logs=False): """ Adds records when they are only present in the When you create your Tableau extract the first time all columns will be present in Metadata records like this: From ccb8c59a77e627de9fa68c9e92f08b47adfbab76 Mon Sep 17 00:00:00 2001 From: Jay Rosenthal Date: Tue, 4 Jun 2024 05:40:52 -0700 Subject: [PATCH 17/44] more code --- .github/workflows/python-package.yml | 4 +- tableau_utilities/scripts/apply_configs.py | 171 ++++++++++++++------- tableau_utilities/scripts/gen_config.py | 9 +- tests/test_apply_configs.py | 150 ++++++++++++++++++ 4 files changed, 274 insertions(+), 60 deletions(-) create mode 100644 tests/test_apply_configs.py diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 4755bea..14e8970 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -32,9 +32,9 @@ jobs: flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - - name: Run Pytest mock tests + - name: Run Pytest on the tests in tests/ run: - pytest tests/test_datasource_remove_empty_folders.py -p no:warnings + pytest tests/ -p no:warnings - name: Run pytest on tableau-utilities run: | cd tableau_utilities && pytest -v diff --git a/tableau_utilities/scripts/apply_configs.py b/tableau_utilities/scripts/apply_configs.py index 7efe8e5..36e11d1 100644 --- a/tableau_utilities/scripts/apply_configs.py +++ b/tableau_utilities/scripts/apply_configs.py @@ -1,85 +1,146 @@ from copy import deepcopy +import pprint +from tableau_utilities.tableau_file.tableau_file import Datasource from tableau_utilities.scripts.gen_config import build_configs +from tableau_utilities.scripts.datasource import add_metadata_records_as_columns -def invert_config(config): - """ Helper function to invert the column config and calc config. - Output -> {datasource: {column: info}} +class ApplyConfigs: + def __init__(self, datasource_name, datasource_path, column_config, calculated_column_config, debugging_logs): + self.datasource_name = datasource_name + self.datasource_path = datasource_path + self.column_config = column_config + self.calculated_column_config = calculated_column_config + self.debugging_logs = debugging_logs - Args: - iterator (dict): The iterator to append invert data to. - config (dict): The config to invert. - """ - temp_config = {} - for column, i in config.items(): - for datasource in i['datasources']: - new_info = deepcopy(i) - del new_info['datasources'] - new_info['local-name'] = datasource['local-name'] - new_info['remote_name'] = datasource['sql_alias'] if 'sql_alias' in datasource else None - iterator.setdefault(datasource['name'], {column: new_info}) - iterator[datasource['name']].setdefault(column, new_info) + def invert_config(self, config): + """ Helper function to invert the column config and calc config. + Output -> {datasource: {column: info}} -def combine_configs(): - pass + Args: + iterator (dict): The iterator to append invert data to. + config (dict): The config to invert. + """ -def compare_configs(config, datasource_cureent_config, datasource_name): - """ Compares the config to a datasource. Generates a list of changes to make the datasource match the config + temp_config = {} - Returns: - dict: a dictionary with the columns that need updating + for column, i in config.items(): + for datasource in i['datasources']: + new_info = deepcopy(i) + del new_info['datasources'] + new_info['local-name'] = datasource['local-name'] + new_info['remote_name'] = datasource['sql_alias'] if 'sql_alias' in datasource else None + temp_config.setdefault(datasource['name'], {column: new_info}) + temp_config[datasource['name']].setdefault(column, new_info) - """ - pass + if self.debugging_logs: + pp = pprint.PrettyPrinter(indent=4, width=80, depth=None, compact=False) + pp.pprint(temp_config) + return temp_config -def execute_changes(column_config, calculated_field_config, datasource): - """ Applies changes to make + def combine_configs(self): + pass - Args: - config: - datasource: + def prepare_configs(self, config_A, config_B): + """ Takes 2 configs to invert, combine, and remove irrelevant datasource information. Columns in a main config + can be in 1 or many Tableau datasources. So when managing multiple datasources it's likely to have columns that + need removal - Returns: + Args: + config_A: + config_B: - """ - pass + Returns: -def apply_config_to_datasource(datasource_name, datasource_path, column_config, calculated_field_config, debugging_logs): - """ Applies changes to make + """ + # datasource = self.invert_config(self.column_config) + # self.invert_config(self.calculated_column_config) + # combined_config = {**dict1, **dict2} + pass - Args: - datasource_name: - datasource_path: - column_config: - calculated_field_config: - debugging_logs: + def compare_columns(self): + """ Compares the config to a datasource. Generates a list of changes to make the datasource match the config + Returns: + dict: a dictionary with the columns that need updating + """ - Returns: - None + # compare the caption. If the caption matches compare the attributes + pass - """ + def compare_configs(self, config, datasource_cureent_config, datasource_name): + """ Compares the config to a datasource. Generates a list of changes to make the datasource match the config - # Build the config dictionaries from the datasource - datasource_current_column_config, datasource_current_calculated_column_config = build_configs(datasource_path, - datasource_name) + Returns: + dict: a dictionary with the columns that need updating + """ + # compare the caption. If the caption matches compare the attributes + pass - # Get the changes to make for the column config - # Get the changes to make for the calculation config - # Apply the changes for the column config - # Apply the changes for the calc config + def execute_changes(self, column_config, calculated_field_config, datasource): + """ Applies changes to make - # Clean up the empty folders + Args: + config: + datasource: - # Save the file - pass + Returns: + + """ + pass + + def apply_config_to_datasource(self): + """ Applies changes to make + + Args: + datasource_name: + datasource_path: + column_config: + calculated_field_config: + debugging_logs: + + + + Returns: + None + + """ + + datasource = Datasource(self.datasource_path) + + # Run column init on the datasource to make sure columns aren't hiding in Metadata records + datasource = add_metadata_records_as_columns(datasource, self.debugging_logs) + + # Build the config dictionaries from the datasource + datasource_current_column_config, datasource_current_calculated_column_config = build_configs(datasource, + self.datasource_name) + # Prepare the configs by inverting, combining and removing configs for other datasources + config = self.prepare_configs() + current_datasource_config = self.prepare_configs() + + # datasource = self.invert_config(self.column_config) + # self.invert_config(self.calculated_column_config) + # combined_config = {**dict1, **dict2} + + + + # Get the changes to make for the column config + # Get the changes to make for the calculation config + + # Apply the changes for the column config + # Apply the changes for the calc config + + # Clean up the empty folders + + # Save the file + pass def apply_configs(args): @@ -90,4 +151,6 @@ def apply_configs(args): column_config = args.column_config calculated_column_config = args.calculated_column_config - apply_config_to_datasource() + AC = ApplyConfigs(datasource_name, datasource_path, column_config, calculated_column_config, debugging_logs) + + AC.apply_config_to_datasource() diff --git a/tableau_utilities/scripts/gen_config.py b/tableau_utilities/scripts/gen_config.py index fc87bd8..c403de0 100644 --- a/tableau_utilities/scripts/gen_config.py +++ b/tableau_utilities/scripts/gen_config.py @@ -253,11 +253,11 @@ def build_folder_mapping(folders): return mappings -def build_configs(datasource_path, datasource_name, debugging_logs=False, definitions_csv_path=None): +def build_configs(datasource, datasource_name, debugging_logs=False, definitions_csv_path=None): """ Args: - datasource_path: The path to a datasource file + datasource: A Tableau utilities datasource object datasource_name: The name of the datasource debugging_logs: True to print debugging logs to the console definitions_csv_path: The path to a .csv with data definitions @@ -268,7 +268,6 @@ def build_configs(datasource_path, datasource_name, debugging_logs=False, defini """ - datasource = Datasource(datasource_path) # Get column information from the metadata records metadata_record_config = get_metadata_record_config( datasource.connection.metadata_records, @@ -340,8 +339,10 @@ def generate_config(args, server: TableauServer = None): print(f'{color.fg_yellow}BUILDING CONFIG {symbol.arrow_r} ' f'{color.fg_grey}{datasource_name} {symbol.sep} {datasource_path}{color.reset}') + datasource = Datasource(datasource_path) + # Build the config dictionaries - column_configs, calculated_column_configs = build_configs(datasource_path, datasource_name, debugging_logs, + column_configs, calculated_column_configs = build_configs(datasource, datasource_name, debugging_logs, definitions_csv_path) # Output the configs to files diff --git a/tests/test_apply_configs.py b/tests/test_apply_configs.py new file mode 100644 index 0000000..6e9d9b0 --- /dev/null +++ b/tests/test_apply_configs.py @@ -0,0 +1,150 @@ +import pytest +from tableau_utilities.scripts.apply_configs import ApplyConfigs + + +@pytest.fixture +def apply_configs(): + return ApplyConfigs(datasource_name="", datasource_path="", column_config={}, calculated_column_config={}, debugging_logs=False) + +def test_invert_config_single_datasource(apply_configs): + sample_config = { + "Column1": { + "description": "Description of Column1", + "folder": "Folder1", + "persona": "string_dimension", + "datasources": [ + { + "name": "my_datasource_1", + "local-name": "MY_COLUMN_1", + "sql_alias": "MY_COLUMN_1_ALIAS" + } + ] + } + } + + expected_output = { + "my_datasource_1": { + "Column1": { + "description": "Description of Column1", + "folder": "Folder1", + "persona": "string_dimension", + "local-name": "MY_COLUMN_1", + "remote_name": "MY_COLUMN_1_ALIAS" + } + } + } + + result = apply_configs.invert_config(sample_config) + assert result == expected_output + +def test_invert_config_multiple_datasources(apply_configs): + sample_config = { + "Column2": { + "description": "Description of Column2", + "folder": "Folder2", + "persona": "string_dimension", + "datasources": [ + { + "name": "my_datasource_1", + "local-name": "MY_COLUMN_1", + "sql_alias": "MY_COLUMN_1_ALIAS" + }, + { + "name": "my_datasource_2", + "local-name": "MY_COLUMN_2", + "sql_alias": "MY_COLUMN_2_ALIAS" + } + ] + } + } + + expected_output = { + "my_datasource_1": { + "Column2": { + "description": "Description of Column2", + "folder": "Folder2", + "persona": "string_dimension", + "local-name": "MY_COLUMN_1", + "remote_name": "MY_COLUMN_1_ALIAS" + } + }, + "my_datasource_2": { + "Column2": { + "description": "Description of Column2", + "folder": "Folder2", + "persona": "string_dimension", + "local-name": "MY_COLUMN_2", + "remote_name": "MY_COLUMN_2_ALIAS" + } + } + } + + result = apply_configs.invert_config(sample_config) + assert result == expected_output + +def test_invert_config_combined(apply_configs): + sample_config = { + "Column1": { + "description": "Description of Column1", + "folder": "Folder1", + "persona": "string_dimension", + "datasources": [ + { + "name": "my_datasource_1", + "local-name": "MY_COLUMN_1", + "sql_alias": "MY_COLUMN_1_ALIAS" + } + ] + }, + "Column2": { + "description": "Description of Column2", + "folder": "Folder2", + "persona": "string_dimension", + "datasources": [ + { + "name": "my_datasource_1", + "local-name": "MY_COLUMN_1", + "sql_alias": "MY_COLUMN_1_ALIAS" + }, + { + "name": "my_datasource_2", + "local-name": "MY_COLUMN_2", + "sql_alias": "MY_COLUMN_2_ALIAS" + } + ] + } + } + + expected_output = { + "my_datasource_1": { + "Column1": { + "description": "Description of Column1", + "folder": "Folder1", + "persona": "string_dimension", + "local-name": "MY_COLUMN_1", + "remote_name": "MY_COLUMN_1_ALIAS" + }, + "Column2": { + "description": "Description of Column2", + "folder": "Folder2", + "persona": "string_dimension", + "local-name": "MY_COLUMN_1", + "remote_name": "MY_COLUMN_1_ALIAS" + } + }, + "my_datasource_2": { + "Column2": { + "description": "Description of Column2", + "folder": "Folder2", + "persona": "string_dimension", + "local-name": "MY_COLUMN_2", + "remote_name": "MY_COLUMN_2_ALIAS" + } + } + } + + result = apply_configs.invert_config(sample_config) + assert result == expected_output + +if __name__ == '__main__': + pytest.main() From 779781db0bf0c139f7059f96107259cb2758deec Mon Sep 17 00:00:00 2001 From: Jay Rosenthal Date: Tue, 4 Jun 2024 05:56:06 -0700 Subject: [PATCH 18/44] Clean naming --- .../tableau_datasource_update.py | 88 +++++++++---------- tableau_utilities/scripts/apply_configs.py | 30 +++---- 2 files changed, 59 insertions(+), 59 deletions(-) diff --git a/airflow_example/dags/tableau_datasource_update/tableau_datasource_update.py b/airflow_example/dags/tableau_datasource_update/tableau_datasource_update.py index 7d38f40..d740ae1 100644 --- a/airflow_example/dags/tableau_datasource_update/tableau_datasource_update.py +++ b/airflow_example/dags/tableau_datasource_update/tableau_datasource_update.py @@ -314,54 +314,54 @@ def __compare_folders(self, datasource_id, tds_folders, cfg_folders): def execute(self, context): """ Update Tableau datasource according to config. """ - github_conn = BaseHook.get_connection(self.github_conn_id) - config = cfg.Config( - githup_token=github_conn.password, - repo_name=github_conn.extra_dejson.get('repo_name'), - repo_branch=github_conn.extra_dejson.get('repo_branch'), - subfolder=github_conn.extra_dejson.get('subfolder') - ) - - ts = get_tableau_server(self.tableau_conn_id) - expected_conn_attrs = self.__set_connection_attributes() - - # Get the ID for each datasource in the config - for ds in ts.get.datasources(): - if ds not in config.datasources: - continue - config.datasources[ds].id = ds.id - - for datasource in config.datasources: - if not datasource.id: - logging.error('!! Datasource not found in Tableau Online: %s / %s', - datasource.project_name, datasource.name) - continue - dsid = datasource.id + # github_conn = BaseHook.get_connection(self.github_conn_id) + # config = cfg.Config( + # githup_token=github_conn.password, + # repo_name=github_conn.extra_dejson.get('repo_name'), + # repo_branch=github_conn.extra_dejson.get('repo_branch'), + # subfolder=github_conn.extra_dejson.get('subfolder') + # ) + # + # ts = get_tableau_server(self.tableau_conn_id) + # expected_conn_attrs = self.__set_connection_attributes() + # + # # Get the ID for each datasource in the config + # for ds in ts.get.datasources(): + # if ds not in config.datasources: + # continue + # config.datasources[ds].id = ds.id + # + # # for datasource in config.datasources: + # if not datasource.id: + # logging.error('!! Datasource not found in Tableau Online: %s / %s', + # datasource.project_name, datasource.name) + # continue + # dsid = datasource.id # Set default dict attributes for tasks, for each datasource self.tasks[dsid] = {a: [] for a in UPDATE_ACTIONS} self.tasks[dsid]['project'] = datasource.project_name self.tasks[dsid]['datasource_name'] = datasource.name - if not config.in_maintenance_window and AIRFLOW_ENV not in ['STAGING', 'DEV']: - self.tasks[dsid]['skip'] = 'Outside maintenance window' - logging.info('(SKIP) Outside maintenance window: %s', datasource.name) - continue - elif datasource.name in EXCLUDED_DATASOURCES: - self.tasks[dsid]['skip'] = 'Marked to exclude' - logging.info('(SKIP) Marked to exclude: %s', datasource.name) - continue - logging.info('Checking Datasource: %s', datasource.name) - # Download the Datasource for comparison - dl_path = f"downloads/{dsid}/" - os.makedirs(dl_path, exist_ok=True) - ds_path = ts.download.datasource(dsid, file_dir=dl_path, include_extract=False) - tds = Datasource(ds_path) - # Cleanup downloaded file after assigning the Datasource - shutil.rmtree(dl_path, ignore_errors=True) - # Add connection task, if there is a difference - self.__compare_connection(dsid, datasource.name, tds.connection, expected_conn_attrs) - # Add folder tasks, if folders need to be added/deleted - self.__compare_folders(dsid, tds.folders_common, datasource.folders) - # Add Column tasks, if there are missing columns, or columns need to be updated + # if not config.in_maintenance_window and AIRFLOW_ENV not in ['STAGING', 'DEV']: + # self.tasks[dsid]['skip'] = 'Outside maintenance window' + # logging.info('(SKIP) Outside maintenance window: %s', datasource.name) + # continue + # elif datasource.name in EXCLUDED_DATASOURCES: + # self.tasks[dsid]['skip'] = 'Marked to exclude' + # logging.info('(SKIP) Marked to exclude: %s', datasource.name) + # continue + # logging.info('Checking Datasource: %s', datasource.name) + # # Download the Datasource for comparison + # dl_path = f"downloads/{dsid}/" + # os.makedirs(dl_path, exist_ok=True) + # ds_path = ts.download.datasource(dsid, file_dir=dl_path, include_extract=False) + # tds = Datasource(ds_path) + # # Cleanup downloaded file after assigning the Datasource + # shutil.rmtree(dl_path, ignore_errors=True) + # # Add connection task, if there is a difference + # self.__compare_connection(dsid, datasource.name, tds.connection, expected_conn_attrs) + # # Add folder tasks, if folders need to be added/deleted + # self.__compare_folders(dsid, tds.folders_common, datasource.folders) + # # Add Column tasks, if there are missing columns, or columns need to be updated for column in datasource.columns: # Check if the column metadata needs to be updated self.__compare_column_metadata(dsid, tds, column) diff --git a/tableau_utilities/scripts/apply_configs.py b/tableau_utilities/scripts/apply_configs.py index 36e11d1..af5e12b 100644 --- a/tableau_utilities/scripts/apply_configs.py +++ b/tableau_utilities/scripts/apply_configs.py @@ -6,15 +6,17 @@ from tableau_utilities.scripts.datasource import add_metadata_records_as_columns class ApplyConfigs: - def __init__(self, datasource_name, datasource_path, column_config, calculated_column_config, debugging_logs): + """ Applies a set of configs to a datasource. Configs prefixed with target_ will be applied to the datasource. + Configs prefixed with datasource_ represent the current state of the datasource before changes. + + """ + def __init__(self, datasource_name, datasource_path, target_column_config, target_calculated_column_config, debugging_logs): self.datasource_name = datasource_name self.datasource_path = datasource_path - self.column_config = column_config - self.calculated_column_config = calculated_column_config + self.target_column_config = target_column_config + self.target_calculated_column_config = target_calculated_column_config self.debugging_logs = debugging_logs - - def invert_config(self, config): """ Helper function to invert the column config and calc config. Output -> {datasource: {column: info}} @@ -97,7 +99,8 @@ def execute_changes(self, column_config, calculated_field_config, datasource): pass def apply_config_to_datasource(self): - """ Applies changes to make + """ Applies a set of configs (column_config and calculated_column_config) to a datasource. + If a column is in a datasource but NOT in the config that column will be unchanged. Args: datasource_name: @@ -119,18 +122,15 @@ def apply_config_to_datasource(self): datasource = add_metadata_records_as_columns(datasource, self.debugging_logs) # Build the config dictionaries from the datasource - datasource_current_column_config, datasource_current_calculated_column_config = build_configs(datasource, - self.datasource_name) + datasource_column_config, datasource_calculated_column_config = build_configs(datasource, self.datasource_name) # Prepare the configs by inverting, combining and removing configs for other datasources - config = self.prepare_configs() - current_datasource_config = self.prepare_configs() + target_config = self.prepare_configs(self.target_column_config, self.target_calculated_column_config) + datasource_config = self.prepare_configs(datasource_column_config, datasource_calculated_column_config) # datasource = self.invert_config(self.column_config) # self.invert_config(self.calculated_column_config) # combined_config = {**dict1, **dict2} - - # Get the changes to make for the column config # Get the changes to make for the calculation config @@ -148,9 +148,9 @@ def apply_configs(args): debugging_logs = args.debugging_logs datasource_name = args.name datasource_path = args.file_path - column_config = args.column_config - calculated_column_config = args.calculated_column_config + target_column_config = args.column_config + target_calculated_column_config = args.calculated_column_config - AC = ApplyConfigs(datasource_name, datasource_path, column_config, calculated_column_config, debugging_logs) + AC = ApplyConfigs(datasource_name, datasource_path, target_column_config, target_calculated_column_config, debugging_logs) AC.apply_config_to_datasource() From 10921fe759b1dd586eee6ed192d41a2597e1c058 Mon Sep 17 00:00:00 2001 From: Jay Rosenthal Date: Tue, 4 Jun 2024 06:43:46 -0700 Subject: [PATCH 19/44] Add lots of stuff --- tableau_utilities/scripts/apply_configs.py | 75 ++++++++++++++++------ tests/test_apply_configs.py | 49 +++++++++++++- 2 files changed, 103 insertions(+), 21 deletions(-) diff --git a/tableau_utilities/scripts/apply_configs.py b/tableau_utilities/scripts/apply_configs.py index af5e12b..aee4f19 100644 --- a/tableau_utilities/scripts/apply_configs.py +++ b/tableau_utilities/scripts/apply_configs.py @@ -1,32 +1,40 @@ from copy import deepcopy import pprint +from typing import Dict, Any from tableau_utilities.tableau_file.tableau_file import Datasource from tableau_utilities.scripts.gen_config import build_configs from tableau_utilities.scripts.datasource import add_metadata_records_as_columns class ApplyConfigs: - """ Applies a set of configs to a datasource. Configs prefixed with target_ will be applied to the datasource. + """Applies a set of configs to a datasource. Configs prefixed with target_ will be applied to the datasource. Configs prefixed with datasource_ represent the current state of the datasource before changes. - """ - def __init__(self, datasource_name, datasource_path, target_column_config, target_calculated_column_config, debugging_logs): - self.datasource_name = datasource_name - self.datasource_path = datasource_path - self.target_column_config = target_column_config - self.target_calculated_column_config = target_calculated_column_config - self.debugging_logs = debugging_logs - def invert_config(self, config): - """ Helper function to invert the column config and calc config. - Output -> {datasource: {column: info}} + def __init__(self, + datasource_name: str, + datasource_path: str, + target_column_config: Dict[str, Any], + target_calculated_column_config: Dict[str, Any], + debugging_logs: bool) -> None: + self.datasource_name: str = datasource_name + self.datasource_path: str = datasource_path + self.target_column_config: Dict[str, Any] = target_column_config + self.target_calculated_column_config: Dict[str, Any] = target_calculated_column_config + self.debugging_logs: bool = debugging_logs + + def invert_config(self, config: Dict[str, Any]) -> Dict[str, Any]: + """Helper function to invert the column config and calc config. + Output -> {datasource: {column: info}} Args: - iterator (dict): The iterator to append invert data to. config (dict): The config to invert. + + Returns: + dict: The inverted config. """ - temp_config = {} + inverted_config = {} for column, i in config.items(): for datasource in i['datasources']: @@ -34,18 +42,33 @@ def invert_config(self, config): del new_info['datasources'] new_info['local-name'] = datasource['local-name'] new_info['remote_name'] = datasource['sql_alias'] if 'sql_alias' in datasource else None - temp_config.setdefault(datasource['name'], {column: new_info}) - temp_config[datasource['name']].setdefault(column, new_info) + inverted_config.setdefault(datasource['name'], {column: new_info}) + inverted_config[datasource['name']].setdefault(column, new_info) if self.debugging_logs: pp = pprint.PrettyPrinter(indent=4, width=80, depth=None, compact=False) - pp.pprint(temp_config) + pp.pprint(inverted_config) - return temp_config + return inverted_config def combine_configs(self): pass + def select_matching_datasource_config(self, config): + """ Limit + + Args: + comfig: + + Returns: + A config with any datasource that is not self.datasource_name removed + + """ + + config = config[self.datasource_name] + return config + + def prepare_configs(self, config_A, config_B): """ Takes 2 configs to invert, combine, and remove irrelevant datasource information. Columns in a main config can be in 1 or many Tableau datasources. So when managing multiple datasources it's likely to have columns that @@ -58,6 +81,20 @@ def prepare_configs(self, config_A, config_B): Returns: """ + + # invert the configs + config_A = self.invert_config(config_A) + config_B = self.invert_config(config_B) + + # Get only the configs to the current datasource + config_A = self.select_matching_datasource_config(config_A) + config_B = self.select_matching_datasource_config(config_B) + + # Combine configs + + + + # datasource = self.invert_config(self.column_config) # self.invert_config(self.calculated_column_config) # combined_config = {**dict1, **dict2} @@ -127,9 +164,7 @@ def apply_config_to_datasource(self): target_config = self.prepare_configs(self.target_column_config, self.target_calculated_column_config) datasource_config = self.prepare_configs(datasource_column_config, datasource_calculated_column_config) - # datasource = self.invert_config(self.column_config) - # self.invert_config(self.calculated_column_config) - # combined_config = {**dict1, **dict2} + # Get the changes to make for the column config # Get the changes to make for the calculation config diff --git a/tests/test_apply_configs.py b/tests/test_apply_configs.py index 6e9d9b0..8bb66f9 100644 --- a/tests/test_apply_configs.py +++ b/tests/test_apply_configs.py @@ -1,10 +1,20 @@ import pytest +from typing import Dict, Any from tableau_utilities.scripts.apply_configs import ApplyConfigs + +# +# @pytest.fixture +# def apply_configs(): +# return ApplyConfigs(datasource_name="my_datasource_1", datasource_path="", target_column_config={}, +# target_calculated_column_config={}, debugging_logs=False) + @pytest.fixture def apply_configs(): - return ApplyConfigs(datasource_name="", datasource_path="", column_config={}, calculated_column_config={}, debugging_logs=False) + return ApplyConfigs(datasource_name="my_datasource_1", datasource_path="", target_column_config={}, + target_calculated_column_config={}, debugging_logs=False) + def test_invert_config_single_datasource(apply_configs): sample_config = { @@ -146,5 +156,42 @@ def test_invert_config_combined(apply_configs): result = apply_configs.invert_config(sample_config) assert result == expected_output +def test_select_matching_datasource_config(): + sample_config = { + "my_datasource_1": { + "Column2": { + "description": "Description of Column2", + "folder": "Folder2", + "persona": "string_dimension", + "local-name": "MY_COLUMN_1", + "remote_name": "MY_COLUMN_1_ALIAS" + } + }, + "my_datasource_2": { + "Column2": { + "description": "Description of Column2", + "folder": "Folder2", + "persona": "string_dimension", + "local-name": "MY_COLUMN_2", + "remote_name": "MY_COLUMN_2_ALIAS" + } + } + } + + expected_output = { + "my_datasource_1": { + "Column2": { + "description": "Description of Column2", + "folder": "Folder2", + "persona": "string_dimension", + "local-name": "MY_COLUMN_1", + "remote_name": "MY_COLUMN_1_ALIAS" + } + }, + } + + result = apply_configs.select_matching_datasource_config(sample_config) + assert result == expected_output + if __name__ == '__main__': pytest.main() From 9c2c5cc6d7eb143660c9c451f75237b068823ea3 Mon Sep 17 00:00:00 2001 From: Jay Rosenthal Date: Tue, 4 Jun 2024 07:12:32 -0700 Subject: [PATCH 20/44] passing tests and printing the number of tests to console --- tableau_utilities/scripts/apply_configs.py | 39 ++- tests/conftest.py | 25 ++ tests/test_apply_configs.py | 303 ++++++++++----------- 3 files changed, 199 insertions(+), 168 deletions(-) create mode 100644 tests/conftest.py diff --git a/tableau_utilities/scripts/apply_configs.py b/tableau_utilities/scripts/apply_configs.py index aee4f19..834b617 100644 --- a/tableau_utilities/scripts/apply_configs.py +++ b/tableau_utilities/scripts/apply_configs.py @@ -5,25 +5,38 @@ from tableau_utilities.tableau_file.tableau_file import Datasource from tableau_utilities.scripts.gen_config import build_configs from tableau_utilities.scripts.datasource import add_metadata_records_as_columns +# +# class ApplyConfigs: +# """Applies a set of configs to a datasource. Configs prefixed with target_ will be applied to the datasource. +# Configs prefixed with datasource_ represent the current state of the datasource before changes. +# """ +# +# def __init__(self, +# datasource_name: str, +# datasource_path: str, +# target_column_config: Dict[str, Any], +# target_calculated_column_config: Dict[str, Any], +# debugging_logs: bool) -> None: +# self.datasource_name: str = datasource_name +# self.datasource_path: str = datasource_path +# self.target_column_config: Dict[str, Any] = target_column_config +# self.target_calculated_column_config: Dict[str, Any] = target_calculated_column_config +# self.debugging_logs: bool = debugging_logs class ApplyConfigs: """Applies a set of configs to a datasource. Configs prefixed with target_ will be applied to the datasource. Configs prefixed with datasource_ represent the current state of the datasource before changes. """ - def __init__(self, - datasource_name: str, - datasource_path: str, - target_column_config: Dict[str, Any], - target_calculated_column_config: Dict[str, Any], - debugging_logs: bool) -> None: - self.datasource_name: str = datasource_name - self.datasource_path: str = datasource_path - self.target_column_config: Dict[str, Any] = target_column_config - self.target_calculated_column_config: Dict[str, Any] = target_calculated_column_config - self.debugging_logs: bool = debugging_logs - - def invert_config(self, config: Dict[str, Any]) -> Dict[str, Any]: + def __init__(self, datasource_name, datasource_path, target_column_config, target_calculated_column_config, debugging_logs): + self.datasource_name = datasource_name + self.datasource_path = datasource_path + self.target_column_config = target_column_config + self.target_calculated_column_config = target_calculated_column_config + self.debugging_logs = debugging_logs + + def invert_config(self, config): + # def invert_config(self, config: Dict[str, Any]) -> Dict[str, Any]: """Helper function to invert the column config and calc config. Output -> {datasource: {column: info}} diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..33e282d --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,25 @@ +import pytest + +file_test_count = {} + +@pytest.hookimpl(tryfirst=True) +def pytest_sessionstart(session): + global file_test_count + file_test_count = {} + +@pytest.hookimpl(tryfirst=True) +def pytest_runtestloop(session): + global file_test_count + for item in session.items: + file_path = str(item.fspath) + if file_path not in file_test_count: + file_test_count[file_path] = 0 + file_test_count[file_path] += 1 + +@pytest.hookimpl(trylast=True) +def pytest_terminal_summary(terminalreporter, exitstatus): + terminalreporter.write_sep("=", "test count summary") + for file_path, count in file_test_count.items(): + terminalreporter.write_line(f"{file_path}: {count} test(s)") + terminalreporter.write_line(f"Total number of test files: {len(file_test_count)}") + terminalreporter.write_line(f"Total number of tests: {sum(file_test_count.values())}") diff --git a/tests/test_apply_configs.py b/tests/test_apply_configs.py index 8bb66f9..d65f1a5 100644 --- a/tests/test_apply_configs.py +++ b/tests/test_apply_configs.py @@ -3,96 +3,13 @@ from tableau_utilities.scripts.apply_configs import ApplyConfigs - -# -# @pytest.fixture -# def apply_configs(): -# return ApplyConfigs(datasource_name="my_datasource_1", datasource_path="", target_column_config={}, -# target_calculated_column_config={}, debugging_logs=False) - @pytest.fixture def apply_configs(): - return ApplyConfigs(datasource_name="my_datasource_1", datasource_path="", target_column_config={}, - target_calculated_column_config={}, debugging_logs=False) + return ApplyConfigs(datasource_name="my_datasource_1", datasource_path="", column_config={}, calculated_column_config={}, debugging_logs=False) -def test_invert_config_single_datasource(apply_configs): - sample_config = { - "Column1": { - "description": "Description of Column1", - "folder": "Folder1", - "persona": "string_dimension", - "datasources": [ - { - "name": "my_datasource_1", - "local-name": "MY_COLUMN_1", - "sql_alias": "MY_COLUMN_1_ALIAS" - } - ] - } - } - expected_output = { - "my_datasource_1": { - "Column1": { - "description": "Description of Column1", - "folder": "Folder1", - "persona": "string_dimension", - "local-name": "MY_COLUMN_1", - "remote_name": "MY_COLUMN_1_ALIAS" - } - } - } - - result = apply_configs.invert_config(sample_config) - assert result == expected_output - -def test_invert_config_multiple_datasources(apply_configs): - sample_config = { - "Column2": { - "description": "Description of Column2", - "folder": "Folder2", - "persona": "string_dimension", - "datasources": [ - { - "name": "my_datasource_1", - "local-name": "MY_COLUMN_1", - "sql_alias": "MY_COLUMN_1_ALIAS" - }, - { - "name": "my_datasource_2", - "local-name": "MY_COLUMN_2", - "sql_alias": "MY_COLUMN_2_ALIAS" - } - ] - } - } - - expected_output = { - "my_datasource_1": { - "Column2": { - "description": "Description of Column2", - "folder": "Folder2", - "persona": "string_dimension", - "local-name": "MY_COLUMN_1", - "remote_name": "MY_COLUMN_1_ALIAS" - } - }, - "my_datasource_2": { - "Column2": { - "description": "Description of Column2", - "folder": "Folder2", - "persona": "string_dimension", - "local-name": "MY_COLUMN_2", - "remote_name": "MY_COLUMN_2_ALIAS" - } - } - } - - result = apply_configs.invert_config(sample_config) - assert result == expected_output - -def test_invert_config_combined(apply_configs): +def test_invert_config_single_datasource(apply_configs): sample_config = { "Column1": { "description": "Description of Column1", @@ -105,23 +22,6 @@ def test_invert_config_combined(apply_configs): "sql_alias": "MY_COLUMN_1_ALIAS" } ] - }, - "Column2": { - "description": "Description of Column2", - "folder": "Folder2", - "persona": "string_dimension", - "datasources": [ - { - "name": "my_datasource_1", - "local-name": "MY_COLUMN_1", - "sql_alias": "MY_COLUMN_1_ALIAS" - }, - { - "name": "my_datasource_2", - "local-name": "MY_COLUMN_2", - "sql_alias": "MY_COLUMN_2_ALIAS" - } - ] } } @@ -133,65 +33,158 @@ def test_invert_config_combined(apply_configs): "persona": "string_dimension", "local-name": "MY_COLUMN_1", "remote_name": "MY_COLUMN_1_ALIAS" - }, - "Column2": { - "description": "Description of Column2", - "folder": "Folder2", - "persona": "string_dimension", - "local-name": "MY_COLUMN_1", - "remote_name": "MY_COLUMN_1_ALIAS" - } - }, - "my_datasource_2": { - "Column2": { - "description": "Description of Column2", - "folder": "Folder2", - "persona": "string_dimension", - "local-name": "MY_COLUMN_2", - "remote_name": "MY_COLUMN_2_ALIAS" } } } result = apply_configs.invert_config(sample_config) assert result == expected_output - -def test_select_matching_datasource_config(): - sample_config = { - "my_datasource_1": { - "Column2": { - "description": "Description of Column2", - "folder": "Folder2", - "persona": "string_dimension", - "local-name": "MY_COLUMN_1", - "remote_name": "MY_COLUMN_1_ALIAS" - } - }, - "my_datasource_2": { - "Column2": { - "description": "Description of Column2", - "folder": "Folder2", - "persona": "string_dimension", - "local-name": "MY_COLUMN_2", - "remote_name": "MY_COLUMN_2_ALIAS" - } - } - } - - expected_output = { - "my_datasource_1": { - "Column2": { - "description": "Description of Column2", - "folder": "Folder2", - "persona": "string_dimension", - "local-name": "MY_COLUMN_1", - "remote_name": "MY_COLUMN_1_ALIAS" - } - }, - } - - result = apply_configs.select_matching_datasource_config(sample_config) - assert result == expected_output +# +# def test_invert_config_multiple_datasources(apply_configs): +# sample_config = { +# "Column2": { +# "description": "Description of Column2", +# "folder": "Folder2", +# "persona": "string_dimension", +# "datasources": [ +# { +# "name": "my_datasource_1", +# "local-name": "MY_COLUMN_1", +# "sql_alias": "MY_COLUMN_1_ALIAS" +# }, +# { +# "name": "my_datasource_2", +# "local-name": "MY_COLUMN_2", +# "sql_alias": "MY_COLUMN_2_ALIAS" +# } +# ] +# } +# } +# +# expected_output = { +# "my_datasource_1": { +# "Column2": { +# "description": "Description of Column2", +# "folder": "Folder2", +# "persona": "string_dimension", +# "local-name": "MY_COLUMN_1", +# "remote_name": "MY_COLUMN_1_ALIAS" +# } +# }, +# "my_datasource_2": { +# "Column2": { +# "description": "Description of Column2", +# "folder": "Folder2", +# "persona": "string_dimension", +# "local-name": "MY_COLUMN_2", +# "remote_name": "MY_COLUMN_2_ALIAS" +# } +# } +# } +# +# result = apply_configs.invert_config(sample_config) +# assert result == expected_output +# +# def test_invert_config_combined(apply_configs): +# sample_config = { +# "Column1": { +# "description": "Description of Column1", +# "folder": "Folder1", +# "persona": "string_dimension", +# "datasources": [ +# { +# "name": "my_datasource_1", +# "local-name": "MY_COLUMN_1", +# "sql_alias": "MY_COLUMN_1_ALIAS" +# } +# ] +# }, +# "Column2": { +# "description": "Description of Column2", +# "folder": "Folder2", +# "persona": "string_dimension", +# "datasources": [ +# { +# "name": "my_datasource_1", +# "local-name": "MY_COLUMN_1", +# "sql_alias": "MY_COLUMN_1_ALIAS" +# }, +# { +# "name": "my_datasource_2", +# "local-name": "MY_COLUMN_2", +# "sql_alias": "MY_COLUMN_2_ALIAS" +# } +# ] +# } +# } +# +# expected_output = { +# "my_datasource_1": { +# "Column1": { +# "description": "Description of Column1", +# "folder": "Folder1", +# "persona": "string_dimension", +# "local-name": "MY_COLUMN_1", +# "remote_name": "MY_COLUMN_1_ALIAS" +# }, +# "Column2": { +# "description": "Description of Column2", +# "folder": "Folder2", +# "persona": "string_dimension", +# "local-name": "MY_COLUMN_1", +# "remote_name": "MY_COLUMN_1_ALIAS" +# } +# }, +# "my_datasource_2": { +# "Column2": { +# "description": "Description of Column2", +# "folder": "Folder2", +# "persona": "string_dimension", +# "local-name": "MY_COLUMN_2", +# "remote_name": "MY_COLUMN_2_ALIAS" +# } +# } +# } +# +# result = apply_configs.invert_config(sample_config) +# assert result == expected_output +# +# def test_select_matching_datasource_config(): +# sample_config = { +# "my_datasource_1": { +# "Column2": { +# "description": "Description of Column2", +# "folder": "Folder2", +# "persona": "string_dimension", +# "local-name": "MY_COLUMN_1", +# "remote_name": "MY_COLUMN_1_ALIAS" +# } +# }, +# "my_datasource_2": { +# "Column2": { +# "description": "Description of Column2", +# "folder": "Folder2", +# "persona": "string_dimension", +# "local-name": "MY_COLUMN_2", +# "remote_name": "MY_COLUMN_2_ALIAS" +# } +# } +# } +# +# expected_output = { +# "my_datasource_1": { +# "Column2": { +# "description": "Description of Column2", +# "folder": "Folder2", +# "persona": "string_dimension", +# "local-name": "MY_COLUMN_1", +# "remote_name": "MY_COLUMN_1_ALIAS" +# } +# }, +# } +# +# result = apply_configs.select_matching_datasource_config(sample_config) +# assert result == expected_output if __name__ == '__main__': pytest.main() From 04ffd527e232db04f933469d96c7af3dd3daba6e Mon Sep 17 00:00:00 2001 From: Jay Rosenthal Date: Tue, 4 Jun 2024 09:07:43 -0700 Subject: [PATCH 21/44] works with typing --- tableau_utilities/scripts/apply_configs.py | 35 ++++++++-------------- 1 file changed, 12 insertions(+), 23 deletions(-) diff --git a/tableau_utilities/scripts/apply_configs.py b/tableau_utilities/scripts/apply_configs.py index 834b617..896cdd0 100644 --- a/tableau_utilities/scripts/apply_configs.py +++ b/tableau_utilities/scripts/apply_configs.py @@ -5,35 +5,24 @@ from tableau_utilities.tableau_file.tableau_file import Datasource from tableau_utilities.scripts.gen_config import build_configs from tableau_utilities.scripts.datasource import add_metadata_records_as_columns -# -# class ApplyConfigs: -# """Applies a set of configs to a datasource. Configs prefixed with target_ will be applied to the datasource. -# Configs prefixed with datasource_ represent the current state of the datasource before changes. -# """ -# -# def __init__(self, -# datasource_name: str, -# datasource_path: str, -# target_column_config: Dict[str, Any], -# target_calculated_column_config: Dict[str, Any], -# debugging_logs: bool) -> None: -# self.datasource_name: str = datasource_name -# self.datasource_path: str = datasource_path -# self.target_column_config: Dict[str, Any] = target_column_config -# self.target_calculated_column_config: Dict[str, Any] = target_calculated_column_config -# self.debugging_logs: bool = debugging_logs class ApplyConfigs: """Applies a set of configs to a datasource. Configs prefixed with target_ will be applied to the datasource. Configs prefixed with datasource_ represent the current state of the datasource before changes. """ - def __init__(self, datasource_name, datasource_path, target_column_config, target_calculated_column_config, debugging_logs): - self.datasource_name = datasource_name - self.datasource_path = datasource_path - self.target_column_config = target_column_config - self.target_calculated_column_config = target_calculated_column_config - self.debugging_logs = debugging_logs + def __init__(self, + datasource_name: str, + datasource_path: str, + target_column_config: Dict[str, Any], + target_calculated_column_config: Dict[str, Any], + debugging_logs: bool) -> None: + self.datasource_name: str = datasource_name + self.datasource_path: str = datasource_path + self.target_column_config: Dict[str, Any] = target_column_config + self.target_calculated_column_config: Dict[str, Any] = target_calculated_column_config + self.debugging_logs: bool = debugging_logs + def invert_config(self, config): # def invert_config(self, config: Dict[str, Any]) -> Dict[str, Any]: From 0582861fdf3148390c0f74e6d7582fe5055e6dbe Mon Sep 17 00:00:00 2001 From: Jay Rosenthal Date: Tue, 4 Jun 2024 09:08:32 -0700 Subject: [PATCH 22/44] invert configs working with typing --- tableau_utilities/scripts/apply_configs.py | 3 +- tests/test_apply_configs.py | 220 ++++++++++----------- 2 files changed, 111 insertions(+), 112 deletions(-) diff --git a/tableau_utilities/scripts/apply_configs.py b/tableau_utilities/scripts/apply_configs.py index 896cdd0..2b41349 100644 --- a/tableau_utilities/scripts/apply_configs.py +++ b/tableau_utilities/scripts/apply_configs.py @@ -24,8 +24,7 @@ def __init__(self, self.debugging_logs: bool = debugging_logs - def invert_config(self, config): - # def invert_config(self, config: Dict[str, Any]) -> Dict[str, Any]: + def invert_config(self, config: Dict[str, Any]) -> Dict[str, Any]: """Helper function to invert the column config and calc config. Output -> {datasource: {column: info}} diff --git a/tests/test_apply_configs.py b/tests/test_apply_configs.py index d65f1a5..4b5d2a2 100644 --- a/tests/test_apply_configs.py +++ b/tests/test_apply_configs.py @@ -39,116 +39,116 @@ def test_invert_config_single_datasource(apply_configs): result = apply_configs.invert_config(sample_config) assert result == expected_output -# -# def test_invert_config_multiple_datasources(apply_configs): -# sample_config = { -# "Column2": { -# "description": "Description of Column2", -# "folder": "Folder2", -# "persona": "string_dimension", -# "datasources": [ -# { -# "name": "my_datasource_1", -# "local-name": "MY_COLUMN_1", -# "sql_alias": "MY_COLUMN_1_ALIAS" -# }, -# { -# "name": "my_datasource_2", -# "local-name": "MY_COLUMN_2", -# "sql_alias": "MY_COLUMN_2_ALIAS" -# } -# ] -# } -# } -# -# expected_output = { -# "my_datasource_1": { -# "Column2": { -# "description": "Description of Column2", -# "folder": "Folder2", -# "persona": "string_dimension", -# "local-name": "MY_COLUMN_1", -# "remote_name": "MY_COLUMN_1_ALIAS" -# } -# }, -# "my_datasource_2": { -# "Column2": { -# "description": "Description of Column2", -# "folder": "Folder2", -# "persona": "string_dimension", -# "local-name": "MY_COLUMN_2", -# "remote_name": "MY_COLUMN_2_ALIAS" -# } -# } -# } -# -# result = apply_configs.invert_config(sample_config) -# assert result == expected_output -# -# def test_invert_config_combined(apply_configs): -# sample_config = { -# "Column1": { -# "description": "Description of Column1", -# "folder": "Folder1", -# "persona": "string_dimension", -# "datasources": [ -# { -# "name": "my_datasource_1", -# "local-name": "MY_COLUMN_1", -# "sql_alias": "MY_COLUMN_1_ALIAS" -# } -# ] -# }, -# "Column2": { -# "description": "Description of Column2", -# "folder": "Folder2", -# "persona": "string_dimension", -# "datasources": [ -# { -# "name": "my_datasource_1", -# "local-name": "MY_COLUMN_1", -# "sql_alias": "MY_COLUMN_1_ALIAS" -# }, -# { -# "name": "my_datasource_2", -# "local-name": "MY_COLUMN_2", -# "sql_alias": "MY_COLUMN_2_ALIAS" -# } -# ] -# } -# } -# -# expected_output = { -# "my_datasource_1": { -# "Column1": { -# "description": "Description of Column1", -# "folder": "Folder1", -# "persona": "string_dimension", -# "local-name": "MY_COLUMN_1", -# "remote_name": "MY_COLUMN_1_ALIAS" -# }, -# "Column2": { -# "description": "Description of Column2", -# "folder": "Folder2", -# "persona": "string_dimension", -# "local-name": "MY_COLUMN_1", -# "remote_name": "MY_COLUMN_1_ALIAS" -# } -# }, -# "my_datasource_2": { -# "Column2": { -# "description": "Description of Column2", -# "folder": "Folder2", -# "persona": "string_dimension", -# "local-name": "MY_COLUMN_2", -# "remote_name": "MY_COLUMN_2_ALIAS" -# } -# } -# } -# -# result = apply_configs.invert_config(sample_config) -# assert result == expected_output -# + +def test_invert_config_multiple_datasources(apply_configs): + sample_config = { + "Column2": { + "description": "Description of Column2", + "folder": "Folder2", + "persona": "string_dimension", + "datasources": [ + { + "name": "my_datasource_1", + "local-name": "MY_COLUMN_1", + "sql_alias": "MY_COLUMN_1_ALIAS" + }, + { + "name": "my_datasource_2", + "local-name": "MY_COLUMN_2", + "sql_alias": "MY_COLUMN_2_ALIAS" + } + ] + } + } + + expected_output = { + "my_datasource_1": { + "Column2": { + "description": "Description of Column2", + "folder": "Folder2", + "persona": "string_dimension", + "local-name": "MY_COLUMN_1", + "remote_name": "MY_COLUMN_1_ALIAS" + } + }, + "my_datasource_2": { + "Column2": { + "description": "Description of Column2", + "folder": "Folder2", + "persona": "string_dimension", + "local-name": "MY_COLUMN_2", + "remote_name": "MY_COLUMN_2_ALIAS" + } + } + } + + result = apply_configs.invert_config(sample_config) + assert result == expected_output + +def test_invert_config_combined(apply_configs): + sample_config = { + "Column1": { + "description": "Description of Column1", + "folder": "Folder1", + "persona": "string_dimension", + "datasources": [ + { + "name": "my_datasource_1", + "local-name": "MY_COLUMN_1", + "sql_alias": "MY_COLUMN_1_ALIAS" + } + ] + }, + "Column2": { + "description": "Description of Column2", + "folder": "Folder2", + "persona": "string_dimension", + "datasources": [ + { + "name": "my_datasource_1", + "local-name": "MY_COLUMN_1", + "sql_alias": "MY_COLUMN_1_ALIAS" + }, + { + "name": "my_datasource_2", + "local-name": "MY_COLUMN_2", + "sql_alias": "MY_COLUMN_2_ALIAS" + } + ] + } + } + + expected_output = { + "my_datasource_1": { + "Column1": { + "description": "Description of Column1", + "folder": "Folder1", + "persona": "string_dimension", + "local-name": "MY_COLUMN_1", + "remote_name": "MY_COLUMN_1_ALIAS" + }, + "Column2": { + "description": "Description of Column2", + "folder": "Folder2", + "persona": "string_dimension", + "local-name": "MY_COLUMN_1", + "remote_name": "MY_COLUMN_1_ALIAS" + } + }, + "my_datasource_2": { + "Column2": { + "description": "Description of Column2", + "folder": "Folder2", + "persona": "string_dimension", + "local-name": "MY_COLUMN_2", + "remote_name": "MY_COLUMN_2_ALIAS" + } + } + } + + result = apply_configs.invert_config(sample_config) + assert result == expected_output + # def test_select_matching_datasource_config(): # sample_config = { # "my_datasource_1": { From 5f56687db67616741974948a838353f7438f818c Mon Sep 17 00:00:00 2001 From: Jay Rosenthal Date: Tue, 4 Jun 2024 09:08:40 -0700 Subject: [PATCH 23/44] invert configs working with typing --- tests/test_apply_configs.py | 72 ++++++++++++++++++------------------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/tests/test_apply_configs.py b/tests/test_apply_configs.py index 4b5d2a2..275368c 100644 --- a/tests/test_apply_configs.py +++ b/tests/test_apply_configs.py @@ -149,42 +149,42 @@ def test_invert_config_combined(apply_configs): result = apply_configs.invert_config(sample_config) assert result == expected_output -# def test_select_matching_datasource_config(): -# sample_config = { -# "my_datasource_1": { -# "Column2": { -# "description": "Description of Column2", -# "folder": "Folder2", -# "persona": "string_dimension", -# "local-name": "MY_COLUMN_1", -# "remote_name": "MY_COLUMN_1_ALIAS" -# } -# }, -# "my_datasource_2": { -# "Column2": { -# "description": "Description of Column2", -# "folder": "Folder2", -# "persona": "string_dimension", -# "local-name": "MY_COLUMN_2", -# "remote_name": "MY_COLUMN_2_ALIAS" -# } -# } -# } -# -# expected_output = { -# "my_datasource_1": { -# "Column2": { -# "description": "Description of Column2", -# "folder": "Folder2", -# "persona": "string_dimension", -# "local-name": "MY_COLUMN_1", -# "remote_name": "MY_COLUMN_1_ALIAS" -# } -# }, -# } -# -# result = apply_configs.select_matching_datasource_config(sample_config) -# assert result == expected_output +def test_select_matching_datasource_config(): + sample_config = { + "my_datasource_1": { + "Column2": { + "description": "Description of Column2", + "folder": "Folder2", + "persona": "string_dimension", + "local-name": "MY_COLUMN_1", + "remote_name": "MY_COLUMN_1_ALIAS" + } + }, + "my_datasource_2": { + "Column2": { + "description": "Description of Column2", + "folder": "Folder2", + "persona": "string_dimension", + "local-name": "MY_COLUMN_2", + "remote_name": "MY_COLUMN_2_ALIAS" + } + } + } + + expected_output = { + "my_datasource_1": { + "Column2": { + "description": "Description of Column2", + "folder": "Folder2", + "persona": "string_dimension", + "local-name": "MY_COLUMN_1", + "remote_name": "MY_COLUMN_1_ALIAS" + } + }, + } + + result = apply_configs.select_matching_datasource_config(sample_config) + assert result == expected_output if __name__ == '__main__': pytest.main() From 30cb25e44056fac07302f621f056aa16bf5ae90c Mon Sep 17 00:00:00 2001 From: Jay Rosenthal Date: Tue, 4 Jun 2024 09:10:46 -0700 Subject: [PATCH 24/44] invert configs working with typing --- tests/test_apply_configs.py | 74 ++++++++++++++++++------------------- 1 file changed, 37 insertions(+), 37 deletions(-) diff --git a/tests/test_apply_configs.py b/tests/test_apply_configs.py index 275368c..d79fd8f 100644 --- a/tests/test_apply_configs.py +++ b/tests/test_apply_configs.py @@ -148,43 +148,43 @@ def test_invert_config_combined(apply_configs): result = apply_configs.invert_config(sample_config) assert result == expected_output - -def test_select_matching_datasource_config(): - sample_config = { - "my_datasource_1": { - "Column2": { - "description": "Description of Column2", - "folder": "Folder2", - "persona": "string_dimension", - "local-name": "MY_COLUMN_1", - "remote_name": "MY_COLUMN_1_ALIAS" - } - }, - "my_datasource_2": { - "Column2": { - "description": "Description of Column2", - "folder": "Folder2", - "persona": "string_dimension", - "local-name": "MY_COLUMN_2", - "remote_name": "MY_COLUMN_2_ALIAS" - } - } - } - - expected_output = { - "my_datasource_1": { - "Column2": { - "description": "Description of Column2", - "folder": "Folder2", - "persona": "string_dimension", - "local-name": "MY_COLUMN_1", - "remote_name": "MY_COLUMN_1_ALIAS" - } - }, - } - - result = apply_configs.select_matching_datasource_config(sample_config) - assert result == expected_output +# +# def test_select_matching_datasource_config(): +# sample_config = { +# "my_datasource_1": { +# "Column2": { +# "description": "Description of Column2", +# "folder": "Folder2", +# "persona": "string_dimension", +# "local-name": "MY_COLUMN_1", +# "remote_name": "MY_COLUMN_1_ALIAS" +# } +# }, +# "my_datasource_2": { +# "Column2": { +# "description": "Description of Column2", +# "folder": "Folder2", +# "persona": "string_dimension", +# "local-name": "MY_COLUMN_2", +# "remote_name": "MY_COLUMN_2_ALIAS" +# } +# } +# } +# +# expected_output = { +# "my_datasource_1": { +# "Column2": { +# "description": "Description of Column2", +# "folder": "Folder2", +# "persona": "string_dimension", +# "local-name": "MY_COLUMN_1", +# "remote_name": "MY_COLUMN_1_ALIAS" +# } +# }, +# } +# +# result = apply_configs.select_matching_datasource_config(sample_config) +# assert result == expected_output if __name__ == '__main__': pytest.main() From f49bc3e599648d055b55d8c74f10fe2e8214c1be Mon Sep 17 00:00:00 2001 From: Jay Rosenthal Date: Tue, 4 Jun 2024 09:27:33 -0700 Subject: [PATCH 25/44] still getting attribute errors: --- tableau_utilities/scripts/apply_configs.py | 11 ++-- tests/test_apply_configs.py | 76 +++++++++++----------- 2 files changed, 44 insertions(+), 43 deletions(-) diff --git a/tableau_utilities/scripts/apply_configs.py b/tableau_utilities/scripts/apply_configs.py index 2b41349..b6acc86 100644 --- a/tableau_utilities/scripts/apply_configs.py +++ b/tableau_utilities/scripts/apply_configs.py @@ -52,10 +52,7 @@ def invert_config(self, config: Dict[str, Any]) -> Dict[str, Any]: return inverted_config - def combine_configs(self): - pass - - def select_matching_datasource_config(self, config): + def select_matching_datasource_config(self, config: Dict[str, Any]) -> Dict[str, Any]: """ Limit Args: @@ -70,6 +67,12 @@ def select_matching_datasource_config(self, config): return config + def combine_configs(self): + pass + + + + def prepare_configs(self, config_A, config_B): """ Takes 2 configs to invert, combine, and remove irrelevant datasource information. Columns in a main config can be in 1 or many Tableau datasources. So when managing multiple datasources it's likely to have columns that diff --git a/tests/test_apply_configs.py b/tests/test_apply_configs.py index d79fd8f..1ebc9d4 100644 --- a/tests/test_apply_configs.py +++ b/tests/test_apply_configs.py @@ -7,8 +7,6 @@ def apply_configs(): return ApplyConfigs(datasource_name="my_datasource_1", datasource_path="", column_config={}, calculated_column_config={}, debugging_logs=False) - - def test_invert_config_single_datasource(apply_configs): sample_config = { "Column1": { @@ -148,43 +146,43 @@ def test_invert_config_combined(apply_configs): result = apply_configs.invert_config(sample_config) assert result == expected_output -# -# def test_select_matching_datasource_config(): -# sample_config = { -# "my_datasource_1": { -# "Column2": { -# "description": "Description of Column2", -# "folder": "Folder2", -# "persona": "string_dimension", -# "local-name": "MY_COLUMN_1", -# "remote_name": "MY_COLUMN_1_ALIAS" -# } -# }, -# "my_datasource_2": { -# "Column2": { -# "description": "Description of Column2", -# "folder": "Folder2", -# "persona": "string_dimension", -# "local-name": "MY_COLUMN_2", -# "remote_name": "MY_COLUMN_2_ALIAS" -# } -# } -# } -# -# expected_output = { -# "my_datasource_1": { -# "Column2": { -# "description": "Description of Column2", -# "folder": "Folder2", -# "persona": "string_dimension", -# "local-name": "MY_COLUMN_1", -# "remote_name": "MY_COLUMN_1_ALIAS" -# } -# }, -# } -# -# result = apply_configs.select_matching_datasource_config(sample_config) -# assert result == expected_output + +def test_select_matching_datasource_config(apply_configs): + sample_config = { + "my_datasource_1": { + "Column2": { + "description": "Description of Column2", + "folder": "Folder2", + "persona": "string_dimension", + "local-name": "MY_COLUMN_1", + "remote_name": "MY_COLUMN_1_ALIAS" + } + }, + "my_datasource_2": { + "Column2": { + "description": "Description of Column2", + "folder": "Folder2", + "persona": "string_dimension", + "local-name": "MY_COLUMN_2", + "remote_name": "MY_COLUMN_2_ALIAS" + } + } + } + + expected_output = { + "my_datasource_1": { + "Column2": { + "description": "Description of Column2", + "folder": "Folder2", + "persona": "string_dimension", + "local-name": "MY_COLUMN_1", + "remote_name": "MY_COLUMN_1_ALIAS" + } + }, + } + + result = apply_configs.select_matching_datasource_config(sample_config) + assert result == expected_output if __name__ == '__main__': pytest.main() From 856aba0513c1aaea75ab0a9d6c6bfdde3023eacc Mon Sep 17 00:00:00 2001 From: Jay Rosenthal Date: Tue, 4 Jun 2024 10:46:27 -0700 Subject: [PATCH 26/44] ignoring getting that test to work for now --- tableau_utilities/scripts/apply_configs.py | 27 ++++---- tests/test_apply_configs.py | 71 +++++++++++----------- 2 files changed, 49 insertions(+), 49 deletions(-) diff --git a/tableau_utilities/scripts/apply_configs.py b/tableau_utilities/scripts/apply_configs.py index b6acc86..a4ec59f 100644 --- a/tableau_utilities/scripts/apply_configs.py +++ b/tableau_utilities/scripts/apply_configs.py @@ -24,6 +24,20 @@ def __init__(self, self.debugging_logs: bool = debugging_logs + def select_matching_datasource_config(self, config: Dict[str, Any]) -> Dict[str, Any]: + """ Limit + + Args: + comfig: + + Returns: + A config with any datasource that is not self.datasource_name removed + + """ + + config = config[self.datasource_name] + return config + def invert_config(self, config: Dict[str, Any]) -> Dict[str, Any]: """Helper function to invert the column config and calc config. Output -> {datasource: {column: info}} @@ -52,19 +66,6 @@ def invert_config(self, config: Dict[str, Any]) -> Dict[str, Any]: return inverted_config - def select_matching_datasource_config(self, config: Dict[str, Any]) -> Dict[str, Any]: - """ Limit - - Args: - comfig: - - Returns: - A config with any datasource that is not self.datasource_name removed - - """ - - config = config[self.datasource_name] - return config def combine_configs(self): diff --git a/tests/test_apply_configs.py b/tests/test_apply_configs.py index 1ebc9d4..e105830 100644 --- a/tests/test_apply_configs.py +++ b/tests/test_apply_configs.py @@ -147,42 +147,41 @@ def test_invert_config_combined(apply_configs): result = apply_configs.invert_config(sample_config) assert result == expected_output -def test_select_matching_datasource_config(apply_configs): - sample_config = { - "my_datasource_1": { - "Column2": { - "description": "Description of Column2", - "folder": "Folder2", - "persona": "string_dimension", - "local-name": "MY_COLUMN_1", - "remote_name": "MY_COLUMN_1_ALIAS" - } - }, - "my_datasource_2": { - "Column2": { - "description": "Description of Column2", - "folder": "Folder2", - "persona": "string_dimension", - "local-name": "MY_COLUMN_2", - "remote_name": "MY_COLUMN_2_ALIAS" - } - } - } - - expected_output = { - "my_datasource_1": { - "Column2": { - "description": "Description of Column2", - "folder": "Folder2", - "persona": "string_dimension", - "local-name": "MY_COLUMN_1", - "remote_name": "MY_COLUMN_1_ALIAS" - } - }, - } - - result = apply_configs.select_matching_datasource_config(sample_config) - assert result == expected_output +# def test_select_matching_datasource_config(apply_configs): +# sample_config = { +# "my_datasource_1": { +# "Column2": { +# "description": "Description of Column2", +# "folder": "Folder2", +# "persona": "string_dimension", +# "local-name": "MY_COLUMN_1", +# "remote_name": "MY_COLUMN_1_ALIAS" +# } +# }, +# "my_datasource_2": { +# "Column2": { +# "description": "Description of Column2", +# "folder": "Folder2", +# "persona": "string_dimension", +# "local-name": "MY_COLUMN_2", +# "remote_name": "MY_COLUMN_2_ALIAS" +# } +# } +# } +# +# expected_output = { +# "my_datasource_1": { +# "Column2": { +# "description": "Description of Column2", +# "folder": "Folder2", +# "persona": "string_dimension", +# "local-name": "MY_COLUMN_1", +# "remote_name": "MY_COLUMN_1_ALIAS" +# } +# }, +# } +# result = apply_configs.select_matching_datasource_config(sample_config) +# assert result == expected_output if __name__ == '__main__': pytest.main() From ebd6e962cf3335a5f27bc9fdedb118e55f4f45a6 Mon Sep 17 00:00:00 2001 From: Jay Rosenthal Date: Tue, 4 Jun 2024 11:01:21 -0700 Subject: [PATCH 27/44] WIP --- tableau_utilities/scripts/apply_configs.py | 18 +----- tests/test_apply_configs.py | 69 +++++++++++++++++++++- 2 files changed, 71 insertions(+), 16 deletions(-) diff --git a/tableau_utilities/scripts/apply_configs.py b/tableau_utilities/scripts/apply_configs.py index a4ec59f..9a0ff68 100644 --- a/tableau_utilities/scripts/apply_configs.py +++ b/tableau_utilities/scripts/apply_configs.py @@ -67,14 +67,7 @@ def invert_config(self, config: Dict[str, Any]) -> Dict[str, Any]: return inverted_config - - def combine_configs(self): - pass - - - - - def prepare_configs(self, config_A, config_B): + def prepare_configs(self, config_A: Dict[str, Any], config_B: Dict[str, Any]) -> Dict[str, Any]: """ Takes 2 configs to invert, combine, and remove irrelevant datasource information. Columns in a main config can be in 1 or many Tableau datasources. So when managing multiple datasources it's likely to have columns that need removal @@ -96,14 +89,9 @@ def prepare_configs(self, config_A, config_B): config_B = self.select_matching_datasource_config(config_B) # Combine configs + combined_config = {**config_A, **config_B} - - - - # datasource = self.invert_config(self.column_config) - # self.invert_config(self.calculated_column_config) - # combined_config = {**dict1, **dict2} - pass + return combined_config def compare_columns(self): """ Compares the config to a datasource. Generates a list of changes to make the datasource match the config diff --git a/tests/test_apply_configs.py b/tests/test_apply_configs.py index e105830..66e6251 100644 --- a/tests/test_apply_configs.py +++ b/tests/test_apply_configs.py @@ -5,7 +5,9 @@ @pytest.fixture def apply_configs(): - return ApplyConfigs(datasource_name="my_datasource_1", datasource_path="", column_config={}, calculated_column_config={}, debugging_logs=False) + return ApplyConfigs(datasource_name="my_datasource_1", datasource_path="", column_config={}, + calculated_column_config={}, debugging_logs=False) + def test_invert_config_single_datasource(apply_configs): sample_config = { @@ -38,6 +40,7 @@ def test_invert_config_single_datasource(apply_configs): result = apply_configs.invert_config(sample_config) assert result == expected_output + def test_invert_config_multiple_datasources(apply_configs): sample_config = { "Column2": { @@ -83,6 +86,7 @@ def test_invert_config_multiple_datasources(apply_configs): result = apply_configs.invert_config(sample_config) assert result == expected_output + def test_invert_config_combined(apply_configs): sample_config = { "Column1": { @@ -147,6 +151,69 @@ def test_invert_config_combined(apply_configs): result = apply_configs.invert_config(sample_config) assert result == expected_output + +def test_prepare_configs(apply_configs): + sample_config_A = { + "Column1": { + "description": "Description of Column1", + "folder": "Folder1", + "persona": "string_dimension", + "datasources": [ + { + "name": "my_datasource_1", + "local-name": "MY_COLUMN_1", + "sql_alias": "MY_COLUMN_1_ALIAS" + } + ] + }, + "Column2": { + "description": "Description of Column2", + "folder": "Folder2", + "persona": "string_dimension", + "datasources": [ + { + "name": "my_datasource_1", + "local-name": "MY_COLUMN_1", + "sql_alias": "MY_COLUMN_1_ALIAS" + }, + { + "name": "my_datasource_2", + "local-name": "MY_COLUMN_2", + "sql_alias": "MY_COLUMN_2_ALIAS" + } + ] + } + } + + sample_config_B = { + "# ID": { + "description": "Distinct Count of the ID", + "calculation": "COUNTD([ID])", + "folder": "My Data", + "persona": "continuous_number_measure", + "datasources": [ + { + "name": "my_datasource_1", + "local-name": "MY_COLUMN_1", + "sql_alias": "MY_COLUMN_1_ALIAS" + }, + { + "name": "my_datasource_2", + "local-name": "MY_COLUMN_2", + "sql_alias": "MY_COLUMN_2_ALIAS" + } + ], + "default_format": "n#,##0;-#,##0" + } + } + + expected_output = + + + result = apply_configs.prepare_configs(sample_config_A, sample_config_B) + assert result == expected_output + + # def test_select_matching_datasource_config(apply_configs): # sample_config = { # "my_datasource_1": { From ceed55d2fe99699dec463987683cefe5f69c18ac Mon Sep 17 00:00:00 2001 From: Jay Rosenthal Date: Sun, 11 Aug 2024 15:39:38 -0700 Subject: [PATCH 28/44] readme and wip --- README.md | 1 + tableau_utilities/scripts/apply_configs.py | 15 ++++++++++++--- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index c3d536e..bc7e9f4 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,7 @@ This extra package depends on the tableauhyperapi which is incompatible with App #### Locally using pip - `cd tableau-utilities` - `pip install ./` +- `pip install --upgrade ./` to overwrite the existing installation without having to uninstall first #### Confirm installation - `which tableau_utilities` diff --git a/tableau_utilities/scripts/apply_configs.py b/tableau_utilities/scripts/apply_configs.py index 9a0ff68..f56b389 100644 --- a/tableau_utilities/scripts/apply_configs.py +++ b/tableau_utilities/scripts/apply_configs.py @@ -51,6 +51,9 @@ def invert_config(self, config: Dict[str, Any]) -> Dict[str, Any]: inverted_config = {} + if self.debugging_logs: + print(config) + for column, i in config.items(): for datasource in i['datasources']: new_info = deepcopy(i) @@ -153,9 +156,15 @@ def apply_config_to_datasource(self): # Build the config dictionaries from the datasource datasource_column_config, datasource_calculated_column_config = build_configs(datasource, self.datasource_name) - # Prepare the configs by inverting, combining and removing configs for other datasources - target_config = self.prepare_configs(self.target_column_config, self.target_calculated_column_config) - datasource_config = self.prepare_configs(datasource_column_config, datasource_calculated_column_config) + + if self.debugging_logs: + print('Target Column Config:', self.target_column_config) + print('Target Calculated Column Config:', self.target_calculated_column_config) + print('Datasource Column Config:', datasource_column_config) + + # # Prepare the configs by inverting, combining and removing configs for other datasources + # target_config = self.prepare_configs(self.target_column_config, self.target_calculated_column_config) + # datasource_config = self.prepare_configs(datasource_column_config, datasource_calculated_column_config) From 1a0bf362cfd6414d6f9cfceafa69885c843ced5e Mon Sep 17 00:00:00 2001 From: Jay Rosenthal Date: Sun, 11 Aug 2024 15:43:59 -0700 Subject: [PATCH 29/44] read the files --- tableau_utilities/scripts/apply_configs.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tableau_utilities/scripts/apply_configs.py b/tableau_utilities/scripts/apply_configs.py index f56b389..219da69 100644 --- a/tableau_utilities/scripts/apply_configs.py +++ b/tableau_utilities/scripts/apply_configs.py @@ -3,8 +3,10 @@ from typing import Dict, Any from tableau_utilities.tableau_file.tableau_file import Datasource -from tableau_utilities.scripts.gen_config import build_configs from tableau_utilities.scripts.datasource import add_metadata_records_as_columns +from tableau_utilities.scripts.gen_config import build_configs +from tableau_utilities.scripts.merge_config import read_file + class ApplyConfigs: """Applies a set of configs to a datasource. Configs prefixed with target_ will be applied to the datasource. @@ -159,8 +161,9 @@ def apply_config_to_datasource(self): if self.debugging_logs: print('Target Column Config:', self.target_column_config) + print('Target Column Config:', type(self.target_column_config)) print('Target Calculated Column Config:', self.target_calculated_column_config) - print('Datasource Column Config:', datasource_column_config) + # print('Datasource Column Config:', datasource_column_config) # # Prepare the configs by inverting, combining and removing configs for other datasources # target_config = self.prepare_configs(self.target_column_config, self.target_calculated_column_config) @@ -185,8 +188,8 @@ def apply_configs(args): debugging_logs = args.debugging_logs datasource_name = args.name datasource_path = args.file_path - target_column_config = args.column_config - target_calculated_column_config = args.calculated_column_config + target_column_config = read_file(args.column_config) + target_calculated_column_config = read_file(args.calculated_column_config) AC = ApplyConfigs(datasource_name, datasource_path, target_column_config, target_calculated_column_config, debugging_logs) From 2de0544dc033dd5369d855bcf466c90586d2b545 Mon Sep 17 00:00:00 2001 From: Jay Rosenthal Date: Sun, 11 Aug 2024 15:55:04 -0700 Subject: [PATCH 30/44] combined the configs: --- tableau_utilities/scripts/apply_configs.py | 33 +++++++++++++--------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/tableau_utilities/scripts/apply_configs.py b/tableau_utilities/scripts/apply_configs.py index 219da69..09ae11a 100644 --- a/tableau_utilities/scripts/apply_configs.py +++ b/tableau_utilities/scripts/apply_configs.py @@ -3,10 +3,13 @@ from typing import Dict, Any from tableau_utilities.tableau_file.tableau_file import Datasource +from tableau_utilities.general.cli_styling import Color, Symbol from tableau_utilities.scripts.datasource import add_metadata_records_as_columns from tableau_utilities.scripts.gen_config import build_configs from tableau_utilities.scripts.merge_config import read_file +color = Color() +symbol = Symbol() class ApplyConfigs: """Applies a set of configs to a datasource. Configs prefixed with target_ will be applied to the datasource. @@ -53,9 +56,6 @@ def invert_config(self, config: Dict[str, Any]) -> Dict[str, Any]: inverted_config = {} - if self.debugging_logs: - print(config) - for column, i in config.items(): for datasource in i['datasources']: new_info = deepcopy(i) @@ -66,7 +66,7 @@ def invert_config(self, config: Dict[str, Any]) -> Dict[str, Any]: inverted_config[datasource['name']].setdefault(column, new_info) if self.debugging_logs: - pp = pprint.PrettyPrinter(indent=4, width=80, depth=None, compact=False) + pp = pprint.PrettyPrinter(indent=4, width=200, depth=None, compact=False) pp.pprint(inverted_config) return inverted_config @@ -96,6 +96,11 @@ def prepare_configs(self, config_A: Dict[str, Any], config_B: Dict[str, Any]) -> # Combine configs combined_config = {**config_A, **config_B} + if self.debugging_logs: + print(f'{color.fg_yellow}AFTER COMBINING CONFIGS{color.reset}') + pp = pprint.PrettyPrinter(indent=4, width=200, depth=None, compact=False) + pp.pprint(combined_config) + return combined_config def compare_columns(self): @@ -159,15 +164,17 @@ def apply_config_to_datasource(self): # Build the config dictionaries from the datasource datasource_column_config, datasource_calculated_column_config = build_configs(datasource, self.datasource_name) - if self.debugging_logs: - print('Target Column Config:', self.target_column_config) - print('Target Column Config:', type(self.target_column_config)) - print('Target Calculated Column Config:', self.target_calculated_column_config) - # print('Datasource Column Config:', datasource_column_config) - - # # Prepare the configs by inverting, combining and removing configs for other datasources - # target_config = self.prepare_configs(self.target_column_config, self.target_calculated_column_config) - # datasource_config = self.prepare_configs(datasource_column_config, datasource_calculated_column_config) + # if self.debugging_logs: + # print('Target Column Config:', self.target_column_config) + # print('Target Column Config:', type(self.target_column_config)) + # print('Target Calculated Column Config:', self.target_calculated_column_config) + # # print('Datasource Column Config:', datasource_column_config) + + # Prepare the configs by inverting, combining and removing configs for other datasources + target_config = self.prepare_configs(self.target_column_config, self.target_calculated_column_config) + datasource_config = self.prepare_configs(datasource_column_config, datasource_calculated_column_config) + + print From 2d0133e44e51fbc5f14d364413aeef289b765740 Mon Sep 17 00:00:00 2001 From: Jay Rosenthal Date: Sun, 11 Aug 2024 16:15:51 -0700 Subject: [PATCH 31/44] getting attribute error on the test but otherwise seems ok --- tableau_utilities/scripts/apply_configs.py | 19 +- tests/test_apply_configs.py | 237 +++++++++++---------- 2 files changed, 143 insertions(+), 113 deletions(-) diff --git a/tableau_utilities/scripts/apply_configs.py b/tableau_utilities/scripts/apply_configs.py index 09ae11a..cc1cc06 100644 --- a/tableau_utilities/scripts/apply_configs.py +++ b/tableau_utilities/scripts/apply_configs.py @@ -1,6 +1,6 @@ from copy import deepcopy import pprint -from typing import Dict, Any +from typing import Dict, Any, List from tableau_utilities.tableau_file.tableau_file import Datasource from tableau_utilities.general.cli_styling import Color, Symbol @@ -103,6 +103,23 @@ def prepare_configs(self, config_A: Dict[str, Any], config_B: Dict[str, Any]) -> return combined_config + def flatten_to_list_of_fields(self, nested_dict: Dict[str, Dict[str, Any]]) -> List[Dict[str, Any]]: + """ + Flattens a nested dictionary by removing one level of nesting and adding a "Caption" key. + + Args: + nested_dict (Dict[str, Dict[str, Any]]): The nested dictionary to flatten. + + Returns: + List[Dict[str, Any]]: A list of dictionaries with "Caption" as a key. + """ + flattened_list = [] + for key, value in nested_dict.items(): + flattened_entry = {"Caption": key} + flattened_entry.update(value) + flattened_list.append(flattened_entry) + return flattened_list + def compare_columns(self): """ Compares the config to a datasource. Generates a list of changes to make the datasource match the config diff --git a/tests/test_apply_configs.py b/tests/test_apply_configs.py index 66e6251..94edf04 100644 --- a/tests/test_apply_configs.py +++ b/tests/test_apply_configs.py @@ -5,8 +5,8 @@ @pytest.fixture def apply_configs(): - return ApplyConfigs(datasource_name="my_datasource_1", datasource_path="", column_config={}, - calculated_column_config={}, debugging_logs=False) + return ApplyConfigs(datasource_name="my_datasource_1", datasource_path="", target_column_config={}, + target_calculated_column_config={}, debugging_logs=False) def test_invert_config_single_datasource(apply_configs): @@ -86,134 +86,147 @@ def test_invert_config_multiple_datasources(apply_configs): result = apply_configs.invert_config(sample_config) assert result == expected_output - -def test_invert_config_combined(apply_configs): - sample_config = { - "Column1": { - "description": "Description of Column1", - "folder": "Folder1", - "persona": "string_dimension", - "datasources": [ - { - "name": "my_datasource_1", - "local-name": "MY_COLUMN_1", - "sql_alias": "MY_COLUMN_1_ALIAS" - } - ] +def test_flatten_to_list_of_fields(apply_configs): + sample_dict = { + 'My Caption 1': { + 'description': 'A perfect description', + 'folder': 'My Folder', + 'local-name': 'MY_FIELD_1', + 'persona': 'string_dimension', + 'remote_name': 'MY_FIELD_1' }, - "Column2": { - "description": "Description of Column2", - "folder": "Folder2", - "persona": "string_dimension", - "datasources": [ - { - "name": "my_datasource_1", - "local-name": "MY_COLUMN_1", - "sql_alias": "MY_COLUMN_1_ALIAS" - }, - { - "name": "my_datasource_2", - "local-name": "MY_COLUMN_2", - "sql_alias": "MY_COLUMN_2_ALIAS" - } - ] + 'My Caption 2': { + 'description': 'Another perfect description', + 'folder': 'My Folder', + 'local-name': 'MY_FIELD_2', + 'persona': 'string_dimension', + 'remote_name': 'MY_FIELD_2' } } - expected_output = { - "my_datasource_1": { - "Column1": { - "description": "Description of Column1", - "folder": "Folder1", - "persona": "string_dimension", - "local-name": "MY_COLUMN_1", - "remote_name": "MY_COLUMN_1_ALIAS" - }, - "Column2": { - "description": "Description of Column2", - "folder": "Folder2", - "persona": "string_dimension", - "local-name": "MY_COLUMN_1", - "remote_name": "MY_COLUMN_1_ALIAS" - } + expected_output = [ + { + 'Caption': 'My Caption 1', + 'description': 'A perfect description', + 'folder': 'My Folder', + 'local-name': 'MY_FIELD_1', + 'persona': 'string_dimension', + 'remote_name': 'MY_FIELD_1' }, - "my_datasource_2": { - "Column2": { - "description": "Description of Column2", - "folder": "Folder2", - "persona": "string_dimension", - "local-name": "MY_COLUMN_2", - "remote_name": "MY_COLUMN_2_ALIAS" - } + { + 'Caption': 'My Caption 2', + 'description': 'Another perfect description', + 'folder': 'My Folder', + 'local-name': 'MY_FIELD_2', + 'persona': 'string_dimension', + 'remote_name': 'MY_FIELD_2' } - } + ] - result = apply_configs.invert_config(sample_config) + result = apply_configs.flatten_to_list_of_fields(sample_dict) assert result == expected_output -def test_prepare_configs(apply_configs): - sample_config_A = { - "Column1": { - "description": "Description of Column1", - "folder": "Folder1", - "persona": "string_dimension", - "datasources": [ - { - "name": "my_datasource_1", - "local-name": "MY_COLUMN_1", - "sql_alias": "MY_COLUMN_1_ALIAS" - } - ] +# def test_prepare_configs(apply_configs): +# sample_config_A = { +# "Column1": { +# "description": "Description of Column1", +# "folder": "Folder1", +# "persona": "string_dimension", +# "datasources": [ +# { +# "name": "my_datasource_1", +# "local-name": "MY_COLUMN_1", +# "sql_alias": "MY_COLUMN_1_ALIAS" +# } +# ] +# }, +# "Column2": { +# "description": "Description of Column2", +# "folder": "Folder2", +# "persona": "string_dimension", +# "datasources": [ +# { +# "name": "my_datasource_1", +# "local-name": "MY_COLUMN_1", +# "sql_alias": "MY_COLUMN_1_ALIAS" +# }, +# { +# "name": "my_datasource_2", +# "local-name": "MY_COLUMN_2", +# "sql_alias": "MY_COLUMN_2_ALIAS" +# } +# ] +# } +# } +# +# sample_config_B = { +# "# ID": { +# "description": "Distinct Count of the ID", +# "calculation": "COUNTD([ID])", +# "folder": "My Data", +# "persona": "continuous_number_measure", +# "datasources": [ +# { +# "name": "my_datasource_1", +# "local-name": "MY_COLUMN_1", +# "sql_alias": "MY_COLUMN_1_ALIAS" +# }, +# { +# "name": "my_datasource_2", +# "local-name": "MY_COLUMN_2", +# "sql_alias": "MY_COLUMN_2_ALIAS" +# } +# ], +# "default_format": "n#,##0;-#,##0" +# } +# } +# +# expected_output = +# +# +# result = apply_configs.prepare_configs(sample_config_A, sample_config_B) +# assert result == expected_output + +def test_flatten_to_list_of_fields(apply_configs): + sample_dict = { + 'My Caption 1': { + 'description': 'A perfect description', + 'folder': 'My Folder', + 'local-name': 'MY_FIELD_1', + 'persona': 'string_dimension', + 'remote_name': 'MY_FIELD_1' }, - "Column2": { - "description": "Description of Column2", - "folder": "Folder2", - "persona": "string_dimension", - "datasources": [ - { - "name": "my_datasource_1", - "local-name": "MY_COLUMN_1", - "sql_alias": "MY_COLUMN_1_ALIAS" - }, - { - "name": "my_datasource_2", - "local-name": "MY_COLUMN_2", - "sql_alias": "MY_COLUMN_2_ALIAS" - } - ] + 'My Caption 2': { + 'description': 'Another perfect description', + 'folder': 'My Folder', + 'local-name': 'MY_FIELD_2', + 'persona': 'string_dimension', + 'remote_name': 'MY_FIELD_2' } } - sample_config_B = { - "# ID": { - "description": "Distinct Count of the ID", - "calculation": "COUNTD([ID])", - "folder": "My Data", - "persona": "continuous_number_measure", - "datasources": [ - { - "name": "my_datasource_1", - "local-name": "MY_COLUMN_1", - "sql_alias": "MY_COLUMN_1_ALIAS" - }, - { - "name": "my_datasource_2", - "local-name": "MY_COLUMN_2", - "sql_alias": "MY_COLUMN_2_ALIAS" - } - ], - "default_format": "n#,##0;-#,##0" + expected_output = [ + { + 'Caption': 'My Caption 1', + 'description': 'A perfect description', + 'folder': 'My Folder', + 'local-name': 'MY_FIELD_1', + 'persona': 'string_dimension', + 'remote_name': 'MY_FIELD_1' + }, + {'Caption': 'My Caption 2', + 'description': 'Another perfect description', + 'folder': 'My Folder', + 'local-name': 'MY_FIELD_2', + 'persona': 'string_dimension', + 'remote_name': 'MY_FIELD_2' } - } + ] - expected_output = - - - result = apply_configs.prepare_configs(sample_config_A, sample_config_B) + result = apply_configs.flatten_to_list_of_fields(sample_dict) assert result == expected_output - # def test_select_matching_datasource_config(apply_configs): # sample_config = { # "my_datasource_1": { From 93264e53792255788201c05b737fc6853f5527ed Mon Sep 17 00:00:00 2001 From: Jay Rosenthal Date: Sun, 11 Aug 2024 16:34:36 -0700 Subject: [PATCH 32/44] got changes list: --- tableau_utilities/scripts/apply_configs.py | 74 +++++++++++++++++++--- 1 file changed, 66 insertions(+), 8 deletions(-) diff --git a/tableau_utilities/scripts/apply_configs.py b/tableau_utilities/scripts/apply_configs.py index cc1cc06..4aa0e43 100644 --- a/tableau_utilities/scripts/apply_configs.py +++ b/tableau_utilities/scripts/apply_configs.py @@ -103,6 +103,7 @@ def prepare_configs(self, config_A: Dict[str, Any], config_B: Dict[str, Any]) -> return combined_config + def flatten_to_list_of_fields(self, nested_dict: Dict[str, Dict[str, Any]]) -> List[Dict[str, Any]]: """ Flattens a nested dictionary by removing one level of nesting and adding a "Caption" key. @@ -118,7 +119,48 @@ def flatten_to_list_of_fields(self, nested_dict: Dict[str, Dict[str, Any]]) -> L flattened_entry = {"Caption": key} flattened_entry.update(value) flattened_list.append(flattened_entry) + + if self.debugging_logs: + print(f'{color.fg_yellow}AFTER FLATTENING{color.reset}') + for field_config in flattened_list: + print(field_config) + return flattened_list + # + # def merge_configs(self, target_config: List[Dict[str, Any]], datasource_config: List[Dict[str, Any]]) -> List[ + # Dict[str, Any]]: + # """Merges two lists of dictionaries ensuring all 'local-name' from both lists are present. + # + # If the 'local-name' is in both lists, the values from `target_config` are used. + # + # Args: + # target_config (List[Dict[str, Any]]): The target configuration list of dictionaries. + # datasource_config (List[Dict[str, Any]]): The datasource configuration list of dictionaries. + # + # Returns: + # List[Dict[str, Any]]: A merged list of dictionaries. + # """ + # merged_dict = {} + # + # # Add all entries from datasource_config to merged_dict + # for entry in datasource_config: + # local_name = entry['local-name'] + # merged_dict[local_name] = entry + # + # # Update or add entries from target_config to merged_dict + # for entry in target_config: + # local_name = entry['local-name'] + # merged_dict[local_name] = entry + # + # # Convert merged_dict back to a list of dictionaries + # merged_list = list(merged_dict.values()) + # + # # if self.debugging_logs: + # print(f'{color.fg_yellow}AFTER MERGING{color.reset}') + # for field_config in merged_list: + # print(field_config) + # + # return merged_list def compare_columns(self): """ Compares the config to a datasource. Generates a list of changes to make the datasource match the config @@ -128,19 +170,31 @@ def compare_columns(self): """ - # compare the caption. If the caption matches compare the attributes + pass - def compare_configs(self, config, datasource_cureent_config, datasource_name): - """ Compares the config to a datasource. Generates a list of changes to make the datasource match the config + def compare_columns(self, target_config: List[Dict[str, Any]], datasource_config: List[Dict[str, Any]]) -> List[ + Dict[str, Any]]: + """Compares the target config to the datasource config and generates a list of changes to make the datasource match the target config. - Returns: - dict: a dictionary with the columns that need updating + Args: + target_config (List[Dict[str, Any]]): The target configuration list of dictionaries. + datasource_config (List[Dict[str, Any]]): The datasource configuration list of dictionaries. + Returns: + List[Dict[str, Any]]: A list of dictionaries with the columns that need updating. """ + changes_to_make = [] - # compare the caption. If the caption matches compare the attributes - pass + for target_entry in target_config: + if target_entry not in datasource_config: + changes_to_make.append(target_entry) + + print(f'{color.fg_yellow}AFTER MERGING{color.reset}') + for field_config in changes_to_make: + print(field_config) + + return changes_to_make def execute_changes(self, column_config, calculated_field_config, datasource): @@ -191,7 +245,11 @@ def apply_config_to_datasource(self): target_config = self.prepare_configs(self.target_column_config, self.target_calculated_column_config) datasource_config = self.prepare_configs(datasource_column_config, datasource_calculated_column_config) - print + target_config = self.flatten_to_list_of_fields(target_config) + datasource_config = self.flatten_to_list_of_fields(datasource_config) + + # merged_config = self.merge_configs(target_config, datasource_config) + changes_to_make = self.compare_columns(target_config, datasource_config) From f6e50c34eb7434c74d788ef33a4aaadc72cd4f5d Mon Sep 17 00:00:00 2001 From: Jay Rosenthal Date: Sun, 11 Aug 2024 16:42:04 -0700 Subject: [PATCH 33/44] working on the executre: --- tableau_utilities/scripts/apply_configs.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/tableau_utilities/scripts/apply_configs.py b/tableau_utilities/scripts/apply_configs.py index 4aa0e43..8b26ea5 100644 --- a/tableau_utilities/scripts/apply_configs.py +++ b/tableau_utilities/scripts/apply_configs.py @@ -197,7 +197,7 @@ def compare_columns(self, target_config: List[Dict[str, Any]], datasource_config return changes_to_make - def execute_changes(self, column_config, calculated_field_config, datasource): + def execute_changes(self, columns_list, datasource): """ Applies changes to make Args: @@ -207,7 +207,19 @@ def execute_changes(self, column_config, calculated_field_config, datasource): Returns: """ - pass + + + for column in columns_list: + column = datasource.columns.get(column['local-name']) + + column.caption = caption or column.caption + column.role = persona.get('role') or column.role + column.type = persona.get('role_type') or column.type + column.datatype = persona.get('datatype') or column.datatype + column.desc = desc or column.desc + column.calculation = calculation or column.calculation + + def apply_config_to_datasource(self): """ Applies a set of configs (column_config and calculated_column_config) to a datasource. @@ -253,6 +265,7 @@ def apply_config_to_datasource(self): + # Get the changes to make for the column config # Get the changes to make for the calculation config From 7b82ed6c23d8cd22a2dd4fdf28770b380138ca3b Mon Sep 17 00:00:00 2001 From: Jay Rosenthal Date: Sun, 11 Aug 2024 20:28:28 -0700 Subject: [PATCH 34/44] added execute --- tableau_utilities/scripts/apply_configs.py | 47 ++++++++++++++-------- 1 file changed, 30 insertions(+), 17 deletions(-) diff --git a/tableau_utilities/scripts/apply_configs.py b/tableau_utilities/scripts/apply_configs.py index 8b26ea5..e051c15 100644 --- a/tableau_utilities/scripts/apply_configs.py +++ b/tableau_utilities/scripts/apply_configs.py @@ -1,9 +1,14 @@ from copy import deepcopy +import os import pprint +import shutil from typing import Dict, Any, List +from time import time + from tableau_utilities.tableau_file.tableau_file import Datasource from tableau_utilities.general.cli_styling import Color, Symbol +from tableau_utilities.general.config_column_persona import personas from tableau_utilities.scripts.datasource import add_metadata_records_as_columns from tableau_utilities.scripts.gen_config import build_configs from tableau_utilities.scripts.merge_config import read_file @@ -162,16 +167,6 @@ def flatten_to_list_of_fields(self, nested_dict: Dict[str, Dict[str, Any]]) -> L # # return merged_list - def compare_columns(self): - """ Compares the config to a datasource. Generates a list of changes to make the datasource match the config - - Returns: - dict: a dictionary with the columns that need updating - - """ - - - pass def compare_columns(self, target_config: List[Dict[str, Any]], datasource_config: List[Dict[str, Any]]) -> List[ Dict[str, Any]]: @@ -196,29 +191,47 @@ def compare_columns(self, target_config: List[Dict[str, Any]], datasource_config return changes_to_make - - def execute_changes(self, columns_list, datasource): + def execute_changes(self, columns_list: List[Dict[str, Any]], datasource): """ Applies changes to make Args: - config: + columns_list: datasource: Returns: """ - for column in columns_list: column = datasource.columns.get(column['local-name']) - column.caption = caption or column.caption + persona = personas.get(column['persona'].lower(), {}) + + column.caption = column['caption'] or column.caption column.role = persona.get('role') or column.role column.type = persona.get('role_type') or column.type column.datatype = persona.get('datatype') or column.datatype - column.desc = desc or column.desc - column.calculation = calculation or column.calculation + column.desc = column['description'] or column.desc + column.calculation = column['calculation'] or column.calculation + + if self.debugging_logs: + print(f'{color.fg_yellow}column:{color.reset}{column}') + datasource.enforce_column(column, remote_name=column['remote_name'], folder_name=column['folder']) + + + start = time() + print(f'{color.fg_cyan}...Extracting {self.datasource_name}...{color.reset}') + save_folder = f'{self.datasource_name} - AFTER' + os.makedirs(save_folder, exist_ok=True) + if datasource.extension == 'tds': + xml_path = os.path.join(save_folder, self.datasource_name) + shutil.copy(self.datasource_path, xml_path) + else: + xml_path = datasource.unzip(extract_to=save_folder, unzip_all=True) + if self.debugging_logs: + print(f'{color.fg_green}{symbol.success} (Done in {round(time() - start)} sec) ' + f'AFTER - TDS SAVED TO: {color.fg_yellow}{xml_path}{color.reset}') def apply_config_to_datasource(self): From e759932209b7b1512c31951d174dbc25355401db Mon Sep 17 00:00:00 2001 From: Jay Rosenthal Date: Sun, 11 Aug 2024 20:29:46 -0700 Subject: [PATCH 35/44] run execute --- tableau_utilities/scripts/apply_configs.py | 30 +++++++++++----------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/tableau_utilities/scripts/apply_configs.py b/tableau_utilities/scripts/apply_configs.py index e051c15..d75461b 100644 --- a/tableau_utilities/scripts/apply_configs.py +++ b/tableau_utilities/scripts/apply_configs.py @@ -245,8 +245,6 @@ def apply_config_to_datasource(self): calculated_field_config: debugging_logs: - - Returns: None @@ -276,19 +274,21 @@ def apply_config_to_datasource(self): # merged_config = self.merge_configs(target_config, datasource_config) changes_to_make = self.compare_columns(target_config, datasource_config) - - - - # Get the changes to make for the column config - # Get the changes to make for the calculation config - - # Apply the changes for the column config - # Apply the changes for the calc config - - # Clean up the empty folders - - # Save the file - pass + self.execute_changes(changes_to_make, datasource) + + # + # + # + # # Get the changes to make for the column config + # # Get the changes to make for the calculation config + # + # # Apply the changes for the column config + # # Apply the changes for the calc config + # + # # Clean up the empty folders + # + # # Save the file + # pass def apply_configs(args): From edca3684f3a27ac3916c24dde07c95c1a4650bd7 Mon Sep 17 00:00:00 2001 From: Jay Rosenthal Date: Sun, 11 Aug 2024 21:00:42 -0700 Subject: [PATCH 36/44] looks like it ran: --- tableau_utilities/scripts/apply_configs.py | 30 +++++++++++++++------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/tableau_utilities/scripts/apply_configs.py b/tableau_utilities/scripts/apply_configs.py index d75461b..a5c7c57 100644 --- a/tableau_utilities/scripts/apply_configs.py +++ b/tableau_utilities/scripts/apply_configs.py @@ -121,7 +121,7 @@ def flatten_to_list_of_fields(self, nested_dict: Dict[str, Dict[str, Any]]) -> L """ flattened_list = [] for key, value in nested_dict.items(): - flattened_entry = {"Caption": key} + flattened_entry = {"caption": key} flattened_entry.update(value) flattened_list.append(flattened_entry) @@ -202,22 +202,34 @@ def execute_changes(self, columns_list: List[Dict[str, Any]], datasource): """ - for column in columns_list: - column = datasource.columns.get(column['local-name']) + print(f'{color.fg_cyan}...Applying Changes to {self.datasource_name}...{color.reset}') - persona = personas.get(column['persona'].lower(), {}) + for each_column in columns_list: + if self.debugging_logs: + print(f'{color.fg_yellow}column:{color.reset}{each_column}') + + # + + column = datasource.columns.get(each_column['local-name']) + + persona = personas.get(each_column['persona'].lower(), {}) - column.caption = column['caption'] or column.caption + if self.debugging_logs: + print(f'{color.fg_yellow}persona:{color.reset}{persona}') + + column.caption = each_column['caption'] or column.caption column.role = persona.get('role') or column.role column.type = persona.get('role_type') or column.type column.datatype = persona.get('datatype') or column.datatype - column.desc = column['description'] or column.desc - column.calculation = column['calculation'] or column.calculation + column.desc = each_column['description'] or column.desc + + if 'calculation' in each_column: + column.calculation = each_column['calculation'] if self.debugging_logs: - print(f'{color.fg_yellow}column:{color.reset}{column}') + print(f'{color.fg_yellow}column:{color.reset}{each_column}') - datasource.enforce_column(column, remote_name=column['remote_name'], folder_name=column['folder']) + datasource.enforce_column(column, remote_name=each_column['remote_name'], folder_name=each_column['folder']) start = time() From d7d4e6092a6f2b890fc7dc63c054091144db22ca Mon Sep 17 00:00:00 2001 From: Jay Rosenthal Date: Sun, 11 Aug 2024 21:06:46 -0700 Subject: [PATCH 37/44] Edited the file --- tableau_utilities/scripts/apply_configs.py | 30 +++++++++++++--------- 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/tableau_utilities/scripts/apply_configs.py b/tableau_utilities/scripts/apply_configs.py index a5c7c57..061f11e 100644 --- a/tableau_utilities/scripts/apply_configs.py +++ b/tableau_utilities/scripts/apply_configs.py @@ -231,19 +231,24 @@ def execute_changes(self, columns_list: List[Dict[str, Any]], datasource): datasource.enforce_column(column, remote_name=each_column['remote_name'], folder_name=each_column['folder']) - start = time() - print(f'{color.fg_cyan}...Extracting {self.datasource_name}...{color.reset}') - save_folder = f'{self.datasource_name} - AFTER' - os.makedirs(save_folder, exist_ok=True) - if datasource.extension == 'tds': - xml_path = os.path.join(save_folder, self.datasource_name) - shutil.copy(self.datasource_path, xml_path) - else: - xml_path = datasource.unzip(extract_to=save_folder, unzip_all=True) - if self.debugging_logs: - print(f'{color.fg_green}{symbol.success} (Done in {round(time() - start)} sec) ' - f'AFTER - TDS SAVED TO: {color.fg_yellow}{xml_path}{color.reset}') + print(f'{color.fg_cyan}...Saving datasource changes...{color.reset}') + datasource.save() + print(f'{color.fg_green}{symbol.success} (Done in {round(time() - start)} sec) ' + f'Saved datasource changes: {color.fg_yellow}{self.datasource_path}{color.reset}') + + # start = time() + # print(f'{color.fg_cyan}...Extracting {self.datasource_name}...{color.reset}') + # save_folder = f'{self.datasource_name} - AFTER' + # os.makedirs(save_folder, exist_ok=True) + # if datasource.extension == 'tds': + # xml_path = os.path.join(save_folder, self.datasource_name) + # shutil.copy(self.datasource_path, xml_path) + # else: + # xml_path = datasource.unzip(extract_to=save_folder, unzip_all=True) + # if self.debugging_logs: + # print(f'{color.fg_green}{symbol.success} (Done in {round(time() - start)} sec) ' + # f'AFTER - TDS SAVED TO: {color.fg_yellow}{xml_path}{color.reset}') def apply_config_to_datasource(self): @@ -264,6 +269,7 @@ def apply_config_to_datasource(self): datasource = Datasource(self.datasource_path) + # Run column init on the datasource to make sure columns aren't hiding in Metadata records datasource = add_metadata_records_as_columns(datasource, self.debugging_logs) From d56bb4ae07c770701258161cafe5960ff656d85c Mon Sep 17 00:00:00 2001 From: Jay Rosenthal Date: Sun, 11 Aug 2024 21:07:19 -0700 Subject: [PATCH 38/44] altering the file --- tableau_utilities/scripts/apply_configs.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/tableau_utilities/scripts/apply_configs.py b/tableau_utilities/scripts/apply_configs.py index 061f11e..fa49640 100644 --- a/tableau_utilities/scripts/apply_configs.py +++ b/tableau_utilities/scripts/apply_configs.py @@ -237,19 +237,6 @@ def execute_changes(self, columns_list: List[Dict[str, Any]], datasource): print(f'{color.fg_green}{symbol.success} (Done in {round(time() - start)} sec) ' f'Saved datasource changes: {color.fg_yellow}{self.datasource_path}{color.reset}') - # start = time() - # print(f'{color.fg_cyan}...Extracting {self.datasource_name}...{color.reset}') - # save_folder = f'{self.datasource_name} - AFTER' - # os.makedirs(save_folder, exist_ok=True) - # if datasource.extension == 'tds': - # xml_path = os.path.join(save_folder, self.datasource_name) - # shutil.copy(self.datasource_path, xml_path) - # else: - # xml_path = datasource.unzip(extract_to=save_folder, unzip_all=True) - # if self.debugging_logs: - # print(f'{color.fg_green}{symbol.success} (Done in {round(time() - start)} sec) ' - # f'AFTER - TDS SAVED TO: {color.fg_yellow}{xml_path}{color.reset}') - def apply_config_to_datasource(self): """ Applies a set of configs (column_config and calculated_column_config) to a datasource. From b6c01241cb19b9f9bc5a8aa314c16f3fa7071243 Mon Sep 17 00:00:00 2001 From: Jay Rosenthal Date: Sat, 17 Aug 2024 06:14:30 -0700 Subject: [PATCH 39/44] fixed tests - 6 passing --- tests/test_apply_configs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_apply_configs.py b/tests/test_apply_configs.py index 94edf04..5ae98a2 100644 --- a/tests/test_apply_configs.py +++ b/tests/test_apply_configs.py @@ -208,14 +208,14 @@ def test_flatten_to_list_of_fields(apply_configs): expected_output = [ { - 'Caption': 'My Caption 1', + 'caption': 'My Caption 1', 'description': 'A perfect description', 'folder': 'My Folder', 'local-name': 'MY_FIELD_1', 'persona': 'string_dimension', 'remote_name': 'MY_FIELD_1' }, - {'Caption': 'My Caption 2', + {'caption': 'My Caption 2', 'description': 'Another perfect description', 'folder': 'My Folder', 'local-name': 'MY_FIELD_2', From 165ea7550228419d3da055b20214685843f9fcfc Mon Sep 17 00:00:00 2001 From: Jay Rosenthal Date: Sat, 17 Aug 2024 06:29:54 -0700 Subject: [PATCH 40/44] fixed datasouce select test --- tests/test_apply_configs.py | 72 +++++++++++++++++++------------------ 1 file changed, 37 insertions(+), 35 deletions(-) diff --git a/tests/test_apply_configs.py b/tests/test_apply_configs.py index 5ae98a2..1547c79 100644 --- a/tests/test_apply_configs.py +++ b/tests/test_apply_configs.py @@ -227,41 +227,43 @@ def test_flatten_to_list_of_fields(apply_configs): result = apply_configs.flatten_to_list_of_fields(sample_dict) assert result == expected_output -# def test_select_matching_datasource_config(apply_configs): -# sample_config = { -# "my_datasource_1": { -# "Column2": { -# "description": "Description of Column2", -# "folder": "Folder2", -# "persona": "string_dimension", -# "local-name": "MY_COLUMN_1", -# "remote_name": "MY_COLUMN_1_ALIAS" -# } -# }, -# "my_datasource_2": { -# "Column2": { -# "description": "Description of Column2", -# "folder": "Folder2", -# "persona": "string_dimension", -# "local-name": "MY_COLUMN_2", -# "remote_name": "MY_COLUMN_2_ALIAS" -# } -# } -# } -# -# expected_output = { -# "my_datasource_1": { -# "Column2": { -# "description": "Description of Column2", -# "folder": "Folder2", -# "persona": "string_dimension", -# "local-name": "MY_COLUMN_1", -# "remote_name": "MY_COLUMN_1_ALIAS" -# } -# }, -# } -# result = apply_configs.select_matching_datasource_config(sample_config) -# assert result == expected_output +def test_select_matching_datasource_config(apply_configs): + + print('testing select_matching_datasource_config') + print(apply_configs.datasource_name) + + sample_config = { + "my_datasource_1": { + "Column1": { + "description": "Description of Column1", + "folder": "Folder1", + "persona": "string_dimension", + "local-name": "MY_COLUMN_1", + "remote_name": "MY_COLUMN_1_ALIAS" + } + }, + "my_datasource_2": { + "Column2": { + "description": "Description of Column2", + "folder": "Folder2", + "persona": "string_dimension", + "local-name": "MY_COLUMN_2", + "remote_name": "MY_COLUMN_2_ALIAS" + } + } + } + + expected_output = { + "Column1": { + "description": "Description of Column1", + "folder": "Folder1", + "persona": "string_dimension", + "local-name": "MY_COLUMN_1", + "remote_name": "MY_COLUMN_1_ALIAS" + } + } + result = apply_configs.select_matching_datasource_config(sample_config) + assert result == expected_output if __name__ == '__main__': pytest.main() From fef03fc39d3efc58d13163dad09b0fe9835f35c0 Mon Sep 17 00:00:00 2001 From: Jay Rosenthal Date: Sat, 17 Aug 2024 06:30:27 -0700 Subject: [PATCH 41/44] spacing --- tests/test_apply_configs.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/test_apply_configs.py b/tests/test_apply_configs.py index 1547c79..8552e87 100644 --- a/tests/test_apply_configs.py +++ b/tests/test_apply_configs.py @@ -87,6 +87,7 @@ def test_invert_config_multiple_datasources(apply_configs): assert result == expected_output def test_flatten_to_list_of_fields(apply_configs): + sample_dict = { 'My Caption 1': { 'description': 'A perfect description', @@ -189,6 +190,7 @@ def test_flatten_to_list_of_fields(apply_configs): # assert result == expected_output def test_flatten_to_list_of_fields(apply_configs): + sample_dict = { 'My Caption 1': { 'description': 'A perfect description', @@ -229,9 +231,6 @@ def test_flatten_to_list_of_fields(apply_configs): def test_select_matching_datasource_config(apply_configs): - print('testing select_matching_datasource_config') - print(apply_configs.datasource_name) - sample_config = { "my_datasource_1": { "Column1": { From 2db4a563c141824d82302a5a87db0a1c3977a00b Mon Sep 17 00:00:00 2001 From: Jay Rosenthal Date: Sun, 18 Aug 2024 05:54:38 -0700 Subject: [PATCH 42/44] prepare configs test --- tests/test_apply_configs.py | 144 +++++++++++++++++++++--------------- 1 file changed, 84 insertions(+), 60 deletions(-) diff --git a/tests/test_apply_configs.py b/tests/test_apply_configs.py index 8552e87..8981f66 100644 --- a/tests/test_apply_configs.py +++ b/tests/test_apply_configs.py @@ -128,66 +128,89 @@ def test_flatten_to_list_of_fields(apply_configs): assert result == expected_output -# def test_prepare_configs(apply_configs): -# sample_config_A = { -# "Column1": { -# "description": "Description of Column1", -# "folder": "Folder1", -# "persona": "string_dimension", -# "datasources": [ -# { -# "name": "my_datasource_1", -# "local-name": "MY_COLUMN_1", -# "sql_alias": "MY_COLUMN_1_ALIAS" -# } -# ] -# }, -# "Column2": { -# "description": "Description of Column2", -# "folder": "Folder2", -# "persona": "string_dimension", -# "datasources": [ -# { -# "name": "my_datasource_1", -# "local-name": "MY_COLUMN_1", -# "sql_alias": "MY_COLUMN_1_ALIAS" -# }, -# { -# "name": "my_datasource_2", -# "local-name": "MY_COLUMN_2", -# "sql_alias": "MY_COLUMN_2_ALIAS" -# } -# ] -# } -# } -# -# sample_config_B = { -# "# ID": { -# "description": "Distinct Count of the ID", -# "calculation": "COUNTD([ID])", -# "folder": "My Data", -# "persona": "continuous_number_measure", -# "datasources": [ -# { -# "name": "my_datasource_1", -# "local-name": "MY_COLUMN_1", -# "sql_alias": "MY_COLUMN_1_ALIAS" -# }, -# { -# "name": "my_datasource_2", -# "local-name": "MY_COLUMN_2", -# "sql_alias": "MY_COLUMN_2_ALIAS" -# } -# ], -# "default_format": "n#,##0;-#,##0" -# } -# } -# -# expected_output = -# -# -# result = apply_configs.prepare_configs(sample_config_A, sample_config_B) -# assert result == expected_output +def test_prepare_configs(apply_configs): + sample_config_A = { + "Column1": { + "description": "Description of Column1", + "folder": "Folder1", + "persona": "string_dimension", + "datasources": [ + { + "name": "my_datasource_1", + "local-name": "MY_COLUMN_1", + "sql_alias": "MY_COLUMN_1_ALIAS" + } + ] + }, + "Column2": { + "description": "Description of Column2", + "folder": "Folder2", + "persona": "string_dimension", + "datasources": [ + { + "name": "my_datasource_1", + "local-name": "MY_COLUMN_1", + "sql_alias": "MY_COLUMN_1_ALIAS" + }, + { + "name": "my_datasource_2", + "local-name": "MY_COLUMN_2", + "sql_alias": "MY_COLUMN_2_ALIAS" + } + ] + } + } + + sample_config_B = { + "# ID": { + "description": "Distinct Count of the ID", + "calculation": "COUNTD([ID])", + "folder": "My Data", + "persona": "continuous_number_measure", + "datasources": [ + { + "name": "my_datasource_1", + "local-name": "MY_COLUMN_1", + "sql_alias": "MY_COLUMN_1_ALIAS" + }, + { + "name": "my_datasource_2", + "local-name": "MY_COLUMN_2", + "sql_alias": "MY_COLUMN_2_ALIAS" + } + ], + "default_format": "n#,##0;-#,##0" + } + } + + expected_output = { + "Column1": { + "description": "Description of Column1", + "folder": "Folder1", + "persona": "string_dimension", + "local-name": "MY_COLUMN_1", + "remote_name": "MY_COLUMN_1_ALIAS" + }, + "Column2": { + "description": "Description of Column2", + "folder": "Folder2", + "persona": "string_dimension", + "local-name": "MY_COLUMN_1", + "remote_name": "MY_COLUMN_1_ALIAS" + }, + "# ID": { + "description": "Distinct Count of the ID", + "calculation": "COUNTD([ID])", + "default_format": "n#,##0;-#,##0", + "folder": "My Data", + "persona": "continuous_number_measure", + "local-name": "MY_COLUMN_1", + "remote_name": "MY_COLUMN_1_ALIAS" + } + } + + result = apply_configs.prepare_configs(sample_config_A, sample_config_B) + assert result == expected_output def test_flatten_to_list_of_fields(apply_configs): @@ -264,5 +287,6 @@ def test_select_matching_datasource_config(apply_configs): result = apply_configs.select_matching_datasource_config(sample_config) assert result == expected_output + if __name__ == '__main__': pytest.main() From 2fbdae82e29c2394f67fe0e684de7b53e6030cda Mon Sep 17 00:00:00 2001 From: Jay Rosenthal Date: Sun, 18 Aug 2024 05:55:15 -0700 Subject: [PATCH 43/44] add verbose logging to workflow --- .github/workflows/python-package.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 14e8970..01d3d3a 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -34,7 +34,7 @@ jobs: flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - name: Run Pytest on the tests in tests/ run: - pytest tests/ -p no:warnings + pytest tests/ -p no:warnings -vv - name: Run pytest on tableau-utilities run: | cd tableau_utilities && pytest -v From 42c30ee096947a0468f26f0d9029b859e6ea3fda Mon Sep 17 00:00:00 2001 From: Jay Rosenthal Date: Tue, 10 Sep 2024 05:01:18 -0700 Subject: [PATCH 44/44] logging --- tableau_utilities/scripts/gen_config.py | 9 +++++++-- tableau_utilities/scripts/merge_config.py | 6 ++++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/tableau_utilities/scripts/gen_config.py b/tableau_utilities/scripts/gen_config.py index c403de0..8bc9ef5 100644 --- a/tableau_utilities/scripts/gen_config.py +++ b/tableau_utilities/scripts/gen_config.py @@ -10,22 +10,28 @@ from tableau_utilities.tableau_server.tableau_server import TableauServer -def load_csv_with_definitions(file=None): +def load_csv_with_definitions(file=None, debugging_logs=False): """ Returns a dictionary with the definitions from a csv. The columns are expected to include column_name and description Args: file: The path to the .csv file with the definitions. The csv must include a column_name and description. + debugging_logs: Prints information to consolde if true Returns: dictionary mapping column name to definition + """ definitions_mapping = dict() df = pd.read_csv(file) + df.columns = df.columns.str.lower() definitions = df.to_dict('records') + if debugging_logs: + print(definitions) + # Check that the csv contains column_name and description headers column_names = list(df.columns) if 'column_name' not in column_names or 'description' not in column_names: @@ -37,7 +43,6 @@ def load_csv_with_definitions(file=None): return definitions_mapping - def choose_persona(role, role_type, datatype, caption): """ The config relies on a persona which is a combination of role, role_type and datatype for each column. This returns the persona name or raises an exception if the combination is not found diff --git a/tableau_utilities/scripts/merge_config.py b/tableau_utilities/scripts/merge_config.py index 1535a55..22af11f 100644 --- a/tableau_utilities/scripts/merge_config.py +++ b/tableau_utilities/scripts/merge_config.py @@ -112,7 +112,7 @@ def sort_config(config, debugging_logs): if debugging_logs: print('KEY', k) - print('CONGIG', v) + print('CONFIG', v) print('DATASOURCES', v['datasources']) sorted_datasources = sorted(v['datasources'], key=lambda d: d['name']) @@ -177,13 +177,15 @@ def merge_configs(args, server=None): elif merge_with == 'csv': # Read files existing_config = read_file(existing_config) - definitions_mapping = load_csv_with_definitions(file=definitions_csv_path) + definitions_mapping = load_csv_with_definitions(file=definitions_csv_path, debugging_logs=debugging_logs) # Merge new_config = add_definitions_mapping(existing_config, definitions_mapping) # Sort and write the merged config new_config = sort_config(new_config, debugging_logs) write_file(file_name=file_name, config=new_config, debugging_logs=debugging_logs) + print(f'{color.fg_yellow}DEFINITIONS CSV {symbol.arrow_r} ' + f'{color.fg_grey}{definitions_csv_path}{color.reset}') print(f'{color.fg_yellow}EXISTING CONFIG {symbol.arrow_r} ' f'{color.fg_grey}{existing_config_path}{color.reset}') print(f'{color.fg_yellow}ADDITIONAL CONFIG {symbol.arrow_r} '