Clean naming

airflow_example/dags/tableau_datasource_update/tableau_datasource_update.py

@@ -314,54 +314,54 @@ def __compare_folders(self, datasource_id, tds_folders, cfg_folders):
     def execute(self, context):
         """ Update Tableau datasource according to config. """
 
-        github_conn = BaseHook.get_connection(self.github_conn_id)
-        config = cfg.Config(
-            githup_token=github_conn.password,
-            repo_name=github_conn.extra_dejson.get('repo_name'),
-            repo_branch=github_conn.extra_dejson.get('repo_branch'),
-            subfolder=github_conn.extra_dejson.get('subfolder')
-        )
-
-        ts = get_tableau_server(self.tableau_conn_id)
-        expected_conn_attrs = self.__set_connection_attributes()
-
-        # Get the ID for each datasource in the config
-        for ds in ts.get.datasources():
-            if ds not in config.datasources:
-                continue
-            config.datasources[ds].id = ds.id
-
-        for datasource in config.datasources:
-            if not datasource.id:
-                logging.error('!! Datasource not found in Tableau Online: %s / %s',
-                              datasource.project_name, datasource.name)
-                continue
-            dsid = datasource.id
+        # github_conn = BaseHook.get_connection(self.github_conn_id)
+        # config = cfg.Config(
+        #     githup_token=github_conn.password,
+        #     repo_name=github_conn.extra_dejson.get('repo_name'),
+        #     repo_branch=github_conn.extra_dejson.get('repo_branch'),
+        #     subfolder=github_conn.extra_dejson.get('subfolder')
+        # )
+        #
+        # ts = get_tableau_server(self.tableau_conn_id)
+        # expected_conn_attrs = self.__set_connection_attributes()
+        #
+        # # Get the ID for each datasource in the config
+        # for ds in ts.get.datasources():
+        #     if ds not in config.datasources:
+        #         continue
+        #     config.datasources[ds].id = ds.id
+        #
+        # # for datasource in config.datasources:
+        #     if not datasource.id:
+        #         logging.error('!! Datasource not found in Tableau Online: %s / %s',
+        #                       datasource.project_name, datasource.name)
+        #         continue
+        #     dsid = datasource.id
             # Set default dict attributes for tasks, for each datasource
             self.tasks[dsid] = {a: [] for a in UPDATE_ACTIONS}
             self.tasks[dsid]['project'] = datasource.project_name
             self.tasks[dsid]['datasource_name'] = datasource.name
-            if not config.in_maintenance_window and AIRFLOW_ENV not in ['STAGING', 'DEV']:
-                self.tasks[dsid]['skip'] = 'Outside maintenance window'
-                logging.info('(SKIP) Outside maintenance window: %s', datasource.name)
-                continue
-            elif datasource.name in EXCLUDED_DATASOURCES:
-                self.tasks[dsid]['skip'] = 'Marked to exclude'
-                logging.info('(SKIP) Marked to exclude: %s', datasource.name)
-                continue
-            logging.info('Checking Datasource: %s', datasource.name)
-            # Download the Datasource for comparison
-            dl_path = f"downloads/{dsid}/"
-            os.makedirs(dl_path, exist_ok=True)
-            ds_path = ts.download.datasource(dsid, file_dir=dl_path, include_extract=False)
-            tds = Datasource(ds_path)
-            # Cleanup downloaded file after assigning the Datasource
-            shutil.rmtree(dl_path, ignore_errors=True)
-            # Add connection task, if there is a difference
-            self.__compare_connection(dsid, datasource.name, tds.connection, expected_conn_attrs)
-            # Add folder tasks, if folders need to be added/deleted
-            self.__compare_folders(dsid, tds.folders_common, datasource.folders)
-            # Add Column tasks, if there are missing columns, or columns need to be updated
+            # if not config.in_maintenance_window and AIRFLOW_ENV not in ['STAGING', 'DEV']:
+            #     self.tasks[dsid]['skip'] = 'Outside maintenance window'
+            #     logging.info('(SKIP) Outside maintenance window: %s', datasource.name)
+            #     continue
+            # elif datasource.name in EXCLUDED_DATASOURCES:
+            #     self.tasks[dsid]['skip'] = 'Marked to exclude'
+            #     logging.info('(SKIP) Marked to exclude: %s', datasource.name)
+            #     continue
+            # logging.info('Checking Datasource: %s', datasource.name)
+            # # Download the Datasource for comparison
+            # dl_path = f"downloads/{dsid}/"
+            # os.makedirs(dl_path, exist_ok=True)
+            # ds_path = ts.download.datasource(dsid, file_dir=dl_path, include_extract=False)
+            # tds = Datasource(ds_path)
+            # # Cleanup downloaded file after assigning the Datasource
+            # shutil.rmtree(dl_path, ignore_errors=True)
+            # # Add connection task, if there is a difference
+            # self.__compare_connection(dsid, datasource.name, tds.connection, expected_conn_attrs)
+            # # Add folder tasks, if folders need to be added/deleted
+            # self.__compare_folders(dsid, tds.folders_common, datasource.folders)
+            # # Add Column tasks, if there are missing columns, or columns need to be updated
             for column in datasource.columns:
                 # Check if the column metadata needs to be updated
                 self.__compare_column_metadata(dsid, tds, column)

tableau_utilities/scripts/apply_configs.py

@@ -6,15 +6,17 @@
 from tableau_utilities.scripts.datasource import add_metadata_records_as_columns
 
 class ApplyConfigs:
-    def __init__(self, datasource_name, datasource_path, column_config, calculated_column_config, debugging_logs):
+    """ Applies a set of configs to a datasource.  Configs prefixed with target_ will be applied to the datasource.
+    Configs prefixed with datasource_ represent the current state of the datasource before changes.
+
+    """
+    def __init__(self, datasource_name, datasource_path, target_column_config, target_calculated_column_config, debugging_logs):
         self.datasource_name = datasource_name
         self.datasource_path = datasource_path
-        self.column_config = column_config
-        self.calculated_column_config = calculated_column_config
+        self.target_column_config = target_column_config
+        self.target_calculated_column_config = target_calculated_column_config
         self.debugging_logs = debugging_logs
 
-
-
     def invert_config(self, config):
         """ Helper function to invert the column config and calc config.
             Output -> {datasource: {column: info}}
@@ -97,7 +99,8 @@ def execute_changes(self, column_config, calculated_field_config, datasource):
         pass
 
     def apply_config_to_datasource(self):
-        """ Applies changes to make
+        """ Applies a set of configs (column_config and calculated_column_config) to a datasource.
+        If a column is in a datasource but NOT in the config that column will be unchanged.
 
         Args:
             datasource_name:
@@ -119,18 +122,15 @@ def apply_config_to_datasource(self):
         datasource = add_metadata_records_as_columns(datasource, self.debugging_logs)
 
         # Build the config dictionaries from the datasource
-        datasource_current_column_config, datasource_current_calculated_column_config = build_configs(datasource,
-                                                                                                      self.datasource_name)
+        datasource_column_config, datasource_calculated_column_config = build_configs(datasource, self.datasource_name)
         # Prepare the configs by inverting, combining and removing configs for other datasources
-        config = self.prepare_configs()
-        current_datasource_config = self.prepare_configs()
+        target_config = self.prepare_configs(self.target_column_config, self.target_calculated_column_config)
+        datasource_config = self.prepare_configs(datasource_column_config, datasource_calculated_column_config)
 
         # datasource = self.invert_config(self.column_config)
         # self.invert_config(self.calculated_column_config)
         # combined_config = {**dict1, **dict2}
 
-
-
         # Get the changes to make for the column config
         # Get the changes to make for the calculation config
 
@@ -148,9 +148,9 @@ def apply_configs(args):
     debugging_logs = args.debugging_logs
     datasource_name = args.name
     datasource_path = args.file_path
-    column_config = args.column_config
-    calculated_column_config = args.calculated_column_config
+    target_column_config = args.column_config
+    target_calculated_column_config = args.calculated_column_config
 
-    AC = ApplyConfigs(datasource_name, datasource_path, column_config, calculated_column_config, debugging_logs)
+    AC = ApplyConfigs(datasource_name, datasource_path, target_column_config, target_calculated_column_config, debugging_logs)
 
     AC.apply_config_to_datasource()