added base code

datarootsio · Sep 29, 2023 · 844f445 · 844f445
1 parent 3b9ccd6
commit 844f445
Show file tree

Hide file tree

Showing 8 changed files with 687 additions and 155 deletions.
diff --git a/TODO.md b/TODO.md
@@ -7,4 +7,17 @@
 - [ ] Add basic example
 - [ ] Add some simple tests
 - [ ] Feature: Test after Run
-- [ ] Feature: Run subset of models based on graph_operator param
+- [ ] Feature: Run subset of models based on graph_operator param
+
+
+## Tasks
+
+- [ ] . prefect_dbt_flow
+- [ ] +--- __init__.py
+- [ ] +--- cmd.py
+- [ ] +--- flow.py
+- [ ] +--- dbt
+- [ ] |    +--- __init__.py
+- [ ] |    +--- graph.py
+- [ ] |    +--- tasks.py
+
diff --git a/examples/logs/dbt.log b/examples/logs/dbt.log
@@ -0,0 +1,85 @@
+[0m11:28:43.851031 [debug] [MainThread]: Sending event: {'category': 'dbt', 'action': 'invocation', 'label': 'start', 'context': [<snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x105decd10>, <snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x105dea090>, <snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x105dea890>]}
+
+
+============================== 11:28:43.853768 | 831640ac-eb7c-479d-9f7e-476087584df8 ==============================
+[0m11:28:43.853768 [info ] [MainThread]: Running with dbt=1.6.4
+[0m11:28:43.854089 [debug] [MainThread]: running dbt with arguments {'printer_width': '80', 'indirect_selection': 'eager', 'log_cache_events': 'False', 'write_json': 'True', 'partial_parse': 'True', 'cache_selected_only': 'False', 'warn_error': 'None', 'version_check': 'True', 'debug': 'False', 'log_path': 'logs', 'fail_fast': 'False', 'profiles_dir': '/Users/dvaldez/.dbt', 'use_colors': 'True', 'use_experimental_parser': 'False', 'no_print': 'None', 'quiet': 'False', 'warn_error_options': 'WarnErrorOptions(include=[], exclude=[])', 'introspect': 'True', 'static_parser': 'True', 'invocation_command': 'dbt init example', 'target_path': 'None', 'log_format': 'default', 'send_anonymous_usage_stats': 'True'}
+[0m11:28:43.854652 [debug] [MainThread]: Starter project path: /Users/dvaldez/Library/Caches/pypoetry/virtualenvs/prefect-dbt-flow-vM9PP2bi-py3.11/lib/python3.11/site-packages/dbt/include/starter_project
+[0m11:28:43.857776 [info ] [MainThread]: 
+Your new dbt project "example" was created!
+
+For more information on how to configure the profiles.yml file,
+please consult the dbt documentation here:
+
+  https://docs.getdbt.com/docs/configure-your-profile
+
+One more thing:
+
+Need help? Don't hesitate to reach out to us via GitHub issues or on Slack:
+
+  https://community.getdbt.com/
+
+Happy modeling!
+
+[0m11:28:43.858014 [info ] [MainThread]: Setting up your profile.
+[0m11:28:43.858487 [error] [MainThread]: Encountered an error:
+while scanning for the next token
+found character '\t' that cannot start any token
+  in "/Users/dvaldez/.dbt/profiles.yml", line 2, column 1
+[0m11:28:43.861446 [error] [MainThread]: Traceback (most recent call last):
+  File "/Users/dvaldez/Library/Caches/pypoetry/virtualenvs/prefect-dbt-flow-vM9PP2bi-py3.11/lib/python3.11/site-packages/dbt/cli/requires.py", line 87, in wrapper
+    result, success = func(*args, **kwargs)
+                      ^^^^^^^^^^^^^^^^^^^^^
+  File "/Users/dvaldez/Library/Caches/pypoetry/virtualenvs/prefect-dbt-flow-vM9PP2bi-py3.11/lib/python3.11/site-packages/dbt/cli/requires.py", line 72, in wrapper
+    return func(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^
+  File "/Users/dvaldez/Library/Caches/pypoetry/virtualenvs/prefect-dbt-flow-vM9PP2bi-py3.11/lib/python3.11/site-packages/dbt/cli/main.py", line 469, in init
+    results = task.run()
+              ^^^^^^^^^^
+  File "/Users/dvaldez/Library/Caches/pypoetry/virtualenvs/prefect-dbt-flow-vM9PP2bi-py3.11/lib/python3.11/site-packages/dbt/task/init.py", line 294, in run
+    if not self.check_if_can_write_profile(profile_name=profile_name):
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/Users/dvaldez/Library/Caches/pypoetry/virtualenvs/prefect-dbt-flow-vM9PP2bi-py3.11/lib/python3.11/site-packages/dbt/task/init.py", line 200, in check_if_can_write_profile
+    profiles = yaml.safe_load(f) or {}
+               ^^^^^^^^^^^^^^^^^
+  File "/Users/dvaldez/Library/Caches/pypoetry/virtualenvs/prefect-dbt-flow-vM9PP2bi-py3.11/lib/python3.11/site-packages/yaml/__init__.py", line 125, in safe_load
+    return load(stream, SafeLoader)
+           ^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/Users/dvaldez/Library/Caches/pypoetry/virtualenvs/prefect-dbt-flow-vM9PP2bi-py3.11/lib/python3.11/site-packages/yaml/__init__.py", line 81, in load
+    return loader.get_single_data()
+           ^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/Users/dvaldez/Library/Caches/pypoetry/virtualenvs/prefect-dbt-flow-vM9PP2bi-py3.11/lib/python3.11/site-packages/yaml/constructor.py", line 49, in get_single_data
+    node = self.get_single_node()
+           ^^^^^^^^^^^^^^^^^^^^^^
+  File "/Users/dvaldez/Library/Caches/pypoetry/virtualenvs/prefect-dbt-flow-vM9PP2bi-py3.11/lib/python3.11/site-packages/yaml/composer.py", line 36, in get_single_node
+    document = self.compose_document()
+               ^^^^^^^^^^^^^^^^^^^^^^^
+  File "/Users/dvaldez/Library/Caches/pypoetry/virtualenvs/prefect-dbt-flow-vM9PP2bi-py3.11/lib/python3.11/site-packages/yaml/composer.py", line 55, in compose_document
+    node = self.compose_node(None, None)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/Users/dvaldez/Library/Caches/pypoetry/virtualenvs/prefect-dbt-flow-vM9PP2bi-py3.11/lib/python3.11/site-packages/yaml/composer.py", line 84, in compose_node
+    node = self.compose_mapping_node(anchor)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/Users/dvaldez/Library/Caches/pypoetry/virtualenvs/prefect-dbt-flow-vM9PP2bi-py3.11/lib/python3.11/site-packages/yaml/composer.py", line 133, in compose_mapping_node
+    item_value = self.compose_node(node, item_key)
+                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/Users/dvaldez/Library/Caches/pypoetry/virtualenvs/prefect-dbt-flow-vM9PP2bi-py3.11/lib/python3.11/site-packages/yaml/composer.py", line 64, in compose_node
+    if self.check_event(AliasEvent):
+       ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/Users/dvaldez/Library/Caches/pypoetry/virtualenvs/prefect-dbt-flow-vM9PP2bi-py3.11/lib/python3.11/site-packages/yaml/parser.py", line 98, in check_event
+    self.current_event = self.state()
+                         ^^^^^^^^^^^^
+  File "/Users/dvaldez/Library/Caches/pypoetry/virtualenvs/prefect-dbt-flow-vM9PP2bi-py3.11/lib/python3.11/site-packages/yaml/parser.py", line 449, in parse_block_mapping_value
+    if not self.check_token(KeyToken, ValueToken, BlockEndToken):
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/Users/dvaldez/Library/Caches/pypoetry/virtualenvs/prefect-dbt-flow-vM9PP2bi-py3.11/lib/python3.11/site-packages/yaml/scanner.py", line 116, in check_token
+    self.fetch_more_tokens()
+  File "/Users/dvaldez/Library/Caches/pypoetry/virtualenvs/prefect-dbt-flow-vM9PP2bi-py3.11/lib/python3.11/site-packages/yaml/scanner.py", line 258, in fetch_more_tokens
+    raise ScannerError("while scanning for the next token", None,
+yaml.scanner.ScannerError: while scanning for the next token
+found character '\t' that cannot start any token
+  in "/Users/dvaldez/.dbt/profiles.yml", line 2, column 1
+
+[0m11:28:43.862094 [debug] [MainThread]: Command `dbt init` failed at 11:28:43.862028 after 0.02 seconds
+[0m11:28:43.862299 [debug] [MainThread]: Sending event: {'category': 'dbt', 'action': 'invocation', 'label': 'end', 'context': [<snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x10564c350>, <snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x103dcb790>, <snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x100feced0>]}
+[0m11:28:43.862475 [debug] [MainThread]: Flushing usage events