Skip to content

Commit

Permalink
Images, kamu-base-with-data-mt: init-workspace.py use .kamuconfig
Browse files Browse the repository at this point in the history
  • Loading branch information
s373r committed Oct 8, 2024
1 parent de723b7 commit 35d53f5
Showing 1 changed file with 15 additions and 8 deletions.
23 changes: 15 additions & 8 deletions images/kamu-base-with-data-mt/init-workspace.py
Original file line number Diff line number Diff line change
@@ -1,48 +1,55 @@
#!/usr/bin/env python

import os
import sys
import shutil
import subprocess
from pathlib import Path

###############################################################################

CURRENT_PATH = Path(__file__).resolve().parent
S3_REPO_URL = "s3://datasets.kamu.dev/odf/v2/example-mt/"


###############################################################################

def s3_listdir(url):
return [
line.strip().split(' ')[1]
for line in subprocess.run(
f"aws s3 ls {url}",
shell=True,
f"aws s3 ls {url}",
shell=True,
text=True,
check=True,
capture_output=True,
).stdout.splitlines()
]


def s3_cat(url):
return subprocess.run(
f"aws s3 cp {url} -",
shell=True,
f"aws s3 cp {url} -",
shell=True,
text=True,
check=True,
capture_output=True,
).stdout.strip()


###############################################################################

subprocess.run(
"kamu init --multi-tenant --exists-ok",
"kamu init --multi-tenant --exists-ok",
shell=True,
check=True,
)

for did in s3_listdir(S3_REPO_URL):
shutil.copy(CURRENT_PATH / "extra/.kamuconfig", ".kamuconfig")

for did in s3_listdir(S3_REPO_URL)[:1]:
url = S3_REPO_URL + did
alias = s3_cat(f"{S3_REPO_URL}{did}info/alias")
account, name = alias.split('/', 1)

subprocess.run(
f"kamu --account {account} pull --no-alias {url} --as {name}",
shell=True,
Expand Down

0 comments on commit 35d53f5

Please sign in to comment.