Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[ZEPPELIN-5993] Fix GCSNotebookRepoTests #4711

Merged
merged 1 commit into from
Feb 12, 2024
Merged

[ZEPPELIN-5993] Fix GCSNotebookRepoTests #4711

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 8 additions & 1 deletion ...ins/notebookrepo/gcs/src/main/java/org/apache/zeppelin/notebook/repo/GCSNotebookRepo.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,14 @@ public GCSNotebookRepo() {

@VisibleForTesting
public GCSNotebookRepo(ZeppelinConfiguration zConf, Storage storage) throws IOException {
init(zConf);
try {
init(zConf);
} catch (IOException e) {
// Skip Credentials Exception during tests
if (!e.getMessage().contains("Default Credentials")) {
throw e;
}
}
this.storage = storage;
}

Expand Down
72 changes: 41 additions & 31 deletions ...notebookrepo/gcs/src/test/java/org/apache/zeppelin/notebook/repo/GCSNotebookRepoTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
import com.google.cloud.storage.contrib.nio.testing.LocalStorageHelper;
import com.google.common.collect.ImmutableMap;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
Expand All @@ -41,17 +42,16 @@
import org.apache.zeppelin.notebook.Paragraph;
import org.apache.zeppelin.scheduler.Job.Status;
import org.apache.zeppelin.user.AuthenticationInfo;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.Arguments;
import org.junit.jupiter.params.provider.MethodSource;

//TODO(zjffdu) This test fails due to some changes in google, need to fix
@Disabled
class GCSNotebookRepoTest {
private static final AuthenticationInfo AUTH_INFO = AuthenticationInfo.ANONYMOUS;
private static final String DEFAULT_URL = "gs://bucketname";

private GCSNotebookRepo notebookRepo;
private Storage storage;
Expand All @@ -73,7 +73,15 @@ void setUp() throws Exception {
this.storage = LocalStorageHelper.getOptions().getService();
}

@AfterEach
void tearDown() {
if (notebookRepo != null) {
notebookRepo.close();
}
}

private static Note makeRunningNote() {

Note note = new Note();
note.setPath("/test_note");
note.setConfig(ImmutableMap.<String, Object>of("key", "value"));
Expand Down Expand Up @@ -117,10 +125,11 @@ void testList(String bucketName, Optional<String> basePath, String uriPath) thro

@Test
void testGet_nonexistent() throws Exception {
try {
System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_GCS_STORAGE_DIR.getVarName(), DEFAULT_URL);
this.notebookRepo = new GCSNotebookRepo(ZeppelinConfiguration.create(), storage);
assertThrows(IOException.class, () -> {
notebookRepo.get("id", "", AUTH_INFO);
fail();
} catch (IOException e) {}
});
}

@ParameterizedTest
Expand All @@ -145,10 +154,9 @@ void testGet_malformed(String bucketName, Optional<String> basePath, String uriP
System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_GCS_STORAGE_DIR.getVarName(), uriPath);
this.notebookRepo = new GCSNotebookRepo(ZeppelinConfiguration.create(), storage);
createMalformed("id", "/name", bucketName, basePath);
try {
assertThrows(IOException.class, () -> {
notebookRepo.get("id", "/name", AUTH_INFO);
fail();
} catch (IOException e) {}
});
}

@ParameterizedTest
Expand Down Expand Up @@ -177,10 +185,11 @@ void testSave_update(String bucketName, Optional<String> basePath, String uriPat

@Test
void testRemove_nonexistent() throws Exception {
try {
System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_GCS_STORAGE_DIR.getVarName(), DEFAULT_URL);
this.notebookRepo = new GCSNotebookRepo(ZeppelinConfiguration.create(), storage);
assertThrows(IOException.class, () -> {
notebookRepo.remove("id", "/name", AUTH_INFO);
fail();
} catch (IOException e) {}
});
}

@ParameterizedTest
Expand All @@ -195,11 +204,12 @@ void testRemove(String bucketName, Optional<String> basePath, String uriPath) th

@Test
void testRemoveFolder_nonexistent() throws Exception {
System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_GCS_STORAGE_DIR.getVarName(), DEFAULT_URL);
this.notebookRepo = new GCSNotebookRepo(ZeppelinConfiguration.create(), storage);
assertThrows(IOException.class, () -> {
notebookRepo.remove("id", "/name", AUTH_INFO);
fail();
});

}

@ParameterizedTest
Expand All @@ -220,11 +230,12 @@ void testRemoveFolder(String bucketName, Optional<String> basePath, String uriPa


@Test
void testMove_nonexistent() {
try {
void testMove_nonexistent() throws IOException {
System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_GCS_STORAGE_DIR.getVarName(), DEFAULT_URL);
this.notebookRepo = new GCSNotebookRepo(ZeppelinConfiguration.create(), storage);
assertThrows(IOException.class, () -> {
notebookRepo.move("id", "/name", "/name_new", AUTH_INFO);
fail();
} catch (IOException e) {}
});
}

@ParameterizedTest
Expand All @@ -239,9 +250,10 @@ void testMove(String bucketName, Optional<String> basePath, String uriPath) thro

@Test
void testMoveFolder_nonexistent() throws Exception {
System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_GCS_STORAGE_DIR.getVarName(), DEFAULT_URL);
this.notebookRepo = new GCSNotebookRepo(ZeppelinConfiguration.create(), storage);
assertThrows(IOException.class, () -> {
notebookRepo.move("/name", "/name_new", AUTH_INFO);
fail();
});
}

Expand Down Expand Up @@ -282,40 +294,38 @@ private BlobId makeBlobId(String noteId, String notePath, String bucketName, Opt
private void createAt(Note note, String relativePath, String bucketName, Optional<String> basePath) throws IOException {
BlobId id = BlobId.of(bucketName, makeName(relativePath, basePath));
BlobInfo info = BlobInfo.newBuilder(id).setContentType("application/json").build();
storage.create(info, note.toJson().getBytes("UTF-8"));
storage.create(info, note.toJson().getBytes(StandardCharsets.UTF_8));
}

private void create(Note note, String bucketName, Optional<String> basePath) throws IOException {
BlobInfo info = BlobInfo.newBuilder(makeBlobId(note.getId(), note.getPath(), bucketName, basePath))
.setContentType("application/json")
.build();
storage.create(info, note.toJson().getBytes("UTF-8"));
storage.create(info, note.toJson().getBytes(StandardCharsets.UTF_8));
}

private void createMalformed(String noteId, String notePath, String bucketName, Optional<String> basePath) throws IOException {
BlobInfo info = BlobInfo.newBuilder(makeBlobId(noteId, notePath, bucketName, basePath))
.setContentType("application/json")
.build();
storage.create(info, "{ invalid-json }".getBytes("UTF-8"));
storage.create(info, "{ invalid-json }".getBytes(StandardCharsets.UTF_8));
}

/* These tests test path parsing for illegal paths, and do not use the parameterized vars */

@Test
void testInitialization_pathNotSet() throws Exception {
try {
System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_GCS_STORAGE_DIR.getVarName(), "");
new GCSNotebookRepo(ZeppelinConfiguration.create(), storage);
fail();
} catch (IOException e) {}
System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_GCS_STORAGE_DIR.getVarName(), "");
assertThrows(IOException.class, () -> {
notebookRepo = new GCSNotebookRepo(ZeppelinConfiguration.create(), storage);
});
}

@Test
void testInitialization_malformedPath() throws Exception {
try {
System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_GCS_STORAGE_DIR.getVarName(), "foo");
new GCSNotebookRepo(ZeppelinConfiguration.create(), storage);
fail();
} catch (IOException e) {}
System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_GCS_STORAGE_DIR.getVarName(), "foo");
assertThrows(IOException.class, () -> {
notebookRepo = new GCSNotebookRepo(ZeppelinConfiguration.create(), storage);
});
}
}
Loading