WIP on support for additional storage backends

Includes provisional support for mysql, postgres, sqlite, and viruoso.  Needs documentation and testing.

DESCRIPTION

@@ -1,10 +1,16 @@
 Package: rdflib
 Title: Tools to Manipulate and Query Semantic Data
-Version: 0.1.0
-Authors@R: person("Carl", "Boettiger", 
+Version: 0.1.1
+Authors@R: c(person("Carl", "Boettiger", 
                   email = "cboettig@gmail.com", 
                   role = c("aut", "cre", "cph"),
-                  comment=c(ORCID = "0000-0002-1642-628X"))
+                  comment=c(ORCID = "0000-0002-1642-628X")),
+             person("Bryce", "Mecum", 
+                  role = "rev",
+                  comment=c(ORCID = "0000-0002-0381-3766")),
+             person("Anna", "Krystalli", 
+                  role = "rev",
+                  comment=c(ORCID = "0000-0002-2378-4915")))                  
 Description: The Resource Description Framework, or 'RDF' is a widely used
              data representation model that forms the cornerstone of the 
              Semantic Web. 'RDF' represents data as a graph rather than 

R/rdf.R

@@ -34,34 +34,101 @@
 #' @examples
 #' x <- rdf()
 #' 
-rdf <- function(storage = c("memory", "BDB"), path = ".", new_db = FALSE){
-  world <- new("World")
+rdf <- function(storage = c("memory", "BDB", "sqlite", 
+                            "postgres", "mysql", "virtuoso"), 
+                path = ".", 
+                new_db = FALSE){
 
+  world <- new("World")
+  store <- rdf_storage(storage, world, dir = ".", new_db = FALSE)
+  model <- new("Model", world = world, storage = store, options = "")
+  structure(list(world = world, model = model, storage = store),
+            class = "rdf")
+}
+
+
+
+
+rdf_storage <- function(storage = c("memory", 
+                                    "BDB", 
+                                    "sqlite", 
+                                    "postgres", 
+                                    "mysql", 
+                                    "virtuoso"),
+                        world = new("World"),
+                        host = NULL,
+                        port = NULL,
+                        user = NULL,
+                        password = NULL,
+                        database = NULL,
+                        charset = NULL,
+                        dir = NULL,
+                        dsn = "Local Virtuoso",
+                        name = "rdflib",
+                        new_db = FALSE,
+                        fallback = TRUE){
 
-  ## Handle storage type
   storage <- match.arg(storage)
-  if(storage == "BDB"){
-    if(rdf_has_bdb()){
-      ## Store in Berkeley DB
-      if(new_db){
-        options <- paste0("new='yes',hash-type='bdb',dir='", path, "'") 
-      } else {
-        options <- paste0("hash-type='bdb',dir='", path, "'") 
-      }
-    } else {
-      warning("BDB driver not found. Falling back on in-memory storage")
-      options <- "hash-type='memory'"
-    }
-  } else { ## Store in memory
-   options <- "hash-type='memory'"
+
+  new <- NULL
+  if(new_db){
+    new <- "yes"
+  }
+  if(is.null(dir)){
+    dir <- "."
   }
-  storage <- new("Storage", world, "hashes", name = "rdflib", 
-                 options = options)
 
+  options <- options_to_str(
+    switch(storage,
+      memory = list("hash-type" = "memory"),
+         BDB = list(new = new, "hash-type" = "bdb", dir = dir),  
+      sqlite = list(new = new, dir = dir),
+    postgres = list(new = new, host = host, port = port,
+                    database = database, user = user, password = password),
+       mysql = list(new = new, host = host, port = port,
+                    database = database, user = user, password = password),
+    virtuoso = list(dsn = dsn, user = user, password = password, 
+                    database = database, host = host, charset = charset),
+    list()
+    ))
 
-  model <- new("Model", world = world, storage, options = "")
-  structure(list(world = world, model = model, storage = storage),
-            class = "rdf")
+ store <- switch(storage,
+    memory = new("Storage", world, "hashes", name = name, options = options),
+    BDB = new("Storage", world, "hashes", name = name, options = options),
+    sqlite = new("Storage", world, "sqlite", name = name, options = options),
+    postgres = new("Storage", world, "postgresql", name = name, options = options),
+    mysql =  new("Storage", world, "mysql", name = name, options = options),
+    virtuoso =  new("Storage", world, "virtuoso", name = name, options = options)
+  ) 
+
+ continue <- !(utils::capture.output(
+   base::print.default(
+     store@librdf_storage@ref)) == 
+     "<pointer: 0x0>")
+
+  if(!continue){
+    if(fallback){
+       warning(paste(storage, "driver not found. Falling back on in-memory storage"))
+       redland::freeStorage(store)
+       store <- new("Storage", world)
+     } else {
+       stop(paste(storage, "not found"))
+     }
+  }
+
+  store
+}
+
+
+compact <- function(l){ Filter(Negate(is.null), l)}
+options_to_str <- function(x){
+  x <- compact(x)
+  n <- names(x)
+  out <- character(0)
+  for(i in seq_along(x)){
+    out <- paste0(c(out, paste0(n[[i]], "=", "'", x[[i]], "'")), collapse=",")
+  }
+  out
 }
 
 

R/rdf_has_bdb.R

@@ -11,19 +11,54 @@
 #' @examples 
 #' rdf_has_bdb()
 rdf_has_bdb <- function(){
+ rdf_has_backend("BDB")
+}
+
+
+
+rdf_has_backend <- function(storage = c("memory", 
+                                     "BDB", 
+                                     "sqlite", 
+                                     "postgres", 
+                                     "mysql", 
+                                     "virtuoso"),
+                            host = NULL, 
+                            user = NULL,
+                            password = NULL,
+                            port = NULL){
+  storage <- match.arg(storage)
+  if(storage == "memory"){
+    return(TRUE)
+  }
+
   ## Unfortunately convoluted way to check if we have Berkeley DB Support
   world <- new("World")
-  path <-tempdir()
-  options <- paste0("new='yes',hash-type='bdb',dir='", path, "'")
-  storage <- new("Storage", world, "hashes", name = "rdflib", 
-                 options = options)
+
+  store <- switch(storage,
+    BDB      = new("Storage", world, "hashes", name = "rdflib", 
+                   options = paste0("new='yes',hash-type='bdb',dir='",
+                                  tempdir(), "'")),
+    sqlite   = new("Storage", world, "sqlite", name = "sqlite1",
+                   options = "new='yes'"),
+    postgres = new("Storage", world, "postgresql", name = "db",
+                   options = paste0("new='yes',host='", host, "',", 
+                                    "user='", user, 
+                                    "','password='", password, "'")),
+    mysql = new("Storage", world, "mysql", name = "db",
+                options = paste0("new='yes',host='", host, "',", 
+                                 "user='", user,
+                                 "','password='", password, "'")),
+    virtuoso = new("Storage", world, "virtuoso", name = "db1",
+                   options = "dsn='Local Virtuoso',user='", 
+                   user, "','password='", password, "'")
+  )
 
   out <- !(utils::capture.output(
     base::print.default(
-      storage@librdf_storage@ref)) == 
+      store@librdf_storage@ref)) == 
       "<pointer: 0x0>")
 
-  redland::freeStorage(storage)
+  redland::freeStorage(store)
   redland::freeWorld(world)
 
   out

inst/docker/docker-compose.yml

@@ -0,0 +1,19 @@
+version: '3'
+services:
+  mariadb:
+      image: mariadb
+      environment:
+          - MYSQL_ROOT_PASSWORD=rdflib
+  postgres:
+      image: postgres
+      environment:
+          - POSTGRES_PASSWORD=rdflib
+  rdf:
+    image: rdf
+    links: 
+      - postgres
+      - mariadb
+
+
+
+

inst/examples/storage_types.R

@@ -15,7 +15,7 @@ sqlite_storage <- new("Storage", world, "sqlite", name = "sqlite1", options = "n
 ## POSTGRES
 ## Needs postgres backend running
 postgres_storage <- new("Storage", world, "postgresql", name = "postgres1", 
-    options = "new='yes',host='localhost',database='red',user='foo','password='bar'")
+    options = "new='yes',host='postgres',user='postgres','password='secret'")
 
 ## MYSQL
 ## Needs mysql backend running

tests/testthat/test-rdf.R

@@ -23,40 +23,6 @@ testthat::test_that("We can initialize and free rdf objects", {
   testthat::expect_false("RDF_graph" %in% ls())
 })
 
-testthat::test_that("We warn if we cannot use disk-based storage", {
-  testthat::skip_if(rdf_has_bdb())
-  testthat::expect_warning(rdf <- rdf(storage = "BDB"), "BDB driver not found")
-  ## Falls back on memory-based storage, still creates rdf
-  testthat::expect_is(rdf, "rdf")
-  rdf_free(rdf)
-
-})
-
-testthat::test_that("We can use BDB storage", {
-  testthat::skip_if_not(rdf_has_bdb())
-
-  # not sure why this is now failing on appveyor
-  testthat::skip_on_os("windows")
-
-
-  testthat::expect_silent(rdf <- rdf(storage="BDB", new_db = TRUE))
-
-  rdf_add(rdf, "", "dc:name", "bob")
-  expect_match(format(rdf, "nquads"), "bob")
-  testthat::expect_is(rdf, "rdf")
-  rdf_free(rdf)
-
-  ## We can reconnect to disk based storage after freeing
-  rdf2 <- rdf(storage = "BDB", new_db = FALSE)
-  expect_match(format(rdf2, "nquads"), "bob")
-  rdf_free(rdf2)
-
-  unlink("rdflib-po2s.db")
-  unlink("rdflib-so2p.db")
-  unlink("rdflib-sp2o.db")
-})
-
-
 testthat::test_that("we can concatenate rdfs", {
   rdf1 <- rdf_parse(system.file("extdata/ex.xml", package = "rdflib"))
   rdf2 <- rdf_parse(system.file("extdata/ex2.xml", package = "rdflib"))

tests/testthat/test-rdf_storage.R

@@ -0,0 +1,41 @@
+testthat::context("RDF Storage")
+
+rdf_storage()
+rdf_storage("BDB")
+rdf_storage("sqlite")
+
+
+
+testthat::test_that("We warn if we cannot use disk-based storage", {
+  testthat::skip_if(rdf_has_bdb())
+  testthat::expect_warning(rdf <- rdf(storage = "BDB"), "BDB driver not found")
+  ## Falls back on memory-based storage, still creates rdf
+  testthat::expect_is(rdf, "rdf")
+  rdf_free(rdf)
+
+})
+
+testthat::test_that("We can use BDB storage", {
+  testthat::skip_if_not(rdf_has_bdb())
+
+  # not sure why this is now failing on appveyor
+  testthat::skip_on_os("windows")
+
+
+  testthat::expect_silent(rdf <- rdf(storage="BDB", new_db = TRUE))
+
+  rdf_add(rdf, "", "dc:name", "bob")
+  expect_match(format(rdf, "nquads"), "bob")
+  testthat::expect_is(rdf, "rdf")
+  rdf_free(rdf)
+
+  ## We can reconnect to disk based storage after freeing
+  rdf2 <- rdf(storage = "BDB", new_db = FALSE)
+  expect_match(format(rdf2, "nquads"), "bob")
+  rdf_free(rdf2)
+
+  unlink("rdflib-po2s.db")
+  unlink("rdflib-so2p.db")
+  unlink("rdflib-sp2o.db")
+})
+