Feat/ha/add catalog triples to meta models

pom.xml

@@ -14,7 +14,7 @@
     <parent>
       <groupId>org.springframework.boot</groupId>
       <artifactId>spring-boot-starter-parent</artifactId>
-      <version>3.2.3</version>
+      <version>3.2.4</version>
       <relativePath/>
     </parent>
 
@@ -122,13 +122,13 @@
         <dependency>
             <groupId>org.mockito.kotlin</groupId>
             <artifactId>mockito-kotlin</artifactId>
-            <version>5.2.1</version>
+            <version>5.3.1</version>
             <scope>test</scope>
         </dependency>
         <dependency>
             <groupId>org.wiremock</groupId>
             <artifactId>wiremock-standalone</artifactId>
-            <version>3.4.2</version>
+            <version>3.5.2</version>
             <scope>test</scope>
         </dependency>
         <dependency>
@@ -230,7 +230,7 @@
             <plugin>
                 <groupId>org.jacoco</groupId>
                 <artifactId>jacoco-maven-plugin</artifactId>
-                <version>0.8.11</version>
+                <version>0.8.12</version>
                 <executions>
                     <execution>
                         <id>before-unit-test-execution</id>

src/main/kotlin/no/digdir/informasjonsforvaltning/fdk_dataservice_harvester/service/UpdateService.kt

@@ -1,6 +1,7 @@
 package no.digdir.informasjonsforvaltning.fdk_dataservice_harvester.service
 
 import no.digdir.informasjonsforvaltning.fdk_dataservice_harvester.configuration.ApplicationProperties
+import no.digdir.informasjonsforvaltning.fdk_dataservice_harvester.harvester.extractCatalogModel
 import no.digdir.informasjonsforvaltning.fdk_dataservice_harvester.model.CatalogMeta
 import no.digdir.informasjonsforvaltning.fdk_dataservice_harvester.model.DataServiceMeta
 import no.digdir.informasjonsforvaltning.fdk_dataservice_harvester.rdf.addMetaPrefixes
@@ -46,34 +47,38 @@ class UpdateService(
     }
 
     fun updateMetaData() {
-        dataServiceRepository.findAll()
-            .forEach { dataService ->
-                val dataServiceMeta = dataService.createMetaModel()
-
-                turtleService.getDataService(dataService.fdkId, withRecords = false)
-                    ?.let { conceptNoRecords -> parseRDFResponse(conceptNoRecords, Lang.TURTLE, null) }
-                    ?.let { conceptModelNoRecords -> dataServiceMeta.union(conceptModelNoRecords) }
-                    ?.run { turtleService.saveAsDataService(this, fdkId = dataService.fdkId, withRecords = true) }
-            }
-
         catalogRepository.findAll()
             .forEach { catalog ->
+                val catalogMeta = catalog.createMetaModel()
+                val completeMetaModel = ModelFactory.createDefaultModel()
+                completeMetaModel.add(catalogMeta)
+
                 val catalogNoRecords = turtleService.getCatalog(catalog.fdkId, withRecords = false)
                     ?.let { parseRDFResponse(it, Lang.TURTLE, null) }
 
                 if (catalogNoRecords != null) {
                     val catalogURI = "${applicationProperties.catalogUri}/${catalog.fdkId}"
-                    val catalogMeta = catalog.createMetaModel()
+                    val catalogTriples = catalogNoRecords.getResource(catalog.uri)
+                        ?.extractCatalogModel()
+                    catalogTriples?.add(catalogMeta)
 
                     dataServiceRepository.findAllByIsPartOf(catalogURI)
                         .filter { it.modelContainsDataService(catalogNoRecords) }
                         .forEach { dataService ->
                             val serviceMetaModel = dataService.createMetaModel()
-                            catalogMeta.add(serviceMetaModel)
-                        }
+                            completeMetaModel.add(serviceMetaModel)
+
+                            turtleService.getDataService(dataService.fdkId, withRecords = false)
+                                ?.let { dataServiceNoRecords -> parseRDFResponse(dataServiceNoRecords, Lang.TURTLE, null) }
+                                ?.let { dataServiceModelNoRecords -> serviceMetaModel
+                                    .union(dataServiceModelNoRecords)
+                                    .union(catalogTriples)
+                                }
+                                ?.run { turtleService.saveAsDataService(this, fdkId = dataService.fdkId, withRecords = true) }
+                            }
 
                     turtleService.saveAsCatalog(
-                        catalogMeta.union(catalogNoRecords),
+                        completeMetaModel.union(catalogNoRecords),
                         fdkId = catalog.fdkId,
                         withRecords = true
                     )

src/main/resources/application.yml

@@ -49,15 +49,15 @@ application:
 spring:
   config.activate.on-profile: contract-test
   security.oauth2.resourceserver.jwt:
-    jwk-set-uri: http://localhost:5000/auth/realms/fdk/protocol/openid-connect/certs
-    issuer-uri: http://localhost:5000/auth/realms/fdk
+    jwk-set-uri: http://localhost:5050/auth/realms/fdk/protocol/openid-connect/certs
+    issuer-uri: http://localhost:5050/auth/realms/fdk
   rabbitmq:
     host: localhost
-    port: 5000
+    port: 5050
     username: admin
     password: admin
 application:
-  dataserviceUri: http://localhost:5000/dataservices
-  catalogUri: http://localhost:5000/catalogs
-  harvestAdminRootUrl: http://localhost:5000
+  dataserviceUri: http://localhost:5050/dataservices
+  catalogUri: http://localhost:5050/catalogs
+  harvestAdminRootUrl: http://localhost:5050
 server.port: 5555

src/test/kotlin/no/digdir/informasjonsforvaltning/fdk_dataservice_harvester/harvester/HarvesterTest.kt

@@ -47,9 +47,9 @@ class HarvesterTest {
             .thenReturn(Optional.of(DATA_SERVICE_DBO_0))
 
         whenever(valuesMock.catalogUri)
-            .thenReturn("http://localhost:5000/catalogs")
+            .thenReturn("http://localhost:5050/catalogs")
         whenever(valuesMock.dataserviceUri)
-            .thenReturn("http://localhost:5000/dataservices")
+            .thenReturn("http://localhost:5050/dataservices")
 
         val report = harvester.harvestDataServiceCatalog(TEST_HARVEST_SOURCE, TEST_HARVEST_DATE, false)
 
@@ -92,7 +92,7 @@ class HarvesterTest {
 
         val expectedReport = HarvestReport(
             id="harvest",
-            url="http://localhost:5000/harvest",
+            url="http://localhost:5050/harvest",
             dataType="dataservice",
             harvestError=false,
             startTime = "2020-03-12 12:52:16 +0100",
@@ -122,7 +122,7 @@ class HarvesterTest {
 
         val expectedReport = HarvestReport(
             id="harvest",
-            url="http://localhost:5000/harvest",
+            url="http://localhost:5050/harvest",
             dataType="dataservice",
             harvestError=false,
             startTime = "2020-03-12 12:52:16 +0100",
@@ -165,9 +165,9 @@ class HarvesterTest {
             .thenReturn(responseReader.readFile("harvest_response_catalog_diff.ttl"))
 
         whenever(valuesMock.catalogUri)
-            .thenReturn("http://localhost:5000/catalogs")
+            .thenReturn("http://localhost:5050/catalogs")
         whenever(valuesMock.dataserviceUri)
-            .thenReturn("http://localhost:5000/dataservices")
+            .thenReturn("http://localhost:5050/dataservices")
 
         whenever(catalogRepository.findById(CATALOG_DBO_0.uri))
             .thenReturn(Optional.of(CATALOG_DBO_0))
@@ -209,7 +209,7 @@ class HarvesterTest {
 
         val expectedReport = HarvestReport(
             id="harvest",
-            url="http://localhost:5000/harvest",
+            url="http://localhost:5050/harvest",
             dataType="dataservice",
             harvestError=false,
             startTime = "2020-07-12 13:52:16 +0200",
@@ -235,7 +235,7 @@ class HarvesterTest {
 
         val expectedReport = HarvestReport(
             id="harvest",
-            url="http://localhost:5000/harvest",
+            url="http://localhost:5050/harvest",
             dataType="dataservice",
             harvestError=true,
             startTime = "2020-03-12 12:52:16 +0100",
@@ -253,13 +253,13 @@ class HarvesterTest {
             .thenReturn(harvested)
         whenever(turtleService.getHarvestSource(TEST_HARVEST_SOURCE.url!!))
             .thenReturn(responseReader.readFile("harvest_response.ttl"))
-        whenever(dataServiceRepository.findAllByIsPartOf("http://localhost:5000/catalogs/$CATALOG_ID_0"))
+        whenever(dataServiceRepository.findAllByIsPartOf("http://localhost:5050/catalogs/$CATALOG_ID_0"))
             .thenReturn(listOf(DATA_SERVICE_DBO_0))
 
         whenever(valuesMock.catalogUri)
-            .thenReturn("http://localhost:5000/catalogs")
+            .thenReturn("http://localhost:5050/catalogs")
         whenever(valuesMock.dataserviceUri)
-            .thenReturn("http://localhost:5000/datasets")
+            .thenReturn("http://localhost:5050/datasets")
 
         val report = harvester.harvestDataServiceCatalog(TEST_HARVEST_SOURCE, TEST_HARVEST_DATE, false)
 
@@ -270,7 +270,7 @@ class HarvesterTest {
 
         val expectedReport = HarvestReport(
             id="harvest",
-            url="http://localhost:5000/harvest",
+            url="http://localhost:5050/harvest",
             dataType="dataservice",
             harvestError=false,
             startTime = "2020-03-12 12:52:16 +0100",

src/test/kotlin/no/digdir/informasjonsforvaltning/fdk_dataservice_harvester/service/UpdateServiceTest.kt

@@ -32,26 +32,29 @@ class UpdateServiceTest {
         @Test
         fun catalogRecordsIsRecreatedFromMetaDBO() {
             whenever(catalogRepository.findAll())
-                .thenReturn(listOf(CATALOG_DBO_0))
-            whenever(dataServiceRepository.findAll())
-                .thenReturn(listOf(DATA_SERVICE_DBO_0, DATA_SERVICE_DBO_1))
-            whenever(dataServiceRepository.findAllByIsPartOf("http://localhost:5000/catalogs/$CATALOG_ID_0"))
-                .thenReturn(listOf(DATA_SERVICE_DBO_0, DATA_SERVICE_DBO_1))
+                .thenReturn(listOf(CATALOG_DBO_0, CATALOG_DBO_1))
+            whenever(dataServiceRepository.findAllByIsPartOf("http://localhost:5050/catalogs/$CATALOG_ID_0"))
+                .thenReturn(listOf(DATA_SERVICE_DBO_0))
+            whenever(dataServiceRepository.findAllByIsPartOf("http://localhost:5050/catalogs/$CATALOG_ID_1"))
+                .thenReturn(listOf(DATA_SERVICE_DBO_1))
             whenever(turtleService.getCatalog(CATALOG_ID_0, false))
                 .thenReturn(responseReader.readFile("catalog_0_no_records.ttl"))
+            whenever(turtleService.getCatalog(CATALOG_ID_1, false))
+                .thenReturn(responseReader.readFile("catalog_1_no_records.ttl"))
             whenever(turtleService.getDataService(DATASERVICE_ID_0, false))
                 .thenReturn(responseReader.readFile("parsed_dataservice_0.ttl"))
             whenever(turtleService.getDataService(DATASERVICE_ID_1, false))
                 .thenReturn(responseReader.readFile("parsed_dataservice_1.ttl"))
 
             whenever(valuesMock.catalogUri)
-                .thenReturn("http://localhost:5000/catalogs")
+                .thenReturn("http://localhost:5050/catalogs")
             whenever(valuesMock.dataserviceUri)
-                .thenReturn("http://localhost:5000/dataservices")
+                .thenReturn("http://localhost:5050/dataservices")
 
             updateService.updateMetaData()
 
-            val expectedCatalog = responseReader.parseFile("catalog_0.ttl", "TURTLE")
+            val expectedCatalog0 = responseReader.parseFile("catalog_0.ttl", "TURTLE")
+            val expectedCatalog1 = responseReader.parseFile("catalog_1.ttl", "TURTLE")
             val expectedDataService0 = responseReader.parseFile("dataservice_0.ttl", "TURTLE")
             val expectedDataService1 = responseReader.parseFile("dataservice_1.ttl", "TURTLE")
 
@@ -64,10 +67,12 @@ class UpdateServiceTest {
             }
 
             argumentCaptor<Model, String, Boolean>().apply {
-                verify(turtleService, times(1)).saveAsCatalog(first.capture(), second.capture(), third.capture())
-                assertTrue(first.firstValue.isIsomorphicWith(expectedCatalog))
+                verify(turtleService, times(2)).saveAsCatalog(first.capture(), second.capture(), third.capture())
+                assertTrue(first.firstValue.isIsomorphicWith(expectedCatalog0))
+                assertTrue(first.secondValue.isIsomorphicWith(expectedCatalog1))
                 assertEquals(CATALOG_ID_0, second.firstValue)
-                assertEquals(listOf(true), third.allValues)
+                assertEquals(CATALOG_ID_1, second.secondValue)
+                assertEquals(listOf(true, true), third.allValues)
             }
         }
 

src/test/kotlin/no/digdir/informasjonsforvaltning/fdk_dataservice_harvester/utils/ApiTestContainer.kt

@@ -43,7 +43,7 @@ abstract class ApiTestContext {
             populateDB()
 
             try {
-                val con = URL("http://localhost:5000/ping").openConnection() as HttpURLConnection
+                val con = URL("http://localhost:5050/ping").openConnection() as HttpURLConnection
                 con.connect()
                 if (con.responseCode != 200) {
                     logger.debug("Ping to mock server failed")

src/test/kotlin/no/digdir/informasjonsforvaltning/fdk_dataservice_harvester/utils/DatabaseData.kt

@@ -19,7 +19,7 @@ val CATALOG_DBO_0 = CatalogMeta(
 val DATA_SERVICE_DBO_0 = DataServiceMeta(
     uri = "https://testdirektoratet.no/model/dataservice/0",
     fdkId = DATASERVICE_ID_0,
-    isPartOf = "http://localhost:5000/catalogs/$CATALOG_ID_0",
+    isPartOf = "http://localhost:5050/catalogs/$CATALOG_ID_0",
     issued = TEST_HARVEST_DATE.timeInMillis,
     modified = TEST_HARVEST_DATE.timeInMillis
 )
@@ -33,7 +33,7 @@ val CATALOG_DBO_1 = CatalogMeta(
 val DATA_SERVICE_DBO_1 = DataServiceMeta(
     uri = "https://testdirektoratet.no/model/dataservice/1",
     fdkId = DATASERVICE_ID_1,
-    isPartOf = "http://localhost:5000/catalogs/$CATALOG_ID_1",
+    isPartOf = "http://localhost:5050/catalogs/$CATALOG_ID_1",
     issued = TEST_HARVEST_DATE.timeInMillis,
     modified = TEST_HARVEST_DATE.timeInMillis
 )

src/test/kotlin/no/digdir/informasjonsforvaltning/fdk_dataservice_harvester/utils/TestData.kt

@@ -4,7 +4,7 @@ import no.digdir.informasjonsforvaltning.fdk_dataservice_harvester.model.Harvest
 import org.testcontainers.shaded.com.google.common.collect.ImmutableMap
 import java.util.*
 
-const val LOCAL_SERVER_PORT = 5000
+const val LOCAL_SERVER_PORT = 5050
 const val WIREMOCK_TEST_URI = "http://localhost:$LOCAL_SERVER_PORT"
 
 const val MONGO_USER = "testuser"

src/test/kotlin/no/digdir/informasjonsforvaltning/fdk_dataservice_harvester/utils/jwk/JwtToken.kt

@@ -17,7 +17,7 @@ class JwtToken (private val access: Access) {
             .claim("name", "TEST USER")
             .claim("given_name", "TEST")
             .claim("family_name", "USER")
-            .claim("iss", "http://localhost:5000/auth/realms/fdk")
+            .claim("iss", "http://localhost:5050/auth/realms/fdk")
             .claim("authorities", access.authorities)
             .build()
 

src/test/resources/all_catalogs.ttl

@@ -16,15 +16,15 @@
         vcard:hasOrganizationName  "Testdirektoratet"@nb ;
         vcard:hasURL               <https://testdirektoratet.no> .
 
-<http://localhost:5000/dataservices/ea51178e-f843-3025-98c5-7d02ce887f90>
+<http://localhost:5050/dataservices/ea51178e-f843-3025-98c5-7d02ce887f90>
         a                  dcat:CatalogRecord ;
         dct:identifier     "ea51178e-f843-3025-98c5-7d02ce887f90" ;
         dct:issued         "2020-03-12T11:52:16.122Z"^^xsd:dateTime ;
         dct:modified       "2020-03-12T11:52:16.122Z"^^xsd:dateTime ;
-        dct:isPartOf       <http://localhost:5000/catalogs/e422e2a7-287f-349f-876a-dc3541676f21> ;
+        dct:isPartOf       <http://localhost:5050/catalogs/e422e2a7-287f-349f-876a-dc3541676f21> ;
         foaf:primaryTopic  <https://testdirektoratet.no/model/dataservice/0> .
 
-<http://localhost:5000/catalogs/65555cdb-6809-3cc4-bff1-aaa6d9426311>
+<http://localhost:5050/catalogs/65555cdb-6809-3cc4-bff1-aaa6d9426311>
         a                  dcat:CatalogRecord ;
         dct:identifier     "65555cdb-6809-3cc4-bff1-aaa6d9426311" ;
         dct:issued         "2020-03-12T11:52:16.122Z"^^xsd:dateTime ;
@@ -55,17 +55,17 @@
         vcard:hasOrganizationName  "Testdirektoratet"@nb ;
         vcard:hasURL               <https://testdirektoratet.no> .
 
-<http://localhost:5000/catalogs/e422e2a7-287f-349f-876a-dc3541676f21>
+<http://localhost:5050/catalogs/e422e2a7-287f-349f-876a-dc3541676f21>
         a                  dcat:CatalogRecord ;
         dct:identifier     "e422e2a7-287f-349f-876a-dc3541676f21" ;
         dct:issued         "2020-03-12T11:52:16.122Z"^^xsd:dateTime ;
         dct:modified       "2020-03-12T11:52:16.122Z"^^xsd:dateTime ;
         foaf:primaryTopic  <https://testdirektoratet.no/model/dataservice-catalogs/0> .
 
-<http://localhost:5000/dataservices/4d69ecde-f1e8-3f28-8565-360746e8b5ef>
+<http://localhost:5050/dataservices/4d69ecde-f1e8-3f28-8565-360746e8b5ef>
         a                  dcat:CatalogRecord ;
         dct:identifier     "4d69ecde-f1e8-3f28-8565-360746e8b5ef" ;
         dct:issued         "2020-03-12T11:52:16.122Z"^^xsd:dateTime ;
         dct:modified       "2020-03-12T11:52:16.122Z"^^xsd:dateTime ;
-        dct:isPartOf       <http://localhost:5000/catalogs/65555cdb-6809-3cc4-bff1-aaa6d9426311> ;
+        dct:isPartOf       <http://localhost:5050/catalogs/65555cdb-6809-3cc4-bff1-aaa6d9426311> ;
         foaf:primaryTopic  <https://testdirektoratet.no/model/dataservice/1> .

src/test/resources/all_dataservices.ttl

@@ -16,12 +16,12 @@
         vcard:hasOrganizationName  "Testdirektoratet"@nb ;
         vcard:hasURL               <https://testdirektoratet.no> .
 
-<http://localhost:5000/dataservices/ea51178e-f843-3025-98c5-7d02ce887f90>
+<http://localhost:5050/dataservices/ea51178e-f843-3025-98c5-7d02ce887f90>
         a                  dcat:CatalogRecord ;
         dct:identifier     "ea51178e-f843-3025-98c5-7d02ce887f90" ;
         dct:issued         "2020-03-12T11:52:16.122Z"^^xsd:dateTime ;
         dct:modified       "2020-03-12T11:52:16.122Z"^^xsd:dateTime ;
-        dct:isPartOf       <http://localhost:5000/catalogs/e422e2a7-287f-349f-876a-dc3541676f21> ;
+        dct:isPartOf       <http://localhost:5050/catalogs/e422e2a7-287f-349f-876a-dc3541676f21> ;
         foaf:primaryTopic  <https://testdirektoratet.no/model/dataservice/0> .
 
 <https://testdirektoratet.no/model/dataservice/1>
@@ -36,10 +36,10 @@
         vcard:hasOrganizationName  "Testdirektoratet"@nb ;
         vcard:hasURL               <https://testdirektoratet.no> .
 
-<http://localhost:5000/dataservices/4d69ecde-f1e8-3f28-8565-360746e8b5ef>
+<http://localhost:5050/dataservices/4d69ecde-f1e8-3f28-8565-360746e8b5ef>
         a                  dcat:CatalogRecord ;
         dct:identifier     "4d69ecde-f1e8-3f28-8565-360746e8b5ef" ;
         dct:issued         "2020-03-12T11:52:16.122Z"^^xsd:dateTime ;
         dct:modified       "2020-03-12T11:52:16.122Z"^^xsd:dateTime ;
-        dct:isPartOf       <http://localhost:5000/catalogs/65555cdb-6809-3cc4-bff1-aaa6d9426311> ;
+        dct:isPartOf       <http://localhost:5050/catalogs/65555cdb-6809-3cc4-bff1-aaa6d9426311> ;
         foaf:primaryTopic  <https://testdirektoratet.no/model/dataservice/1> .

src/test/resources/catalog_0.ttl

@@ -22,17 +22,17 @@
         dct:title      "Dataservicekatalog for Testdirektoratet"@nb ;
         dcat:service   <https://testdirektoratet.no/model/dataservice/0> .
 
-<http://localhost:5000/catalogs/e422e2a7-287f-349f-876a-dc3541676f21>
+<http://localhost:5050/catalogs/e422e2a7-287f-349f-876a-dc3541676f21>
         a                  dcat:CatalogRecord ;
         dct:identifier     "e422e2a7-287f-349f-876a-dc3541676f21" ;
         dct:issued         "2020-03-12T11:52:16.122Z"^^xsd:dateTime ;
         dct:modified       "2020-03-12T11:52:16.122Z"^^xsd:dateTime ;
         foaf:primaryTopic  <https://testdirektoratet.no/model/dataservice-catalogs/0> .
 
-<http://localhost:5000/dataservices/ea51178e-f843-3025-98c5-7d02ce887f90>
+<http://localhost:5050/dataservices/ea51178e-f843-3025-98c5-7d02ce887f90>
         a                  dcat:CatalogRecord ;
         dct:identifier     "ea51178e-f843-3025-98c5-7d02ce887f90" ;
         dct:issued         "2020-03-12T11:52:16.122Z"^^xsd:dateTime ;
         dct:modified       "2020-03-12T11:52:16.122Z"^^xsd:dateTime ;
-        dct:isPartOf       <http://localhost:5000/catalogs/e422e2a7-287f-349f-876a-dc3541676f21> ;
+        dct:isPartOf       <http://localhost:5050/catalogs/e422e2a7-287f-349f-876a-dc3541676f21> ;
         foaf:primaryTopic  <https://testdirektoratet.no/model/dataservice/0> .