feat: inline dataloading (#1846)

refactor: inline dataloading

CODEOWNERS

@@ -52,7 +52,6 @@ styleguide.md @paullatzelsperger
 /extensions/data-plane/ @bscholtes1A @jimmarino
 /extensions/data-plane-selector/ @paullatzelsperger
 /extensions/data-plane-transfer/ @bscholtes1A
-/extensions/dataloading/ @paullatzelsperger
 /extensions/events/ @ndr-brt
 /extensions/filesystem/ @jimmarino @paullatzelsperger
 /extensions/http/ @paullatzelsperger
@@ -65,7 +64,6 @@ styleguide.md @paullatzelsperger
 /extensions/sql/ @bcronin90 @denisneuling @paullatzelsperger
 /extensions/transaction/ @jimmarino
 
-/launchers/data-loader-cli/ @paullatzelsperger @ronjaquensel
 /launchers/data-plane-server/ @bscholtes1A
 /launchers/dpf-selector/ @paullatzelsperger
 /launchers/ids-connector/ @ronjaquensel

core/common/base/build.gradle.kts

@@ -29,7 +29,6 @@ dependencies {
     api(project(":spi:common:transaction-spi"))
     implementation(project(":common:util"))
     implementation(project(":core:common:policy-engine"))
-    implementation(project(":extensions:dataloading"))
 
     api("com.squareup.okhttp3:okhttp:${okHttpVersion}")
     api("dev.failsafe:failsafe:${failsafeVersion}")

core/control-plane/control-plane-core/build.gradle.kts

@@ -24,7 +24,6 @@ dependencies {
     api(project(":core:control-plane:contract"))
     api(project(":core:control-plane:transfer"))
     implementation(project(":common:util"))
-    implementation(project(":extensions:dataloading"))
 }
 
 publishing {

core/control-plane/control-plane-core/src/main/java/org/eclipse/dataspaceconnector/core/controlplane/ControlPlaneCoreExtension.java

@@ -20,11 +20,11 @@
 import org.eclipse.dataspaceconnector.core.controlplane.defaults.negotiationstore.InMemoryContractNegotiationStore;
 import org.eclipse.dataspaceconnector.core.controlplane.defaults.policystore.InMemoryPolicyDefinitionStore;
 import org.eclipse.dataspaceconnector.core.controlplane.defaults.transferprocessstore.InMemoryTransferProcessStore;
-import org.eclipse.dataspaceconnector.dataloading.AssetLoader;
-import org.eclipse.dataspaceconnector.dataloading.ContractDefinitionLoader;
 import org.eclipse.dataspaceconnector.spi.asset.AssetIndex;
+import org.eclipse.dataspaceconnector.spi.asset.AssetLoader;
 import org.eclipse.dataspaceconnector.spi.asset.DataAddressResolver;
 import org.eclipse.dataspaceconnector.spi.contract.negotiation.store.ContractNegotiationStore;
+import org.eclipse.dataspaceconnector.spi.contract.offer.store.ContractDefinitionLoader;
 import org.eclipse.dataspaceconnector.spi.contract.offer.store.ContractDefinitionStore;
 import org.eclipse.dataspaceconnector.spi.policy.store.PolicyDefinitionStore;
 import org.eclipse.dataspaceconnector.spi.system.Provider;

core/control-plane/control-plane-core/src/main/java/org/eclipse/dataspaceconnector/core/controlplane/defaults/assetindex/InMemoryAssetIndex.java

@@ -15,15 +15,13 @@
 package org.eclipse.dataspaceconnector.core.controlplane.defaults.assetindex;
 
 import org.eclipse.dataspaceconnector.common.collection.CollectionUtil;
-import org.eclipse.dataspaceconnector.dataloading.AssetEntry;
-import org.eclipse.dataspaceconnector.dataloading.AssetLoader;
 import org.eclipse.dataspaceconnector.spi.asset.AssetIndex;
 import org.eclipse.dataspaceconnector.spi.asset.AssetSelectorExpression;
-import org.eclipse.dataspaceconnector.spi.asset.DataAddressResolver;
 import org.eclipse.dataspaceconnector.spi.query.QuerySpec;
 import org.eclipse.dataspaceconnector.spi.query.SortOrder;
 import org.eclipse.dataspaceconnector.spi.types.domain.DataAddress;
 import org.eclipse.dataspaceconnector.spi.types.domain.asset.Asset;
+import org.eclipse.dataspaceconnector.spi.types.domain.asset.AssetEntry;
 import org.jetbrains.annotations.Nullable;
 
 import java.util.Collections;
@@ -41,7 +39,7 @@
 /**
  * An ephemeral asset index, that is also a DataAddressResolver and an AssetLoader
  */
-public class InMemoryAssetIndex implements AssetIndex, DataAddressResolver, AssetLoader {
+public class InMemoryAssetIndex implements AssetIndex {
     private final Map<String, Asset> cache = new ConcurrentHashMap<>();
     private final Map<String, DataAddress> dataAddresses = new ConcurrentHashMap<>();
     private final AssetPredicateConverter predicateFactory;

core/control-plane/control-plane-core/src/test/java/org/eclipse/dataspaceconnector/core/controlplane/defaults/assetindex/InMemoryAssetLoaderIndexTest.java

@@ -14,10 +14,10 @@
 
 package org.eclipse.dataspaceconnector.core.controlplane.defaults.assetindex;
 
-import org.eclipse.dataspaceconnector.dataloading.AssetEntry;
-import org.eclipse.dataspaceconnector.dataloading.AssetLoader;
+import org.eclipse.dataspaceconnector.spi.asset.AssetLoader;
 import org.eclipse.dataspaceconnector.spi.types.domain.DataAddress;
 import org.eclipse.dataspaceconnector.spi.types.domain.asset.Asset;
+import org.eclipse.dataspaceconnector.spi.types.domain.asset.AssetEntry;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 

core/federated-catalog/federated-catalog-core/src/test/java/org/eclipse/dataspaceconnector/catalog/cache/FccTestExtension.java

@@ -21,8 +21,6 @@
 import org.eclipse.dataspaceconnector.policy.model.Permission;
 import org.eclipse.dataspaceconnector.policy.model.Policy;
 import org.eclipse.dataspaceconnector.policy.model.PolicyType;
-import org.eclipse.dataspaceconnector.spi.asset.AssetIndex;
-import org.eclipse.dataspaceconnector.spi.asset.AssetSelectorExpression;
 import org.eclipse.dataspaceconnector.spi.contract.negotiation.ContractNegotiationManager;
 import org.eclipse.dataspaceconnector.spi.contract.offer.ContractOfferQuery;
 import org.eclipse.dataspaceconnector.spi.contract.offer.ContractOfferService;
@@ -49,50 +47,24 @@
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
-import java.util.Objects;
 import java.util.concurrent.CompletableFuture;
 import java.util.stream.Stream;
 
 import static java.util.Collections.emptyList;
 
-@Provides({ RemoteMessageDispatcherRegistry.class, AssetIndex.class, TransferProcessStore.class, ContractDefinitionStore.class, IdentityService.class, ContractNegotiationManager.class, FederatedCacheNodeDirectory.class })
+@Provides({ RemoteMessageDispatcherRegistry.class, TransferProcessStore.class, ContractDefinitionStore.class, IdentityService.class, ContractNegotiationManager.class, FederatedCacheNodeDirectory.class })
 public class FccTestExtension implements ServiceExtension {
 
     @Override
     public void initialize(ServiceExtensionContext context) {
         List<Asset> assets = Collections.emptyList();
         context.registerService(TransferProcessStore.class, new FakeTransferProcessStore());
         context.registerService(RemoteMessageDispatcherRegistry.class, new FakeRemoteMessageDispatcherRegistry());
-        context.registerService(AssetIndex.class, new FakeAssetIndex(assets));
         context.registerService(ContractOfferService.class, new FakeContractOfferService(assets));
         context.registerService(ContractDefinitionStore.class, new FakeContractDefinitionStore());
         context.registerService(FederatedCacheNodeDirectory.class, new InMemoryNodeDirectory());
     }
 
-    private static class FakeAssetIndex implements AssetIndex {
-        private final List<Asset> assets;
-
-        private FakeAssetIndex(List<Asset> assets) {
-            this.assets = Objects.requireNonNull(assets);
-        }
-
-        @Override
-        public Stream<Asset> queryAssets(AssetSelectorExpression expression) {
-            return assets.stream();
-        }
-
-        @Override
-        public Stream<Asset> queryAssets(QuerySpec querySpec) {
-            throw new UnsupportedOperationException("Filtering/Paging not supported");
-        }
-
-        @Override
-        public Asset findById(String assetId) {
-            return assets.stream().filter(a -> a.getId().equals(assetId)).findFirst().orElse(null);
-        }
-
-    }
-
     private static class FakeContractOfferService implements ContractOfferService {
         private final List<Asset> assets;
 

data-protocols/ids/ids-api-multipart-dispatcher-v1/src/test/java/org/eclipse/dataspaceconnector/ids/api/multipart/client/IdsApiMultipartDispatcherV1IntegrationTestServiceExtension.java

@@ -46,7 +46,9 @@
 import org.eclipse.dataspaceconnector.spi.system.ServiceExtension;
 import org.eclipse.dataspaceconnector.spi.system.ServiceExtensionContext;
 import org.eclipse.dataspaceconnector.spi.transfer.store.TransferProcessStore;
+import org.eclipse.dataspaceconnector.spi.types.domain.DataAddress;
 import org.eclipse.dataspaceconnector.spi.types.domain.asset.Asset;
+import org.eclipse.dataspaceconnector.spi.types.domain.asset.AssetEntry;
 import org.eclipse.dataspaceconnector.spi.types.domain.contract.agreement.ContractAgreement;
 import org.eclipse.dataspaceconnector.spi.types.domain.contract.negotiation.ContractNegotiation;
 import org.eclipse.dataspaceconnector.spi.types.domain.contract.negotiation.ContractNegotiationStates;
@@ -148,6 +150,20 @@ public Asset findById(String assetId) {
             return assets.stream().filter(a -> a.getId().equals(assetId)).findFirst().orElse(null);
         }
 
+        @Override
+        public void accept(AssetEntry item) {
+
+        }
+
+        @Override
+        public Asset deleteById(String assetId) {
+            return null;
+        }
+
+        @Override
+        public DataAddress resolveForAsset(String assetId) {
+            return null;
+        }
     }
 
     private static class FakeContractOfferService implements ContractOfferService {

data-protocols/ids/ids-api-multipart-endpoint-v1/src/test/java/org/eclipse/dataspaceconnector/ids/api/multipart/IdsApiMultipartEndpointV1IntegrationTestServiceExtension.java

@@ -42,6 +42,7 @@
 import org.eclipse.dataspaceconnector.spi.transfer.store.TransferProcessStore;
 import org.eclipse.dataspaceconnector.spi.types.domain.DataAddress;
 import org.eclipse.dataspaceconnector.spi.types.domain.asset.Asset;
+import org.eclipse.dataspaceconnector.spi.types.domain.asset.AssetEntry;
 import org.eclipse.dataspaceconnector.spi.types.domain.contract.agreement.ContractAgreement;
 import org.eclipse.dataspaceconnector.spi.types.domain.contract.offer.ContractDefinition;
 import org.eclipse.dataspaceconnector.spi.types.domain.contract.offer.ContractOffer;
@@ -103,7 +104,7 @@ public Result<ClaimToken> verifyJwtToken(TokenRepresentation tokenRepresentation
         }
     }
 
-    private static class FakeAssetIndex implements AssetIndex, DataAddressResolver {
+    private static class FakeAssetIndex implements AssetIndex {
         private final List<Asset> assets;
 
         private FakeAssetIndex(List<Asset> assets) {
@@ -133,6 +134,16 @@ public DataAddress resolveForAsset(String assetId) {
             }
             return DataAddress.Builder.newInstance().type("test").build();
         }
+
+        @Override
+        public Asset deleteById(String assetId) {
+            return null;
+        }
+
+        @Override
+        public void accept(AssetEntry item) {
+
+        }
     }
 
     private static class FakeTransferProcessStore implements TransferProcessStore {
@@ -220,6 +231,11 @@ public ContractDefinition deleteById(String id) {
         public void reload() {
             throw new NotImplementedError();
         }
+
+        @Override
+        public void accept(ContractDefinition item) {
+
+        }
     }
 
     private static class FakeContractValidationService implements ContractValidationService {

docs/developer/modules.md

@@ -6,7 +6,6 @@ org.eclipse.dataspaceconnector:common-util:0.0.1-SNAPSHOT
 org.eclipse.dataspaceconnector:ids:0.0.1-SNAPSHOT
 org.eclipse.dataspaceconnector:aws-s3:0.0.1-SNAPSHOT
 org.eclipse.dataspaceconnector:data-plane-transfer:0.0.1-SNAPSHOT
-org.eclipse.dataspaceconnector:dataloading:0.0.1-SNAPSHOT
 org.eclipse.dataspaceconnector:hashicorp-vault:0.0.1-SNAPSHOT
 org.eclipse.dataspaceconnector:http:0.0.1-SNAPSHOT
 org.eclipse.dataspaceconnector:http-provisioner:0.0.1-SNAPSHOT
@@ -15,7 +14,6 @@ org.eclipse.dataspaceconnector:jdk-logger-monitor:0.0.1-SNAPSHOT
 org.eclipse.dataspaceconnector:junit:0.0.1-SNAPSHOT
 org.eclipse.dataspaceconnector:core-micrometer:0.0.1-SNAPSHOT
 org.eclipse.dataspaceconnector:sql:0.0.1-SNAPSHOT
-org.eclipse.dataspaceconnector:dataloader-cli:0.0.1-SNAPSHOT
 org.eclipse.dataspaceconnector:data-plane-server:0.0.1-SNAPSHOT
 org.eclipse.dataspaceconnector:module-domain:0.0.1-SNAPSHOT
 org.eclipse.dataspaceconnector:module-processor:0.0.1-SNAPSHOT

docs/overview/extensions.md

@@ -11,7 +11,6 @@
 | :extensions:data-plane          | Covers the Data Plane Framework (DPF), which is the entity in charge of performing the actual data transfer between consumer and provider. |
 | :extensions:data-plane-selector |  |
 | :extensions:data-plane-transfer | Provides resources used to delegate data transfer to the Data Plane, or to use the Data Plane as a proxy for querying the data. |
-| :extensions:dataloading         |  |
 | :extensions:events              | Provides a way to register endpoints where the domain events will be sent as soon as they occur. |
 | :extensions:filesystem          |  |
 | :extensions:http                |  |
@@ -88,10 +87,6 @@
 | :data-plane-transfer-spi    |  | [link]({{extensions}}data-plane-transfer/data-plane-transfer-spi) |
 | :data-plane-transfer-sync   |  | [link]({{extensions}}data-plane-transfer/data-plane-transfer-sync) |
 
-## :extensions:dataloading
-
-No submodules
-
 ## :extensions:events
 
 | Module | Description | Details

docs/overview/launchers.md

@@ -1,8 +1,7 @@
 # Launchers
 
-| Module | Description | Details
-| :----- | :---------- | :------ |
-| :launchers:data-loader-cli          | Demonstrates how the data loading extension can be used to load data from JSON files. | [link]({{launchers}}data-loader-cli) |
-| :launchers:data-plane-server        |  | [link]({{launchers}}data-plane-server) |
-| :launchers:dpf-selector             |  | [link]({{launchers}}dpf-selector) |
-| :launchers:ids-connector            | Includes all extensions that are required for an IDS Connector deployment. | [link]({{launchers}}ids-connector) |
+| Module                       | Description                                                                | Details                                |
+|:-----------------------------|:---------------------------------------------------------------------------|:---------------------------------------|
+| :launchers:data-plane-server |                                                                            | [link]({{launchers}}data-plane-server) |
+| :launchers:dpf-selector      |                                                                            | [link]({{launchers}}dpf-selector)      |
+| :launchers:ids-connector     | Includes all extensions that are required for an IDS Connector deployment. | [link]({{launchers}}ids-connector)     |

extensions/api/data-management/asset/build.gradle.kts

@@ -24,11 +24,11 @@ plugins {
 }
 
 dependencies {
+    api(project(":spi:control-plane:control-plane-spi"))
     api(project(":spi:common:transaction-spi"))
     implementation(project(":extensions:api:api-core"))
     implementation(project(":spi:common:auth-spi"))
     implementation(project(":extensions:api:data-management:api-configuration"))
-    implementation(project(":extensions:dataloading"))
 
     implementation("jakarta.ws.rs:jakarta.ws.rs-api:${rsApi}")
 

extensions/api/data-management/asset/src/main/java/org/eclipse/dataspaceconnector/api/datamanagement/asset/AssetApiExtension.java

@@ -24,9 +24,9 @@
 import org.eclipse.dataspaceconnector.api.datamanagement.asset.transform.DataAddressDtoToDataAddressTransformer;
 import org.eclipse.dataspaceconnector.api.datamanagement.configuration.DataManagementApiConfiguration;
 import org.eclipse.dataspaceconnector.api.transformer.DtoTransformerRegistry;
-import org.eclipse.dataspaceconnector.dataloading.AssetLoader;
 import org.eclipse.dataspaceconnector.spi.WebService;
 import org.eclipse.dataspaceconnector.spi.asset.AssetIndex;
+import org.eclipse.dataspaceconnector.spi.asset.AssetLoader;
 import org.eclipse.dataspaceconnector.spi.contract.negotiation.store.ContractNegotiationStore;
 import org.eclipse.dataspaceconnector.spi.event.EventRouter;
 import org.eclipse.dataspaceconnector.spi.observe.asset.AssetObservableImpl;

extensions/api/data-management/asset/src/main/java/org/eclipse/dataspaceconnector/api/datamanagement/asset/service/AssetServiceImpl.java

@@ -15,8 +15,8 @@
 package org.eclipse.dataspaceconnector.api.datamanagement.asset.service;
 
 import org.eclipse.dataspaceconnector.api.result.ServiceResult;
-import org.eclipse.dataspaceconnector.dataloading.AssetLoader;
 import org.eclipse.dataspaceconnector.spi.asset.AssetIndex;
+import org.eclipse.dataspaceconnector.spi.asset.AssetLoader;
 import org.eclipse.dataspaceconnector.spi.contract.negotiation.store.ContractNegotiationStore;
 import org.eclipse.dataspaceconnector.spi.observe.asset.AssetObservable;
 import org.eclipse.dataspaceconnector.spi.query.Criterion;

extensions/api/data-management/asset/src/test/java/org/eclipse/dataspaceconnector/api/datamanagement/asset/AssetApiControllerIntegrationTest.java

@@ -19,10 +19,10 @@
 import org.eclipse.dataspaceconnector.api.datamanagement.asset.model.AssetEntryDto;
 import org.eclipse.dataspaceconnector.api.datamanagement.asset.model.AssetRequestDto;
 import org.eclipse.dataspaceconnector.api.datamanagement.asset.model.DataAddressDto;
-import org.eclipse.dataspaceconnector.dataloading.AssetLoader;
 import org.eclipse.dataspaceconnector.junit.extensions.EdcExtension;
 import org.eclipse.dataspaceconnector.policy.model.Policy;
 import org.eclipse.dataspaceconnector.spi.asset.AssetIndex;
+import org.eclipse.dataspaceconnector.spi.asset.AssetLoader;
 import org.eclipse.dataspaceconnector.spi.contract.negotiation.store.ContractNegotiationStore;
 import org.eclipse.dataspaceconnector.spi.types.domain.DataAddress;
 import org.eclipse.dataspaceconnector.spi.types.domain.asset.Asset;

extensions/api/data-management/asset/src/test/java/org/eclipse/dataspaceconnector/api/datamanagement/asset/service/AssetServiceImplTest.java

@@ -14,9 +14,9 @@
 
 package org.eclipse.dataspaceconnector.api.datamanagement.asset.service;
 
-import org.eclipse.dataspaceconnector.dataloading.AssetLoader;
 import org.eclipse.dataspaceconnector.policy.model.Policy;
 import org.eclipse.dataspaceconnector.spi.asset.AssetIndex;
+import org.eclipse.dataspaceconnector.spi.asset.AssetLoader;
 import org.eclipse.dataspaceconnector.spi.contract.negotiation.store.ContractNegotiationStore;
 import org.eclipse.dataspaceconnector.spi.observe.asset.AssetObservable;
 import org.eclipse.dataspaceconnector.spi.query.QuerySpec;

extensions/api/data-management/contractdefinition/build.gradle.kts

@@ -24,11 +24,11 @@ plugins {
 }
 
 dependencies {
+    api(project(":spi:control-plane:control-plane-spi"))
     api(project(":spi:common:transaction-spi"))
     implementation(project(":extensions:api:api-core"))
     implementation(project(":spi:common:auth-spi"))
     implementation(project(":extensions:api:data-management:api-configuration"))
-    implementation(project(":extensions:dataloading"))
 
     implementation("jakarta.ws.rs:jakarta.ws.rs-api:${rsApi}")
 

extensions/api/data-management/contractdefinition/src/main/java/org/eclipse/dataspaceconnector/api/datamanagement/contractdefinition/ContractDefinitionApiExtension.java

@@ -22,9 +22,9 @@
 import org.eclipse.dataspaceconnector.api.datamanagement.contractdefinition.transform.ContractDefinitionRequestDtoToContractDefinitionTransformer;
 import org.eclipse.dataspaceconnector.api.datamanagement.contractdefinition.transform.ContractDefinitionToContractDefinitionResponseDtoTransformer;
 import org.eclipse.dataspaceconnector.api.transformer.DtoTransformerRegistry;
-import org.eclipse.dataspaceconnector.dataloading.ContractDefinitionLoader;
 import org.eclipse.dataspaceconnector.spi.WebService;
 import org.eclipse.dataspaceconnector.spi.contract.definition.observe.ContractDefinitionObservableImpl;
+import org.eclipse.dataspaceconnector.spi.contract.offer.store.ContractDefinitionLoader;
 import org.eclipse.dataspaceconnector.spi.contract.offer.store.ContractDefinitionStore;
 import org.eclipse.dataspaceconnector.spi.event.EventRouter;
 import org.eclipse.dataspaceconnector.spi.system.Inject;

extensions/api/data-management/contractdefinition/src/main/java/org/eclipse/dataspaceconnector/api/datamanagement/contractdefinition/service/ContractDefinitionServiceImpl.java

@@ -15,8 +15,8 @@
 package org.eclipse.dataspaceconnector.api.datamanagement.contractdefinition.service;
 
 import org.eclipse.dataspaceconnector.api.result.ServiceResult;
-import org.eclipse.dataspaceconnector.dataloading.ContractDefinitionLoader;
 import org.eclipse.dataspaceconnector.spi.contract.definition.observe.ContractDefinitionObservable;
+import org.eclipse.dataspaceconnector.spi.contract.offer.store.ContractDefinitionLoader;
 import org.eclipse.dataspaceconnector.spi.contract.offer.store.ContractDefinitionStore;
 import org.eclipse.dataspaceconnector.spi.query.QuerySpec;
 import org.eclipse.dataspaceconnector.spi.query.QueryValidator;

extensions/api/data-management/contractdefinition/src/test/java/org/eclipse/dataspaceconnector/api/datamanagement/contractdefinition/ContractDefinitionApiControllerIntegrationTest.java

@@ -17,9 +17,9 @@
 
 import io.restassured.specification.RequestSpecification;
 import org.eclipse.dataspaceconnector.api.datamanagement.contractdefinition.model.ContractDefinitionResponseDto;
-import org.eclipse.dataspaceconnector.dataloading.ContractDefinitionLoader;
 import org.eclipse.dataspaceconnector.junit.extensions.EdcExtension;
 import org.eclipse.dataspaceconnector.spi.asset.AssetSelectorExpression;
+import org.eclipse.dataspaceconnector.spi.contract.offer.store.ContractDefinitionLoader;
 import org.eclipse.dataspaceconnector.spi.contract.offer.store.ContractDefinitionStore;
 import org.eclipse.dataspaceconnector.spi.query.QuerySpec;
 import org.eclipse.dataspaceconnector.spi.types.domain.contract.offer.ContractDefinition;