EVA-3227 Match metadata files info with uploaded files (#15)

* match files mentioned in metadata with uploaded files
EBIvariation · Jul 16, 2024 · 5c99a98 · 5c99a98
1 parent 821e52d
commit 5c99a98
Show file tree

Hide file tree

Showing 6 changed files with 407 additions and 25 deletions.
diff --git a/src/main/java/uk/ac/ebi/eva/submission/config/AppConfig.java b/src/main/java/uk/ac/ebi/eva/submission/config/AppConfig.java
@@ -4,6 +4,7 @@
 import org.springframework.context.annotation.Configuration;
 import org.springframework.mail.javamail.JavaMailSender;
 import org.springframework.mail.javamail.JavaMailSenderImpl;
+import org.springframework.web.client.RestTemplate;
 
 @Configuration
 public class AppConfig {
@@ -15,4 +16,9 @@ public JavaMailSender javaMailService() {
         javaMailSender.setPort(25);
         return javaMailSender;
     }
+
+    @Bean
+    public RestTemplate restTemplate() {
+        return new RestTemplate();
+    }
 }
diff --git a/src/main/java/uk/ac/ebi/eva/submission/controller/submissionws/SubmissionController.java b/src/main/java/uk/ac/ebi/eva/submission/controller/submissionws/SubmissionController.java
@@ -19,6 +19,7 @@
 import uk.ac.ebi.eva.submission.controller.BaseController;
 import uk.ac.ebi.eva.submission.entity.Submission;
 import uk.ac.ebi.eva.submission.entity.SubmissionAccount;
+import uk.ac.ebi.eva.submission.exception.MetadataFileInfoMismatchException;
 import uk.ac.ebi.eva.submission.exception.RequiredFieldsMissingException;
 import uk.ac.ebi.eva.submission.exception.SubmissionDoesNotExistException;
 import uk.ac.ebi.eva.submission.model.SubmissionStatus;
@@ -93,11 +94,14 @@ public ResponseEntity<?> markSubmissionUploaded(@RequestHeader("Authorization")
         }
 
         try {
+            submissionService.checkMetadataFileInfoMatchesWithUploadedFiles(submissionAccount, submissionId, metadataJson);
             Submission submission = this.submissionService.uploadMetadataJsonAndMarkUploaded(submissionId, metadataJson);
             submissionService.sendMailNotificationForStatusUpdate(submissionAccount, submissionId, SubmissionStatus.UPLOADED, true);
             return new ResponseEntity<>(stripUserDetails(submission), HttpStatus.OK);
         } catch (RequiredFieldsMissingException ex) {
             return new ResponseEntity<>(ex.getMessage(), HttpStatus.BAD_REQUEST);
+        } catch (MetadataFileInfoMismatchException ex) {
+            return new ResponseEntity<>(ex.getMessage(), HttpStatus.BAD_REQUEST);
         }
     }
 

diff --git a/src/main/java/uk/ac/ebi/eva/submission/exception/MetadataFileInfoMismatchException.java b/src/main/java/uk/ac/ebi/eva/submission/exception/MetadataFileInfoMismatchException.java
@@ -0,0 +1,7 @@
+package uk.ac.ebi.eva.submission.exception;
+
+public class MetadataFileInfoMismatchException extends RuntimeException {
+    public MetadataFileInfoMismatchException(String message) {
+        super(message);
+    }
+}
diff --git a/src/main/java/uk/ac/ebi/eva/submission/service/GlobusDirectoryProvisioner.java b/src/main/java/uk/ac/ebi/eva/submission/service/GlobusDirectoryProvisioner.java
@@ -1,5 +1,7 @@
 package uk.ac.ebi.eva.submission.service;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.http.HttpEntity;
 import org.springframework.http.HttpHeaders;
@@ -14,39 +16,37 @@
 @Service
 public class GlobusDirectoryProvisioner {
 
+    private final Logger logger = LoggerFactory.getLogger(GlobusDirectoryProvisioner.class);
+
     private final GlobusTokenRefreshService globusTokenRefreshService;
 
+    private final RestTemplate restTemplate;
+
     @Value("${globus.submission.endpointId}")
     private String endpointId;
 
     private static final String GLOBUS_TRANSFER_API_BASE_URL =
             "https://transfer.api.globusonline.org/v0.10/operation/endpoint";
 
 
-    public GlobusDirectoryProvisioner(GlobusTokenRefreshService globusTokenRefreshService) {
+    public GlobusDirectoryProvisioner(GlobusTokenRefreshService globusTokenRefreshService, RestTemplate restTemplate) {
         this.globusTokenRefreshService = globusTokenRefreshService;
+        this.restTemplate = restTemplate;
     }
 
     public void createSubmissionDirectory(String directoryToCreate) {
         String fileSeparator = System.getProperty("file.separator");
         String[] directoriesToCreate = directoryToCreate.split(Pattern.quote(fileSeparator));
         StringBuilder directoryPathSoFar = new StringBuilder();
-        for (String directory: directoriesToCreate) {
+        for (String directory : directoriesToCreate) {
             directoryPathSoFar.append(directory);
             createDirectory(directoryPathSoFar.toString());
             directoryPathSoFar.append(fileSeparator);
         }
     }
 
     private void createDirectory(String directoryToCreate) {
-
-        String accessToken = globusTokenRefreshService.getAccessToken();
-        RestTemplate restTemplate = new RestTemplate();
-        HttpHeaders headers = new HttpHeaders();
-        headers.set("Authorization", "Bearer " + accessToken);
-        headers.set("Accept", "application/json");
-        headers.set("Content-Type", "application/json");
-
+        HttpHeaders headers = getGlobusAccessHeaders();
 
         if (this.alreadyExists(headers, directoryToCreate)) {
             return;
@@ -63,23 +63,47 @@ private void createDirectory(String directoryToCreate) {
 
         // Check the response status and handle errors if necessary
         if (response.getStatusCode().is2xxSuccessful()) {
-            System.out.printf("Directory '%s' created successfully%n", directoryToCreate);
+            logger.info("Directory '%s' created successfully%n", directoryToCreate);
         } else {
-            System.out.printf("Failed to create directory '%s': %s", directoryToCreate, response.getStatusCode());
+            logger.error("Failed to create directory '%s': %s", directoryToCreate, response.getStatusCode());
         }
     }
 
-    private boolean alreadyExists(HttpHeaders headers,  String directoryName) {
-        RestTemplate restTemplate = new RestTemplate();
+    private boolean alreadyExists(HttpHeaders headers, String directoryName) {
         HttpEntity<String> entity = new HttpEntity<>("", headers);
         try {
             restTemplate.exchange(
                     GLOBUS_TRANSFER_API_BASE_URL + "/" + endpointId + "/ls?path=" + directoryName,
                     HttpMethod.GET, entity, String.class);
             return true;
-        }
-        catch (HttpClientErrorException ex) {
+        } catch (HttpClientErrorException ex) {
             return false;
         }
     }
+
+    public String listSubmittedFiles(String submissionDirPath) {
+        HttpEntity<String> requestEntity = new HttpEntity<>(getGlobusAccessHeaders());
+        String transferApiUrl = String.format("%s/%s/ls?path=%s", GLOBUS_TRANSFER_API_BASE_URL, endpointId, submissionDirPath);
+        ResponseEntity<String> response = restTemplate.exchange(transferApiUrl, HttpMethod.GET, requestEntity, String.class);
+
+        if (response.getStatusCode().is2xxSuccessful()) {
+            logger.info("Directory %s listed successfully%n", submissionDirPath);
+            return response.getBody();
+        } else {
+            logger.error("Failed to retrieve directory '%s': %s", submissionDirPath, response.getStatusCode());
+            return "";
+        }
+    }
+
+    private HttpHeaders getGlobusAccessHeaders() {
+        String accessToken = globusTokenRefreshService.getAccessToken();
+        HttpHeaders headers = new HttpHeaders();
+        headers.set("Authorization", "Bearer " + accessToken);
+        headers.set("Accept", "application/json");
+        headers.set("Content-Type", "application/json");
+
+        return headers;
+    }
+
+
 }
diff --git a/src/main/java/uk/ac/ebi/eva/submission/service/SubmissionService.java b/src/main/java/uk/ac/ebi/eva/submission/service/SubmissionService.java
@@ -1,12 +1,16 @@
 package uk.ac.ebi.eva.submission.service;
 
+import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Service;
 import uk.ac.ebi.eva.submission.entity.Submission;
 import uk.ac.ebi.eva.submission.entity.SubmissionAccount;
 import uk.ac.ebi.eva.submission.entity.SubmissionDetails;
 import uk.ac.ebi.eva.submission.entity.SubmissionProcessing;
+import uk.ac.ebi.eva.submission.exception.MetadataFileInfoMismatchException;
 import uk.ac.ebi.eva.submission.exception.RequiredFieldsMissingException;
 import uk.ac.ebi.eva.submission.exception.SubmissionDoesNotExistException;
 import uk.ac.ebi.eva.submission.model.SubmissionProcessingStatus;
@@ -20,15 +24,26 @@
 import uk.ac.ebi.eva.submission.util.MailSender;
 
 import java.time.LocalDateTime;
+import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.Optional;
 import java.util.UUID;
+import java.util.stream.Collectors;
+import java.util.stream.StreamSupport;
 
 @Service
 public class SubmissionService {
     private static final String PROJECT = "project";
     private static final String TITLE = "title";
     private static final String DESCRIPTION = "description";
+    private static final String METADATA_FILES_TAG = "files";
+    private static final String METADATA_FILE_NAME = "fileName";
+    private static final String METADATA_FILE_SIZE = "fileSize";
+    private static final String GLOBUS_FILES_TAG = "DATA";
+    private static final String GLOBUS_FILE_NAME = "name";
+    private static final String GLOBUS_FILE_SIZE = "size";
 
     private final SubmissionRepository submissionRepository;
 
@@ -82,6 +97,63 @@ public Submission initiateSubmission(SubmissionAccount submissionAccount) {
         return submissionRepository.save(submission);
     }
 
+    public void checkMetadataFileInfoMatchesWithUploadedFiles(SubmissionAccount submissionAccount, String submissionId, JsonNode metadataJson) {
+        Map<String, Long> metadataFileInfo = new HashMap<>();
+        if (metadataJson.get(METADATA_FILES_TAG) != null) {
+            metadataFileInfo = StreamSupport.stream(metadataJson.get(METADATA_FILES_TAG).spliterator(), false)
+                    .collect(Collectors.toMap(
+                            dataNode -> dataNode.get(METADATA_FILE_NAME).asText(),
+                            dataNode -> dataNode.get(METADATA_FILE_SIZE).asLong()
+                    ));
+        }
+        if (metadataFileInfo.isEmpty()) {
+            throw new MetadataFileInfoMismatchException("Metadata json file does not have any file info");
+        }
+
+        String directoryToList = String.format("%s/%s", submissionAccount.getId(), submissionId);
+        String uploadedFilesInfo = globusDirectoryProvisioner.listSubmittedFiles(directoryToList);
+        if (uploadedFilesInfo.isEmpty()) {
+            throw new MetadataFileInfoMismatchException("Failed to retrieve any file info from submission directory.");
+        } else {
+            Map<String, Long> globusFileInfo = new HashMap<>();
+            try {
+                ObjectMapper mapper = new ObjectMapper();
+                ObjectNode globusFileInfoJson = (ObjectNode) mapper.readTree(uploadedFilesInfo);
+                if (globusFileInfoJson.get(GLOBUS_FILES_TAG) != null) {
+                    globusFileInfo = StreamSupport.stream(globusFileInfoJson.get(GLOBUS_FILES_TAG).spliterator(), false)
+                            .collect(Collectors.toMap(
+                                    dataNode -> dataNode.get(GLOBUS_FILE_NAME).asText(),
+                                    dataNode -> dataNode.get(GLOBUS_FILE_SIZE).asLong()
+                            ));
+                }
+            } catch (JsonProcessingException ex) {
+                throw new MetadataFileInfoMismatchException("Error parsing fileInfo from Submission Directory");
+            }
+
+            List<String> missingFileList = new ArrayList<>();
+            String fileSizeMismatchInfo = "";
+
+            for (Map.Entry<String, Long> fileEntry : metadataFileInfo.entrySet()) {
+                String fileName = fileEntry.getKey();
+                Long metadataFileSize = fileEntry.getValue();
+                if (globusFileInfo.containsKey(fileName)) {
+                    Long fileSizeInGlobus = globusFileInfo.get(fileName);
+                    if (!metadataFileSize.equals(fileSizeInGlobus)) {
+                        fileSizeMismatchInfo += fileName + ": metadata json file size (" + metadataFileSize + ") is not equal to uploaded file size (" + fileSizeInGlobus + ")\n";
+                    }
+                } else {
+                    missingFileList.add(fileName);
+                }
+            }
+
+            if (!missingFileList.isEmpty() || !fileSizeMismatchInfo.isEmpty()) {
+                String missingFileMsg = missingFileList.isEmpty() ? "" : "There are some files mentioned in metadata json but not uploaded. Files : " + String.join(", ", missingFileList) + "\n";
+                String fileSizeMismatchMsg = fileSizeMismatchInfo.isEmpty() ? "" : "There are some files mentioned in metadata json whose size does not match with the files uploaded.\n" + fileSizeMismatchInfo;
+                throw new MetadataFileInfoMismatchException(missingFileMsg + fileSizeMismatchMsg);
+            }
+        }
+    }
+
     public Submission uploadMetadataJsonAndMarkUploaded(String submissionId, JsonNode metadataJson) {
         SubmissionDetails submissionDetails = new SubmissionDetails(submissionId);
         try {