merge master

nationalarchives · Feb 6, 2025 · 4b552cd · 4b552cd
2 parents c356bb4 + 780003c
commit 4b552cd
Show file tree

Hide file tree

Showing 18 changed files with 845 additions and 363 deletions.
diff --git a/schema.graphql b/schema.graphql
@@ -50,6 +50,7 @@ input AddMultipleFileStatusesInput {
 input AddOrUpdateBulkFileMetadataInput {
   consignmentId: UUID!
   fileMetadata: [AddOrUpdateFileMetadata!]!
+  skipValidation: Boolean = false
 }
 
 input AddOrUpdateFileMetadata {

diff --git a/src/main/scala/uk/gov/nationalarchives/tdr/api/graphql/fields/FileMetadataFields.scala b/src/main/scala/uk/gov/nationalarchives/tdr/api/graphql/fields/FileMetadataFields.scala
@@ -5,11 +5,10 @@ import io.circe.generic.auto._
 import sangria.macros.derive.{deriveInputObjectType, deriveObjectType}
 import sangria.marshalling.circe._
 import sangria.schema.{Argument, Field, InputObjectType, ListType, ObjectType, fields}
-import uk.gov.nationalarchives.tdr.api.auth.{ValidateHasChecksumMetadataAccess, ValidateUserOwnsFiles}
+import uk.gov.nationalarchives.tdr.api.auth.ValidateHasChecksumMetadataAccess
 import uk.gov.nationalarchives.tdr.api.graphql.ConsignmentApiContext
 import uk.gov.nationalarchives.tdr.api.metadatainputvalidation.ValidateMetadataInput
 import FieldTypes._
-import uk.gov.nationalarchives.tdr.api.service.FileMetadataService.ClosureType
 
 object FileMetadataFields {
   trait FileMetadataBase {
@@ -33,7 +32,7 @@ object FileMetadataFields {
   case class UpdateBulkFileMetadataInput(consignmentId: UUID, fileIds: Seq[UUID], metadataProperties: Seq[UpdateFileMetadataInput])
   case class AddOrUpdateMetadata(filePropertyName: String, value: String) extends FileMetadataBase
   case class AddOrUpdateFileMetadata(fileId: UUID, metadata: Seq[AddOrUpdateMetadata])
-  case class AddOrUpdateBulkFileMetadataInput(consignmentId: UUID, fileMetadata: Seq[AddOrUpdateFileMetadata])
+  case class AddOrUpdateBulkFileMetadataInput(consignmentId: UUID, fileMetadata: Seq[AddOrUpdateFileMetadata], skipValidation: Boolean = false)
 
   case class DeleteFileMetadata(fileIds: Seq[UUID], filePropertyNames: Seq[String])
 

diff --git a/src/main/scala/uk/gov/nationalarchives/tdr/api/http/GraphQLServerBase.scala b/src/main/scala/uk/gov/nationalarchives/tdr/api/http/GraphQLServerBase.scala
@@ -72,9 +72,10 @@ trait GraphQLServerBase {
     val displayPropertiesService = new DisplayPropertiesService(displayPropertiesRepository)
     val validateFileMetadataService = new ValidateFileMetadataService(customMetadataPropertiesService, displayPropertiesService, fileMetadataRepository, fileStatusRepository)
     val consignmentStatusService = new ConsignmentStatusService(consignmentStatusRepository, fileStatusRepository, uuidSource, timeSource)
-    val fileMetadataService = new FileMetadataService(fileMetadataRepository, consignmentStatusService, customMetadataPropertiesService, validateFileMetadataService)
+    val fileStatusService = new FileStatusService(fileStatusRepository, customMetadataPropertiesService, displayPropertiesService)
+    val fileMetadataService =
+      new FileMetadataService(fileMetadataRepository, consignmentStatusService, customMetadataPropertiesService, validateFileMetadataService, fileStatusService)
     val ffidMetadataService = new FFIDMetadataService(ffidMetadataRepository, ffidMetadataMatchesRepository, timeSource, uuidSource)
-    val fileStatusService = new FileStatusService(fileStatusRepository)
     val referenceGeneratorService = new ReferenceGeneratorService(config, SimpleHttpClient())
     val fileService = new FileService(
       fileRepository,

diff --git a/...ain/scala/uk/gov/nationalarchives/tdr/api/metadatainputvalidation/MetadataInputTags.scala b/...ain/scala/uk/gov/nationalarchives/tdr/api/metadatainputvalidation/MetadataInputTags.scala
@@ -21,10 +21,10 @@ case class ValidateMetadataInput[T](argument: Argument[T]) extends MetadataInput
   override def validateAsync(ctx: Context[ConsignmentApiContext, _])(implicit executionContext: ExecutionContext): Future[BeforeFieldResult[ConsignmentApiContext, Unit]] = {
     val arg: T = ctx.arg[T](argument.name)
 
-    val (inputFileIds: Seq[UUID], inputConsignmentId: UUID) = arg match {
-      case updateInput: UpdateBulkFileMetadataInput           => (updateInput.fileIds, updateInput.consignmentId)
-      case deleteInput: DeleteFileMetadataInput               => (deleteInput.fileIds, deleteInput.consignmentId)
-      case addOrUpdateInput: AddOrUpdateBulkFileMetadataInput => (addOrUpdateInput.fileMetadata.map(_.fileId), addOrUpdateInput.consignmentId)
+    val (inputFileIds: Seq[UUID], inputConsignmentId: UUID, skipValidation: Boolean) = arg match {
+      case updateInput: UpdateBulkFileMetadataInput           => (updateInput.fileIds, updateInput.consignmentId, false)
+      case deleteInput: DeleteFileMetadataInput               => (deleteInput.fileIds, deleteInput.consignmentId, false)
+      case addOrUpdateInput: AddOrUpdateBulkFileMetadataInput => (addOrUpdateInput.fileMetadata.map(_.fileId), addOrUpdateInput.consignmentId, addOrUpdateInput.skipValidation)
     }
     val token = ctx.ctx.accessToken
     val userId = token.userId
@@ -37,7 +37,7 @@ case class ValidateMetadataInput[T](argument: Argument[T]) extends MetadataInput
 
     for {
       fileFields <- ctx.ctx.fileService.getFileDetails(inputFileIds)
-      noAccess = fileFields.exists(_.userId != userId) && !draftMetadataValidatorAccess
+      noAccess = (fileFields.exists(_.userId != userId) || skipValidation) && !draftMetadataValidatorAccess
     } yield {
       noAccess match {
         case true => throw AuthorisationException("Access denied to file metadata")

diff --git a/src/main/scala/uk/gov/nationalarchives/tdr/api/service/CustomMetadataPropertiesService.scala b/src/main/scala/uk/gov/nationalarchives/tdr/api/service/CustomMetadataPropertiesService.scala
@@ -4,6 +4,7 @@ import uk.gov.nationalarchives.Tables.{FilepropertyRow, Filepropertydependencies
 import uk.gov.nationalarchives.tdr.api.db.repository.CustomMetadataPropertiesRepository
 import uk.gov.nationalarchives.tdr.api.graphql.fields.CustomMetadataFields
 import uk.gov.nationalarchives.tdr.api.graphql.fields.CustomMetadataFields._
+import uk.gov.nationalarchives.tdr.api.service.FileStatusService.{ClosureMetadata, DescriptiveMetadata}
 
 import scala.concurrent.{ExecutionContext, Future}
 
@@ -48,6 +49,12 @@ class CustomMetadataPropertiesService(customMetadataPropertiesRepository: Custom
       }
     }
 
+  def toAdditionalMetadataFieldGroups(fields: Seq[CustomMetadataField]): Seq[FieldGroup] = {
+    val closureFields = fields.filter(f => f.propertyGroup.contains("MandatoryClosure") || f.propertyGroup.contains("OptionalClosure"))
+    val descriptiveFields = fields.filter(f => f.propertyGroup.contains("OptionalMetadata"))
+    Seq(FieldGroup(ClosureMetadata, closureFields), FieldGroup(DescriptiveMetadata, descriptiveFields))
+  }
+
   private def rowsToMetadata(
       fp: FilepropertyRow,
       values: Map[String, Seq[FilepropertyvaluesRow]],
@@ -99,3 +106,5 @@ class CustomMetadataPropertiesService(customMetadataPropertiesRepository: Custom
     case _                => throw new Exception(s"Invalid property type $propertyType")
   }
 }
+
+case class FieldGroup(groupName: String, fields: Seq[CustomMetadataField])
diff --git a/src/main/scala/uk/gov/nationalarchives/tdr/api/service/FileMetadataService.scala b/src/main/scala/uk/gov/nationalarchives/tdr/api/service/FileMetadataService.scala
@@ -1,7 +1,7 @@
 package uk.gov.nationalarchives.tdr.api.service
 
 import sangria.macros.derive.GraphQLDeprecated
-import uk.gov.nationalarchives.Tables.FilemetadataRow
+import uk.gov.nationalarchives.Tables.{FilemetadataRow, FilestatusRow}
 import uk.gov.nationalarchives.tdr.api.db.repository.FileMetadataRepository
 import uk.gov.nationalarchives.tdr.api.graphql.DataExceptions.InputDataException
 import uk.gov.nationalarchives.tdr.api.graphql.fields.AntivirusMetadataFields.AntivirusMetadata
@@ -21,7 +21,8 @@ class FileMetadataService(
     fileMetadataRepository: FileMetadataRepository,
     consignmentStatusService: ConsignmentStatusService,
     customMetadataService: CustomMetadataPropertiesService,
-    validateFileMetadataService: ValidateFileMetadataService
+    validateFileMetadataService: ValidateFileMetadataService,
+    fileStatusService: FileStatusService
 )(implicit val ec: ExecutionContext) {
 
   def getSumOfFileSizes(consignmentId: UUID): Future[Long] = fileMetadataRepository.getSumOfFileSizes(consignmentId)
@@ -54,30 +55,27 @@ class FileMetadataService(
     for {
       _ <- fileMetadataRepository.deleteFileMetadata(uniqueFileIds, distinctPropertyNames)
       addedRows <- fileMetadataRepository.addFileMetadata(generateFileMetadataInput(uniqueFileIds, distinctMetadataProperties, userId))
-      _ <- validateFileMetadataService.validateAndAddAdditionalMetadataStatuses(uniqueFileIds, distinctPropertyNames)
+      _ <- validateFileMetadataService.validateAdditionalMetadata(uniqueFileIds, distinctPropertyNames)
       _ <- consignmentStatusService.updateMetadataConsignmentStatus(consignmentId, List(DescriptiveMetadata, ClosureMetadata))
       metadataPropertiesAdded = addedRows.map(r => { FileMetadata(r.propertyname, r.value) }).toSet
     } yield BulkFileMetadata(uniqueFileIds.toSeq, metadataPropertiesAdded.toSeq)
   }
 
-  def addOrUpdateBulkFileMetadata(input: AddOrUpdateBulkFileMetadataInput, userId: UUID): Future[List[FileMetadataWithFileId]] = {
+  def addOrUpdateBulkFileMetadata(metadataInput: AddOrUpdateBulkFileMetadataInput, userId: UUID): Future[List[FileMetadataWithFileId]] = {
     for {
       customMetadata <- customMetadataService.getCustomMetadata
       protectedMetadata = customMetadata.filter(!_.editable).map(_.name)
-      _ = input.fileMetadata.map { addOrUpdateFileMetadata =>
+      _ = metadataInput.fileMetadata.map { addOrUpdateFileMetadata =>
         addOrUpdateFileMetadata.metadata.map { metadata =>
           if (protectedMetadata.contains(metadata.filePropertyName)) {
             throw InputDataException(s"Protected metadata property found: ${metadata.filePropertyName}")
           }
         }
       }
-      _ <- input.fileMetadata.map(fileMetadata => fileMetadataRepository.deleteFileMetadata(fileMetadata.fileId, fileMetadata.metadata.map(_.filePropertyName).toSet)).head
-      addedRows <- fileMetadataRepository.addFileMetadata(generateFileMetadataInput(input.fileMetadata, userId))
-      _ <- validateFileMetadataService.validateAndAddAdditionalMetadataStatuses(
-        fileIds = input.fileMetadata.map(_.fileId).toSet,
-        propertiesToValidate = input.fileMetadata.head.metadata.map(_.filePropertyName).toSet
-      )
-      _ <- consignmentStatusService.updateMetadataConsignmentStatus(input.consignmentId, List(DescriptiveMetadata, ClosureMetadata))
+      _ <- metadataInput.fileMetadata.map(fileMetadata => fileMetadataRepository.deleteFileMetadata(fileMetadata.fileId, fileMetadata.metadata.map(_.filePropertyName).toSet)).head
+      addedRows <- fileMetadataRepository.addFileMetadata(generateFileMetadataInput(metadataInput.fileMetadata, userId))
+      _ <- processFileMetadata(metadataInput)
+      _ <- consignmentStatusService.updateMetadataConsignmentStatus(metadataInput.consignmentId, List(DescriptiveMetadata, ClosureMetadata))
       metadataPropertiesAdded = addedRows.map(r => FileMetadataWithFileId(r.propertyname, r.fileid, r.value)).toList
     } yield metadataPropertiesAdded
   }
@@ -115,7 +113,7 @@ class FileMetadataService(
       }.toSeq
       _ <- fileMetadataRepository.deleteFileMetadata(fileIds, allPropertiesToDelete)
       _ <- fileMetadataRepository.addFileMetadata(metadataToReset)
-      _ <- validateFileMetadataService.validateAndAddAdditionalMetadataStatuses(fileIds, allPropertiesToDelete)
+      _ <- validateFileMetadataService.validateAdditionalMetadata(fileIds, allPropertiesToDelete)
       _ <- consignmentStatusService.updateMetadataConsignmentStatus(consignmentId, List(DescriptiveMetadata, ClosureMetadata))
     } yield DeleteFileMetadata(fileIds.toSeq, allPropertiesToDelete.toSeq)
   }
@@ -154,6 +152,17 @@ class FileMetadataService(
         fileId -> getFileMetadataValues(fileMetadata)
       }
     }
+
+  private def processFileMetadata(metadataInput: AddOrUpdateBulkFileMetadataInput): Future[Seq[FilestatusRow]] = {
+    if (metadataInput.skipValidation) {
+      fileStatusService.addAdditionalMetadataStatuses(metadataInput.fileMetadata)
+    } else {
+      validateFileMetadataService.validateAdditionalMetadata(
+        fileIds = metadataInput.fileMetadata.map(_.fileId).toSet,
+        propertiesToValidate = metadataInput.fileMetadata.head.metadata.map(_.filePropertyName).toSet
+      )
+    }
+  }
 }
 
 object FileMetadataService {

diff --git a/src/main/scala/uk/gov/nationalarchives/tdr/api/service/FileStatusService.scala b/src/main/scala/uk/gov/nationalarchives/tdr/api/service/FileStatusService.scala
@@ -3,14 +3,15 @@ package uk.gov.nationalarchives.tdr.api.service
 import uk.gov.nationalarchives.Tables.FilestatusRow
 import uk.gov.nationalarchives.tdr.api.db.repository.FileStatusRepository
 import uk.gov.nationalarchives.tdr.api.graphql.fields.ConsignmentFields._
-import uk.gov.nationalarchives.tdr.api.graphql.fields.FileStatusFields.{AddMultipleFileStatusesInput, FileStatus}
+import uk.gov.nationalarchives.tdr.api.graphql.fields.FileMetadataFields.AddOrUpdateFileMetadata
+import uk.gov.nationalarchives.tdr.api.graphql.fields.FileStatusFields.{AddFileStatusInput, AddMultipleFileStatusesInput, FileStatus}
 import uk.gov.nationalarchives.tdr.api.service.FileStatusService._
 
 import java.util.UUID
 import scala.concurrent.{ExecutionContext, Future}
 
-class FileStatusService(fileStatusRepository: FileStatusRepository)(implicit
-    val executionContext: ExecutionContext
+class FileStatusService(fileStatusRepository: FileStatusRepository, customMetadataService: CustomMetadataPropertiesService, displayPropertiesService: DisplayPropertiesService)(
+    implicit val executionContext: ExecutionContext
 ) {
 
   private def toFileStatuses(rows: Seq[FilestatusRow]): Seq[FileStatus] = {
@@ -21,6 +22,34 @@ class FileStatusService(fileStatusRepository: FileStatusRepository)(implicit
     fileStatusRepository.addFileStatuses(addMultipleFileStatusesInput.statuses).map(_.map(row => FileStatus(row.fileid, row.statustype, row.value)).toList)
   }
 
+  def addAdditionalMetadataStatuses(fileMetadataList: Seq[AddOrUpdateFileMetadata]): Future[Seq[FilestatusRow]] = {
+    for {
+      customMetadataFields <- customMetadataService.getCustomMetadata
+      propertyNames <- displayPropertiesService.getActiveDisplayPropertyNames
+      additionalMetadataStatuses = {
+        val additionalMetadataGroups = customMetadataService.toAdditionalMetadataFieldGroups(customMetadataFields.filter(p => propertyNames.contains(p.name)))
+        val metadataGroupsWithDefaultValues =
+          additionalMetadataGroups.map(p => p.groupName -> p.fields.map(field => field.name -> field.defaultValue.getOrElse("")).toMap).toMap
+
+        fileMetadataList.flatMap(fileMetadata => {
+          metadataGroupsWithDefaultValues.map { case (groupName, fields) =>
+            val hasDefaultValues = fields.forall(p => fileMetadata.metadata.find(_.filePropertyName == p._1).exists(_.value == p._2))
+            val status = if (hasDefaultValues) {
+              NotEntered
+            } else {
+              Completed
+            }
+            AddFileStatusInput(fileMetadata.fileId, groupName, status)
+          }
+        })
+      }
+      _ <- fileStatusRepository.deleteFileStatus(additionalMetadataStatuses.map(_.fileId).toSet, Set(ClosureMetadata, DescriptiveMetadata))
+      rows <- fileStatusRepository.addFileStatuses(additionalMetadataStatuses.toList)
+    } yield {
+      rows
+    }
+  }
+
   def getConsignmentFileProgress(consignmentId: UUID): Future[FileChecks] = {
     fileStatusRepository
       .getFileStatus(consignmentId, Set(FFID, ChecksumMatch, Antivirus))

diff --git a/src/main/scala/uk/gov/nationalarchives/tdr/api/service/ValidateFileMetadataService.scala b/src/main/scala/uk/gov/nationalarchives/tdr/api/service/ValidateFileMetadataService.scala
@@ -2,7 +2,6 @@ package uk.gov.nationalarchives.tdr.api.service
 
 import uk.gov.nationalarchives.Tables.FilestatusRow
 import uk.gov.nationalarchives.tdr.api.db.repository.{FileMetadataRepository, FileStatusRepository}
-import uk.gov.nationalarchives.tdr.api.graphql.fields.CustomMetadataFields.{CustomMetadataField, CustomMetadataValues}
 import uk.gov.nationalarchives.tdr.api.graphql.fields.FileStatusFields.AddFileStatusInput
 import uk.gov.nationalarchives.tdr.api.service.FileStatusService._
 import uk.gov.nationalarchives.tdr.api.utils.MetadataValidationUtils
@@ -18,28 +17,13 @@ class ValidateFileMetadataService(
     fileStatusRepository: FileStatusRepository
 )(implicit val ec: ExecutionContext) {
 
-  def toPropertyNames(fields: Seq[CustomMetadataField]): Set[String] = fields.map(_.name).toSet
-
-  def toAdditionalMetadataFieldGroups(fields: Seq[CustomMetadataField]): Seq[FieldGroup] = {
-    val closureFields = fields.filter(f => f.propertyGroup.contains("MandatoryClosure") || f.propertyGroup.contains("OptionalClosure"))
-    val descriptiveFields = fields.filter(f => f.propertyGroup.contains("OptionalMetadata"))
-    Seq(FieldGroup(ClosureMetadata, closureFields), FieldGroup(DescriptiveMetadata, descriptiveFields))
-  }
-
-  def toValueDependenciesGroups(field: CustomMetadataField): Seq[FieldGroup] = {
-    val values: List[CustomMetadataValues] = field.values
-    values.map(v => {
-      FieldGroup(v.value, v.dependencies)
-    })
-  }
-
-  def validateAndAddAdditionalMetadataStatuses(fileIds: Set[UUID], propertiesToValidate: Set[String]): Future[List[FilestatusRow]] = {
+  def validateAdditionalMetadata(fileIds: Set[UUID], propertiesToValidate: Set[String]): Future[List[FilestatusRow]] = {
     for {
       propertyNames <- displayPropertiesService.getActiveDisplayPropertyNames
       result <- {
         if (propertiesToValidate.exists(propertyNames.contains)) {
           for {
-            additionalMetadataStatuses <- validateAdditionalMetadata(fileIds, propertyNames)
+            additionalMetadataStatuses <- validate(fileIds, propertyNames)
             _ <- fileStatusRepository.deleteFileStatus(fileIds, Set(ClosureMetadata, DescriptiveMetadata))
             rows <- fileStatusRepository.addFileStatuses(additionalMetadataStatuses)
           } yield rows.toList
@@ -52,12 +36,12 @@ class ValidateFileMetadataService(
     }
   }
 
-  private def validateAdditionalMetadata(fileIds: Set[UUID], propertyNames: Seq[String]): Future[List[AddFileStatusInput]] = {
+  private def validate(fileIds: Set[UUID], propertyNames: Seq[String]): Future[List[AddFileStatusInput]] = {
     for {
       customMetadataFields <- customMetadataService.getCustomMetadata
       existingMetadataProperties <- fileMetadataRepository.getFileMetadata(None, Some(fileIds), Some(propertyNames.toSet))
     } yield {
-      val additionalMetadataGroups = toAdditionalMetadataFieldGroups(customMetadataFields.filter(p => propertyNames.contains(p.name)))
+      val additionalMetadataGroups = customMetadataService.toAdditionalMetadataFieldGroups(customMetadataFields.filter(p => propertyNames.contains(p.name)))
 
       val existingFileProperties =
         fileIds.map(fileId => fileId -> existingMetadataProperties.filter(_.fileid == fileId).groupBy(_.propertyname).map(p => p._1 -> p._2.map(_.value).mkString(","))).toMap
@@ -95,6 +79,4 @@ class ValidateFileMetadataService(
       }.toList
     }
   }
-
-  case class FieldGroup(groupName: String, fields: Seq[CustomMetadataField])
 }
diff --git a/...est/resources/json/addorupdatebulkfilemetadata_mutation_notfileowner_skip_validation.json b/...est/resources/json/addorupdatebulkfilemetadata_mutation_notfileowner_skip_validation.json
@@ -0,0 +1,51 @@
+{
+  "query": "mutation addOrUpdateBulkFileMetadata($addOrUpdateBulkFileMetadataInput: AddOrUpdateBulkFileMetadataInput!) {addOrUpdateBulkFileMetadata(addOrUpdateBulkFileMetadataInput:$addOrUpdateBulkFileMetadataInput) {filePropertyName,fileId,value}}",
+  "variables": {
+    "addOrUpdateBulkFileMetadataInput": {
+      "consignmentId": "eb197bfb-43f7-40ca-9104-8f6cbda88506",
+      "skipValidation": true,
+      "fileMetadata": [
+        {
+          "fileId": "51c55218-1322-4453-9ef8-2300ef1c0fef",
+          "metadata": [
+            {
+              "filePropertyName": "newProperty1",
+              "value": "value1"
+            },
+            {
+              "filePropertyName": "existingPropertyUpdated1",
+              "value": "newValue1"
+            }
+          ]
+        },
+        {
+          "fileId": "7076f399-b596-4161-a95d-e686c6435710",
+          "metadata": [
+            {
+              "filePropertyName": "newProperty1",
+              "value": "value1"
+            }
+          ]
+        },
+        {
+          "fileId": "373ce1c5-6e06-423d-8b86-ca5eaebef457",
+          "metadata": [
+            {
+              "filePropertyName": "existingPropertyUpdated1",
+              "value": "newValue1"
+            }
+          ]
+        },
+        {
+          "fileId": "5302acac-1396-44fe-9094-dc262414a03a",
+          "metadata": [
+            {
+              "filePropertyName": "existingPropertyUpdated1",
+              "value": "newValue1"
+            }
+          ]
+        }
+      ]
+    }
+  }
+}