merged main

.scalafmt.conf

@@ -1,4 +1,4 @@
-version = 3.8.3
+version = 3.8.6
 preset = default
 runner.dialect = scala213
 maxColumn = 180

project/Dependencies.scala

@@ -8,16 +8,16 @@ object Dependencies {
 
   lazy val scalaCsv = "com.github.tototoshi" %% "scala-csv" % "2.0.0"
   lazy val scalaTest = "org.scalatest" %% "scalatest" % "3.2.19"
-  lazy val metadataValidation = "uk.gov.nationalarchives" %% "tdr-metadata-validation" % "0.0.83" exclude ("uk.gov.nationalarchives", "da-metadata-schema_3")
-  lazy val schemaUtils = "uk.gov.nationalarchives" %% "tdr-schema-utils" % "0.0.83"
-  lazy val metadataSchema = "uk.gov.nationalarchives" % "da-metadata-schema_3" % "0.0.41"
-  lazy val generatedGraphql = "uk.gov.nationalarchives" %% "tdr-generated-graphql" % "0.0.400-SNAPSHOT"
-  lazy val graphqlClient = "uk.gov.nationalarchives" %% "tdr-graphql-client" % "0.0.194"
-  lazy val authUtils = "uk.gov.nationalarchives" %% "tdr-auth-utils" % "0.0.220"
+  lazy val metadataValidation = "uk.gov.nationalarchives" %% "tdr-metadata-validation" % "0.0.98" exclude ("uk.gov.nationalarchives", "da-metadata-schema_3")
+  lazy val schemaUtils = "uk.gov.nationalarchives" %% "tdr-schema-utils" % "0.0.98"
+  lazy val metadataSchema = "uk.gov.nationalarchives" % "da-metadata-schema_3" % "0.0.44"
+  lazy val generatedGraphql = "uk.gov.nationalarchives" %% "tdr-generated-graphql" % "0.0.402-SNAPSHOT"
+  lazy val graphqlClient = "uk.gov.nationalarchives" %% "tdr-graphql-client" % "0.0.205"
+  lazy val authUtils = "uk.gov.nationalarchives" %% "tdr-auth-utils" % "0.0.228"
   lazy val typeSafeConfig = "com.typesafe" % "config" % "1.4.3"
   lazy val awsLambda = "com.amazonaws" % "aws-lambda-java-core" % "1.2.3"
   lazy val awsLambdaJavaEvents = "com.amazonaws" % "aws-lambda-java-events" % "3.14.0"
-  lazy val s3Utils = "uk.gov.nationalarchives" %% "s3-utils" % "0.1.222"
+  lazy val s3Utils = "uk.gov.nationalarchives" %% "s3-utils" % "0.1.231"
   lazy val awsSsm = "software.amazon.awssdk" % "ssm" % "2.26.27"
   lazy val log4catsSlf4j = "org.typelevel" %% "log4cats-slf4j" % log4CatsVersion
   lazy val mockitoScala = "org.mockito" %% "mockito-scala" % mockitoScalaVersion

project/build.properties

@@ -1 +1 @@
-sbt.version=1.10.5
+sbt.version=1.10.7

project/plugins.sbt

@@ -1,2 +1,2 @@
-addSbtPlugin("com.eed3si9n" %% "sbt-assembly" % "2.3.0")
-addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.5.2")
+addSbtPlugin("com.eed3si9n" %% "sbt-assembly" % "2.3.1")
+addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.5.4")

src/main/resources/application.conf

@@ -5,8 +5,8 @@ api {
 auth {
   url = "https://auth.tdr-integration.nationalarchives.gov.uk/"
   url = ${?AUTH_URL}
-  clientId = "tdr-backend-checks"
-  clientSecretPath = "/intg/keycloak/backend_checks_client/secret"
+  clientId = "tdr-draft-metadata"
+  clientSecretPath = "/intg/keycloak/draft_metadata_client/secret"
   clientSecretPath = ${?CLIENT_SECRET_PATH}
   realm = "tdr"
 }

src/main/scala/uk/gov/nationalarchives/draftmetadatavalidator/GraphQlApi.scala

@@ -34,23 +34,29 @@ class GraphQlApi(
   def getCustomMetadata(consignmentId: UUID, clientSecret: String)(implicit executionContext: ExecutionContext): IO[List[cm.CustomMetadata]] = for {
     token <- keycloak.serviceAccountToken(clientId, clientSecret).toIO
     metadata <- customMetadataClient.getResult(token, cm.document, cm.Variables(consignmentId).some).toIO
-    data <- IO.fromOption(metadata.data)(new RuntimeException("No custom metadata definitions found"))
+    data <- IO.fromOption(metadata.data)(
+      new RuntimeException(metadata.errors.map(_.message).headOption.getOrElse("No custom metadata definitions found"))
+    )
   } yield data.customMetadata
 
   def updateConsignmentStatus(consignmentId: UUID, clientSecret: String, statusType: String, statusValue: String)(implicit executionContext: ExecutionContext): IO[Option[Int]] =
     for {
       token <- keycloak.serviceAccountToken(clientId, clientSecret).toIO
       metadata <- updateConsignmentStatus.getResult(token, ucs.document, ucs.Variables(ConsignmentStatusInput(consignmentId, statusType, statusValue.some)).some).toIO
-      data <- IO.fromOption(metadata.data)(new RuntimeException("Unable to update consignment status"))
+      data <- IO.fromOption(metadata.data)(
+        new RuntimeException(metadata.errors.map(_.message).headOption.getOrElse("Unable to update consignment status"))
+      )
     } yield data.updateConsignmentStatus
 
   def addOrUpdateBulkFileMetadata(consignmentId: UUID, clientSecret: String, fileMetadata: List[AddOrUpdateFileMetadata])(implicit
       executionContext: ExecutionContext
   ): IO[List[AddOrUpdateBulkFileMetadata]] =
     for {
       token <- keycloak.serviceAccountToken(clientId, clientSecret).toIO
-      metadata <- addOrUpdateBulkFileMetadata.getResult(token, afm.document, afm.Variables(AddOrUpdateBulkFileMetadataInput(consignmentId, fileMetadata)).some).toIO
-      data <- IO.fromOption(metadata.data)(new RuntimeException("Unable to add or update bulk file metadata"))
+      metadata <- addOrUpdateBulkFileMetadata.getResult(token, afm.document, afm.Variables(AddOrUpdateBulkFileMetadataInput(consignmentId, fileMetadata, Some(true))).some).toIO
+      data <- IO.fromOption(metadata.data)(
+        new RuntimeException(metadata.errors.map(_.message).headOption.getOrElse("Unable to add or update bulk file metadata"))
+      )
     } yield data.addOrUpdateBulkFileMetadata
 
   def getConsignmentFilesMetadata(consignmentId: UUID, clientSecret: String, databaseMetadataHeaders: List[String]): IO[Option[gcfm.Data]] = {

src/main/scala/uk/gov/nationalarchives/draftmetadatavalidator/Lambda.scala

@@ -3,9 +3,7 @@ package uk.gov.nationalarchives.draftmetadatavalidator
 import cats.effect.IO
 import cats.effect.kernel.Resource
 import cats.syntax.semigroup._
-import ValidationErrors._
-import com.amazonaws.services.lambda.runtime.events.APIGatewayProxyResponseEvent
-import com.amazonaws.services.lambda.runtime.{Context, RequestHandler}
+import com.amazonaws.services.lambda.runtime.Context
 import graphql.codegen.AddOrUpdateBulkFileMetadata.addOrUpdateBulkFileMetadata.AddOrUpdateBulkFileMetadata
 import graphql.codegen.AddOrUpdateBulkFileMetadata.{addOrUpdateBulkFileMetadata => afm}
 import graphql.codegen.GetConsignmentFilesMetadata.{getConsignmentFilesMetadata => gcfm}
@@ -26,6 +24,7 @@ import uk.gov.nationalarchives.aws.utils.s3.S3Clients._
 import uk.gov.nationalarchives.aws.utils.s3.S3Utils
 import uk.gov.nationalarchives.draftmetadatavalidator.ApplicationConfig._
 import uk.gov.nationalarchives.draftmetadatavalidator.Lambda.{ValidationExecutionError, ValidationParameters, getErrorFilePath, getFilePath}
+import uk.gov.nationalarchives.draftmetadatavalidator.ValidationErrors._
 import uk.gov.nationalarchives.draftmetadatavalidator.utils.{DependencyVersionReader, MetadataUtils}
 import uk.gov.nationalarchives.tdr.GraphQLClient
 import uk.gov.nationalarchives.tdr.keycloak.{KeycloakUtils, TdrKeycloakDeployment}
@@ -42,8 +41,9 @@ import java.util
 import java.util.{Properties, UUID}
 import scala.concurrent.ExecutionContext.Implicits.global
 import scala.io.Source
+import scala.jdk.CollectionConverters.MapHasAsJava
 
-class Lambda extends RequestHandler[java.util.Map[String, Object], APIGatewayProxyResponseEvent] {
+class Lambda {
 
   implicit val backend: SttpBackend[Identity, Any] = HttpURLConnectionBackend()
   implicit val keycloakDeployment: TdrKeycloakDeployment = TdrKeycloakDeployment(authUrl, "tdr", timeToLiveSecs)
@@ -66,7 +66,8 @@ class Lambda extends RequestHandler[java.util.Map[String, Object], APIGatewayPro
     updateSchemaLibraryVersionClient
   )
 
-  def handleRequest(input: java.util.Map[String, Object], context: Context): APIGatewayProxyResponseEvent = {
+  def handleRequest(input: java.util.Map[String, Object], context: Context): java.util.Map[String, Object] = {
+    val startTime = System.currentTimeMillis()
     val consignmentId = extractConsignmentId(input)
     val schemaToValidate: Set[JsonSchemaDefinition] = Set(BASE_SCHEMA, CLOSURE_SCHEMA_CLOSED, CLOSURE_SCHEMA_OPEN)
     val validationParameters: ValidationParameters = ValidationParameters(
@@ -78,7 +79,7 @@ class Lambda extends RequestHandler[java.util.Map[String, Object], APIGatewayPro
       requiredSchema = Some(REQUIRED_SCHEMA)
     )
 
-    val requestHandler: IO[APIGatewayProxyResponseEvent] = for {
+    val resultIO = for {
       fileIdData <- graphQlApi.getConsignmentFilesMetadata(
         consignmentId = UUID.fromString(consignmentId),
         clientSecret = getClientSecret(clientSecretPath, endpoint),
@@ -93,21 +94,14 @@ class Lambda extends RequestHandler[java.util.Map[String, Object], APIGatewayPro
       _ <- if (errorFileData.validationErrors.isEmpty) persistMetadata(validationParameters, clientToPersistenceId) else IO.unit
       _ <- updateConsignmentSchemaLibraryVersion(errorFileData, validationParameters)
       _ <- updateStatus(errorFileData, validationParameters)
-    } yield {
-      val response = new APIGatewayProxyResponseEvent()
-      response.setStatusCode(200)
-      response
-    }
+    } yield ()
 
-    requestHandler
-      .handleErrorWith(error => {
-        logger.error(s"Unexpected validation problem:${error.getMessage}")
-        val unexpectedFailureResponse = new APIGatewayProxyResponseEvent()
-        unexpectedFailureResponse.setStatusCode(500)
-        unexpectedFailureResponse.withBody(s"Unexpected validation problem:${error.getMessage}")
-        IO(unexpectedFailureResponse)
-      })
-      .unsafeRunSync()(cats.effect.unsafe.implicits.global)
+    logger.info(s"Metadata validation was run for $consignmentId")
+    resultIO.unsafeRunSync()(cats.effect.unsafe.implicits.global)
+    Map[String, Object](
+      "consignmentId" -> consignmentId,
+      "validationTime" -> s"${(System.currentTimeMillis() - startTime) / 1000.0} seconds"
+    ).asJava
   }
 
   private def doValidation(validationParameters: ValidationParameters, clientIdToPersistenceId: Map[String, UUID]): IO[ErrorFileData] = {

src/test/scala/uk/gov/nationalarchives/draftmetadatavalidator/GraphQlApiSpec.scala

@@ -17,6 +17,8 @@ import org.scalatest.flatspec.AnyFlatSpec
 import org.scalatest.matchers.should.Matchers
 import sangria.ast.Document
 import sttp.client3.{HttpURLConnectionBackend, Identity, SttpBackend}
+import uk.gov.nationalarchives.tdr.GraphQLClient.Extensions
+import uk.gov.nationalarchives.tdr.error.GraphQlError
 import uk.gov.nationalarchives.tdr.keycloak.{KeycloakUtils, TdrKeycloakDeployment}
 import uk.gov.nationalarchives.tdr.{GraphQLClient, GraphQlResponse}
 
@@ -47,19 +49,20 @@ class GraphQlApiSpec extends AnyFlatSpec with MockitoSugar with Matchers with Ei
 
   "getCustomMetadata" should "throw an exception when no custom metadata are found" in {
     val api = getGraphQLAPI
+    val graphQlError = GraphQLClient.Error("Unable to get custom metadata", Nil, Nil, Some(Extensions(Some("NOT_AUTHORISED"))))
 
     doAnswer(() => Future(new BearerAccessToken("token")))
       .when(keycloak)
       .serviceAccountToken[Identity](any[String], any[String])(any[SttpBackend[Identity, Any]], any[ClassTag[Identity[_]]], any[TdrKeycloakDeployment])
 
-    doAnswer(() => Future(GraphQlResponse[cm.Data](None, Nil)))
+    doAnswer(() => Future(GraphQlResponse[cm.Data](None, List(GraphQlError(graphQlError)))))
       .when(customMetadataClient)
       .getResult[Identity](any[BearerAccessToken], any[Document], any[Option[cm.Variables]])(any[SttpBackend[Identity, Any]], any[ClassTag[Identity[_]]])
 
     val exception = intercept[RuntimeException] {
       api.getCustomMetadata(consignmentId, "secret").unsafeRunSync()
     }
-    exception.getMessage should equal(s"No custom metadata definitions found")
+    exception.getMessage should equal(s"Unable to get custom metadata")
   }
 
   "getCustomMetadata" should "return the custom metadata" in {
@@ -80,12 +83,13 @@ class GraphQlApiSpec extends AnyFlatSpec with MockitoSugar with Matchers with Ei
 
   "updateConsignmentStatus" should "throw an exception when the api fails to update the consignment status" in {
     val api = getGraphQLAPI
+    val graphQlError = GraphQLClient.Error("Unable to update consignment status", Nil, Nil, Some(Extensions(Some("NOT_AUTHORISED"))))
 
     doAnswer(() => Future(new BearerAccessToken("token")))
       .when(keycloak)
       .serviceAccountToken[Identity](any[String], any[String])(any[SttpBackend[Identity, Any]], any[ClassTag[Identity[_]]], any[TdrKeycloakDeployment])
 
-    doAnswer(() => Future(GraphQlResponse[ucs.Data](None, Nil)))
+    doAnswer(() => Future(GraphQlResponse[ucs.Data](None, List(GraphQlError(graphQlError)))))
       .when(updateConsignmentStatusClient)
       .getResult[Identity](any[BearerAccessToken], any[Document], any[Option[ucs.Variables]])(any[SttpBackend[Identity, Any]], any[ClassTag[Identity[_]]])
 
@@ -115,12 +119,13 @@ class GraphQlApiSpec extends AnyFlatSpec with MockitoSugar with Matchers with Ei
 
   "addOrUpdateBulkFileMetadata" should "throw an exception when the api fails to add or update the file metadata" in {
     val api = getGraphQLAPI
+    val graphQlError = GraphQLClient.Error("Unable to add or update bulk file metadata", Nil, Nil, Some(Extensions(Some("NOT_AUTHORISED"))))
 
     doAnswer(() => Future(new BearerAccessToken("token")))
       .when(keycloak)
       .serviceAccountToken[Identity](any[String], any[String])(any[SttpBackend[Identity, Any]], any[ClassTag[Identity[_]]], any[TdrKeycloakDeployment])
 
-    doAnswer(() => Future(GraphQlResponse[ucs.Data](None, Nil)))
+    doAnswer(() => Future(GraphQlResponse[ucs.Data](None, List(GraphQlError(graphQlError)))))
       .when(addOrUpdateBulkFileMetadataClient)
       .getResult[Identity](any[BearerAccessToken], any[Document], any[Option[afm.Variables]])(any[SttpBackend[Identity, Any]], any[ClassTag[Identity[_]]])
 

src/test/scala/uk/gov/nationalarchives/draftmetadatavalidator/LambdaSpec.scala

@@ -1,7 +1,7 @@
 package uk.gov.nationalarchives.draftmetadatavalidator
 
 import com.amazonaws.services.lambda.runtime.Context
-import com.github.tomakehurst.wiremock.client.WireMock.{aResponse, get, put, urlEqualTo}
+import com.github.tomakehurst.wiremock.client.WireMock._
 import com.github.tomakehurst.wiremock.http.RequestMethod
 import com.github.tomakehurst.wiremock.stubbing.{ServeEvent, StubMapping}
 import graphql.codegen.UpdateConsignmentStatus.{updateConsignmentStatus => ucs}
@@ -11,9 +11,11 @@ import graphql.codegen.types.{AddOrUpdateBulkFileMetadataInput, AddOrUpdateFileM
 import io.circe.generic.auto._
 import io.circe.parser.decode
 import org.mockito.MockitoSugar.mock
-import org.scalatest.matchers.must.Matchers.{be, convertToAnyMustWrapper}
+import org.scalatest.matchers.must.Matchers.{be, convertToAnyMustWrapper, include}
 import org.scalatest.matchers.should.Matchers.{convertToAnyShouldWrapper, equal}
+import sttp.model.StatusCode
 import uk.gov.nationalarchives.draftmetadatavalidator.TestUtils.testFileIdMetadata
+import uk.gov.nationalarchives.tdr.error.HttpException
 
 import java.nio.file.{Files, Paths}
 import java.text.SimpleDateFormat
@@ -52,8 +54,7 @@ class LambdaSpec extends ExternalServicesSpec {
     mockS3GetResponse("sample.csv")
     mockS3ErrorFilePutResponse()
     val input = Map("consignmentId" -> consignmentId).asJava
-    val response = new Lambda().handleRequest(input, mockContext)
-    response.getStatusCode should equal(200)
+    new Lambda().handleRequest(input, mockContext)
 
     val s3Interactions: Iterable[ServeEvent] = wiremockS3.getAllServeEvents.asScala.filter(serveEvent => serveEvent.getRequest.getMethod == RequestMethod.PUT).toList
     s3Interactions.size shouldBe 1
@@ -73,7 +74,7 @@ class LambdaSpec extends ExternalServicesSpec {
 
     val addOrUpdateBulkFileMetadataEvent = getServeEvent("addOrUpdateBulkFileMetadata").get
     val request2: AddOrUpdateBulkFileMetadataGraphqlRequestData = decode[AddOrUpdateBulkFileMetadataGraphqlRequestData](addOrUpdateBulkFileMetadataEvent.getRequest.getBodyAsString)
-      .getOrElse(AddOrUpdateBulkFileMetadataGraphqlRequestData("", afm.Variables(AddOrUpdateBulkFileMetadataInput(UUID.fromString(consignmentId.toString), Nil))))
+      .getOrElse(AddOrUpdateBulkFileMetadataGraphqlRequestData("", afm.Variables(AddOrUpdateBulkFileMetadataInput(UUID.fromString(consignmentId.toString), Nil, None))))
     val addOrUpdateBulkFileMetadataInput = request2.variables.addOrUpdateBulkFileMetadataInput
 
     val updateConsignmentSchemaLibraryVersionEvent: ServeEvent = getServeEvent("updateSchemaLibraryVersion").get
@@ -84,13 +85,41 @@ class LambdaSpec extends ExternalServicesSpec {
 
     addOrUpdateBulkFileMetadataInput.fileMetadata.size should be(3)
     addOrUpdateBulkFileMetadataInput.fileMetadata should be(expectedFileMetadataInput(fileIdMetadata))
+    addOrUpdateBulkFileMetadataInput.skipValidation should be(Some(true))
 
     updateConsignmentStatusInput.statusType must be("DraftMetadata")
     updateConsignmentStatusInput.statusValue must be(Some("Completed"))
     updateConsignmentSchemaLibraryVersion.schemaLibraryVersion mustNot be("failed")
     updateConsignmentSchemaLibraryVersion.schemaLibraryVersion mustNot be("Failed to get schema library version")
   }
 
+  "handleRequest" should "return 500 response and throw an error message when a call to the api fails" in {
+    authOkJson()
+    val fileIdMetadata = testFileIdMetadata(Seq("test/test1.txt", "test/test2.txt", "test/test3.txt"))
+    graphqlOkJson(testFileIdMetadata = fileIdMetadata)
+    mockS3GetResponse("sample.csv")
+    mockS3ErrorFilePutResponse()
+
+    wiremockGraphqlServer.stubFor(
+      post(urlEqualTo(graphQlPath))
+        .withRequestBody(containing("addOrUpdateBulkFileMetadata"))
+        .willReturn(serverError().withBody("Failed to persist metadata"))
+    )
+
+    val input = Map("consignmentId" -> consignmentId).asJava
+    val exception = intercept[HttpException] {
+      new Lambda().handleRequest(input, mockContext)
+    }
+
+    val s3Interactions: Iterable[ServeEvent] = wiremockS3.getAllServeEvents.asScala
+      .filter(serveEvent => serveEvent.getRequest.getMethod == RequestMethod.PUT)
+      .toList
+    s3Interactions.size shouldBe 1
+
+    exception.code should equal(StatusCode(500))
+    exception.getMessage should include("Failed to persist metadata")
+  }
+
   "handleRequest" should "download the draft metadata csv file, check for schema errors and save error file with errors to s3" in {
     val fileIdMetadata = testFileIdMetadata(Seq("test/test1.txt", "test/test2.txt", "test/test3.txt"))
     authOkJson()
@@ -178,8 +207,7 @@ class LambdaSpec extends ExternalServicesSpec {
   private def checkFileError(errorFile: String) = {
     mockS3ErrorFilePutResponse()
     val input = Map("consignmentId" -> consignmentId).asJava
-    val response = new Lambda().handleRequest(input, mockContext)
-    response.getStatusCode should equal(200)
+    new Lambda().handleRequest(input, mockContext)
 
     val s3Interactions: Iterable[ServeEvent] = wiremockS3.getAllServeEvents.asScala.filter(serveEvent => serveEvent.getRequest.getMethod == RequestMethod.PUT).toList
     s3Interactions.size shouldBe 1