From ce7e5d3c00ad7f990bfc29d07ad483c8221d54dc Mon Sep 17 00:00:00 2001 From: Tomasz Rosiek Date: Fri, 28 Sep 2018 09:59:00 +0100 Subject: [PATCH 01/96] PLATY-736 Migrated buildsystem to the new build --- project/MicroService.scala | 19 +++++++++++++++---- project/MicroServiceBuild.scala | 9 --------- project/plugins.sbt | 20 ++++++++------------ 3 files changed, 23 insertions(+), 25 deletions(-) diff --git a/project/MicroService.scala b/project/MicroService.scala index 6e922cf..73e19e6 100644 --- a/project/MicroService.scala +++ b/project/MicroService.scala @@ -4,6 +4,12 @@ import play.sbt.routes.RoutesKeys.routesGenerator import sbt.Keys._ import sbt.Tests.{Group, SubProcess} import sbt._ +import uk.gov.hmrc.SbtArtifactory +import uk.gov.hmrc.sbtdistributables.SbtDistributablesPlugin +import uk.gov.hmrc.versioning.SbtGitVersioning +import uk.gov.hmrc.sbtdistributables.SbtDistributablesPlugin +import uk.gov.hmrc.versioning.SbtGitVersioning +import uk.gov.hmrc.versioning.SbtGitVersioning.autoImport.majorVersion trait MicroService { @@ -16,7 +22,9 @@ trait MicroService { val appName: String lazy val appDependencies: Seq[ModuleID] = ??? - lazy val plugins: Seq[Plugins] = Nil + lazy val plugins: Seq[Plugins] = Seq( + play.sbt.PlayScala, SbtAutoBuildPlugin, SbtGitVersioning, SbtDistributablesPlugin, SbtArtifactory + ) lazy val playSettings: Seq[Setting[_]] = Seq.empty routesGenerator := InjectedRoutesGenerator @@ -36,11 +44,14 @@ trait MicroService { } lazy val microservice = Project(appName, file(".")) - .enablePlugins(Seq(play.sbt.PlayScala) ++ plugins: _*) + .enablePlugins(plugins: _*) .settings(playSettings: _*) - .settings(scalaSettings ++ scoverageSettings: _*) - .settings(publishingSettings: _*) + .settings(scoverageSettings: _*) + .settings(playSettings: _*) + .settings(scalaSettings: _*) .settings(defaultSettings(): _*) + .settings(SbtDistributablesPlugin.publishingSettings: _*) + .settings(majorVersion := 0) .settings( PlayKeys.playDefaultPort := 9571, targetJvm := "jvm-1.8", diff --git a/project/MicroServiceBuild.scala b/project/MicroServiceBuild.scala index 512f172..c75374f 100644 --- a/project/MicroServiceBuild.scala +++ b/project/MicroServiceBuild.scala @@ -1,18 +1,9 @@ import sbt._ -import uk.gov.hmrc.SbtAutoBuildPlugin -import uk.gov.hmrc.sbtdistributables.SbtDistributablesPlugin -import uk.gov.hmrc.versioning.SbtGitVersioning object MicroServiceBuild extends Build with MicroService { val appName = "upscan-initiate" - override lazy val plugins: Seq[Plugins] = Seq( - SbtAutoBuildPlugin, - SbtGitVersioning, - SbtDistributablesPlugin - ) - override lazy val appDependencies: Seq[ModuleID] = AppDependencies() } diff --git a/project/plugins.sbt b/project/plugins.sbt index 69ab71a..b2da013 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -1,29 +1,25 @@ -credentials += Credentials(Path.userHome / ".sbt" / ".credentials") +import sbt._ -resolvers += Resolver.url("hmrc-sbt-plugin-releases", url("https://dl.bintray.com/hmrc/sbt-plugin-releases"))(Resolver.ivyStylePatterns) +resolvers += Resolver.url("HMRC Sbt Plugin Releases", url("https://dl.bintray.com/hmrc/sbt-plugin-releases"))(Resolver.ivyStylePatterns) +resolvers += "HMRC Releases" at "https://dl.bintray.com/hmrc/releases" -resolvers += "Typesafe Releases" at "http://repo.typesafe.com/typesafe/releases/" -resolvers += Resolver.url("scoverage-bintray", url("https://dl.bintray.com/sksamuel/sbt-plugins/"))(Resolver.ivyStylePatterns) - -addSbtPlugin("com.github.gseitz" % "sbt-release" % "0.8.3") +addSbtPlugin("com.github.gseitz" % "sbt-release" % "0.8.5") addSbtPlugin("com.typesafe.play" % "sbt-plugin" % "2.5.12") -addSbtPlugin("uk.gov.hmrc" % "sbt-auto-build" % "1.11.0") +addSbtPlugin("uk.gov.hmrc" % "sbt-auto-build" % "1.13.0") addSbtPlugin("uk.gov.hmrc" % "sbt-settings" % "3.8.0") -addSbtPlugin("uk.gov.hmrc" % "sbt-git-versioning" % "0.10.0") +addSbtPlugin("uk.gov.hmrc" % "sbt-artifactory" % "0.13.0") -addSbtPlugin("uk.gov.hmrc" % "sbt-distributables" % "1.1.0") +addSbtPlugin("uk.gov.hmrc" % "sbt-git-versioning" % "1.15.0") -addSbtPlugin("net.virtual-void" % "sbt-dependency-graph" % "0.8.2") +addSbtPlugin("uk.gov.hmrc" % "sbt-distributables" % "1.1.0") addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.3.5") -addSbtPlugin("com.typesafe.play" % "sbt-plugin" % "2.5.12") - addSbtPlugin("org.scalastyle" %% "scalastyle-sbt-plugin" % "0.8.0") addSbtPlugin("net.virtual-void" % "sbt-dependency-graph" % "0.8.2") From 44e918068997b3d105656f49fcd28093fa14ae2d Mon Sep 17 00:00:00 2001 From: Tomasz Rosiek Date: Fri, 26 Oct 2018 09:23:40 +0100 Subject: [PATCH 02/96] PLATY-788 Upgraded dependencies --- project/MicroServiceBuild.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/project/MicroServiceBuild.scala b/project/MicroServiceBuild.scala index c75374f..708b408 100644 --- a/project/MicroServiceBuild.scala +++ b/project/MicroServiceBuild.scala @@ -12,7 +12,7 @@ private object AppDependencies { import play.core.PlayVersion val compile = Seq( - "uk.gov.hmrc" %% "bootstrap-play-25" % "1.7.0", + "uk.gov.hmrc" %% "bootstrap-play-25" % "3.13.0", "com.amazonaws" % "aws-java-sdk-s3" % "1.11.261", "com.typesafe.akka" %% "akka-stream" % "2.5.6" ) @@ -23,8 +23,8 @@ private object AppDependencies { } private def commonTestDependencies(scope: String) = Seq( - "uk.gov.hmrc" %% "hmrctest" % "3.0.0" % scope, - "uk.gov.hmrc" %% "http-verbs-test" % "1.1.0" % scope, + "uk.gov.hmrc" %% "hmrctest" % "3.2.0" % scope, + "uk.gov.hmrc" %% "http-verbs-test" % "1.2.0" % scope, "org.scalatest" %% "scalatest" % "2.2.6" % scope, "org.pegdown" % "pegdown" % "1.6.0" % scope, "com.typesafe.play" %% "play-test" % PlayVersion.current % scope, From 3a33be094ef26d109578b0a1876cfbceaf638118 Mon Sep 17 00:00:00 2001 From: Tim Golding Date: Mon, 5 Nov 2018 15:32:08 +0000 Subject: [PATCH 03/96] PLATY-802: Update README w.r.t. expiration period of S3 pre-signed URLs --- README.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 75fb8cd..0103acf 100644 --- a/README.md +++ b/README.md @@ -365,11 +365,12 @@ These commands will give you an access to SBT shell where you can run the servic ## Appendix ### Quick reference figures -| Metric | Value | -| ------------- |:-------------: | -| Lifetime of POST URL | Up to 7 days | -| Callback request retry time | 60 seconds | -| Maximum callback notification retries | 30 | +| Metric | Value | Comments | +| ------------- |:-------------: | | +| Expiration of S3 upload pre-signed URL | Up to 7 days | A relatively long period, since we can't control exactly when users will initiate the upload process | +| Expiration of S3 download pre-signed URL (scanned docs) | Up to 1 day | Upscan is not intended as a storage solution for services | +| Callback request retry time | 60 seconds | | +| Maximum callback notification retries | 30 | | [[Back to the top]](#top) From e046b21ab03e63969dbe4b683a5ed5e0d4e982c5 Mon Sep 17 00:00:00 2001 From: Tim Golding Date: Mon, 5 Nov 2018 15:36:20 +0000 Subject: [PATCH 04/96] PLATY-802: Fixed table formatting in README. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 0103acf..9485394 100644 --- a/README.md +++ b/README.md @@ -366,7 +366,7 @@ These commands will give you an access to SBT shell where you can run the servic ### Quick reference figures | Metric | Value | Comments | -| ------------- |:-------------: | | +| ------------------------------------------------------- | -------------- |----------| | Expiration of S3 upload pre-signed URL | Up to 7 days | A relatively long period, since we can't control exactly when users will initiate the upload process | | Expiration of S3 download pre-signed URL (scanned docs) | Up to 1 day | Upscan is not intended as a storage solution for services | | Callback request retry time | 60 seconds | | From 310af16e63452c1150bb5a45a02ce09a2a32f9bb Mon Sep 17 00:00:00 2001 From: Tim Golding Date: Tue, 20 Nov 2018 15:54:29 +0000 Subject: [PATCH 05/96] PLATY-818: Upgrading dependencies. --- project/MicroServiceBuild.scala | 2 +- project/plugins.sbt | 11 +++++------ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/project/MicroServiceBuild.scala b/project/MicroServiceBuild.scala index 708b408..7d08445 100644 --- a/project/MicroServiceBuild.scala +++ b/project/MicroServiceBuild.scala @@ -12,7 +12,7 @@ private object AppDependencies { import play.core.PlayVersion val compile = Seq( - "uk.gov.hmrc" %% "bootstrap-play-25" % "3.13.0", + "uk.gov.hmrc" %% "bootstrap-play-25" % "3.15.0", "com.amazonaws" % "aws-java-sdk-s3" % "1.11.261", "com.typesafe.akka" %% "akka-stream" % "2.5.6" ) diff --git a/project/plugins.sbt b/project/plugins.sbt index b2da013..f7f1218 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -1,8 +1,7 @@ import sbt._ -resolvers += Resolver.url("HMRC Sbt Plugin Releases", url("https://dl.bintray.com/hmrc/sbt-plugin-releases"))(Resolver.ivyStylePatterns) -resolvers += "HMRC Releases" at "https://dl.bintray.com/hmrc/releases" - +resolvers += Resolver.bintrayRepo("hmrc", "sbt-plugin-releases") +resolvers += Resolver.bintrayRepo("hmrc", "releases") addSbtPlugin("com.github.gseitz" % "sbt-release" % "0.8.5") @@ -10,13 +9,13 @@ addSbtPlugin("com.typesafe.play" % "sbt-plugin" % "2.5.12") addSbtPlugin("uk.gov.hmrc" % "sbt-auto-build" % "1.13.0") -addSbtPlugin("uk.gov.hmrc" % "sbt-settings" % "3.8.0") +addSbtPlugin("uk.gov.hmrc" % "sbt-settings" % "3.9.0") -addSbtPlugin("uk.gov.hmrc" % "sbt-artifactory" % "0.13.0") +addSbtPlugin("uk.gov.hmrc" % "sbt-artifactory" % "0.14.0") addSbtPlugin("uk.gov.hmrc" % "sbt-git-versioning" % "1.15.0") -addSbtPlugin("uk.gov.hmrc" % "sbt-distributables" % "1.1.0") +addSbtPlugin("uk.gov.hmrc" % "sbt-distributables" % "1.2.0") addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.3.5") From 43c70f7a25238cf08508bfe1d8c256a496c77a08 Mon Sep 17 00:00:00 2001 From: Tim Golding Date: Thu, 22 Nov 2018 14:44:50 +0000 Subject: [PATCH 06/96] PLATY-818: Upgrading dependencies. --- project/plugins.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/plugins.sbt b/project/plugins.sbt index f7f1218..c111e29 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -1,6 +1,6 @@ import sbt._ -resolvers += Resolver.bintrayRepo("hmrc", "sbt-plugin-releases") +resolvers += Resolver.bintrayIvyRepo("hmrc", "sbt-plugin-releases") resolvers += Resolver.bintrayRepo("hmrc", "releases") addSbtPlugin("com.github.gseitz" % "sbt-release" % "0.8.5") From c91e9caea47640328d0d34d5e8f7a20f9aef4bba Mon Sep 17 00:00:00 2001 From: Tim Golding Date: Fri, 23 Nov 2018 15:02:19 +0000 Subject: [PATCH 07/96] PLATY-817: Adding file metadata for start/end processing times. --- app/UpscanInitiateModule.scala | 6 ++-- app/connectors/s3/S3UploadFormGenerator.scala | 28 ++++++++++--------- .../s3/S3UploadFormGeneratorProvider.scala | 8 ++++-- app/controllers/PrepareUploadController.scala | 11 ++++++-- app/domain/PrepareUploadService.scala | 14 ++++++---- conf/application-json-logger.xml | 2 +- .../s3/S3UploadFormGeneratorSpec.scala | 2 -- .../PrepareUploadControllerSpec.scala | 26 +++++++++-------- test/domain/PrepareUploadServiceSpec.scala | 18 +++++++----- 9 files changed, 67 insertions(+), 48 deletions(-) diff --git a/app/UpscanInitiateModule.scala b/app/UpscanInitiateModule.scala index 461e7c4..242499a 100644 --- a/app/UpscanInitiateModule.scala +++ b/app/UpscanInitiateModule.scala @@ -1,11 +1,13 @@ -import config.{PlayBasedServiceConfiguration, ServiceConfiguration} +import java.time.Clock +import config.{PlayBasedServiceConfiguration, ServiceConfiguration} import play.api.inject.{Binding, Module} import play.api.{Configuration, Environment} class UpscanInitiateModule extends Module { override def bindings(environment: Environment, configuration: Configuration): Seq[Binding[_]] = Seq( - bind[ServiceConfiguration].to[PlayBasedServiceConfiguration] + bind[ServiceConfiguration].to[PlayBasedServiceConfiguration], + bind[Clock].toInstance(Clock.systemDefaultZone()) ) } diff --git a/app/connectors/s3/S3UploadFormGenerator.scala b/app/connectors/s3/S3UploadFormGenerator.scala index 37b1452..87bc5a8 100644 --- a/app/connectors/s3/S3UploadFormGenerator.scala +++ b/app/connectors/s3/S3UploadFormGenerator.scala @@ -5,7 +5,7 @@ import java.time.format.DateTimeFormatter.ISO_INSTANT import java.time.{Instant, ZoneOffset} import domain.{UploadFormGenerator, UploadParameters} -import play.api.libs.json.{JsArray, Json} +import play.api.libs.json.{JsArray, JsValue, Json} final case class AwsCredentials( accessKeyId: String, @@ -58,17 +58,18 @@ class S3UploadFormGenerator( timeStamp: String, signingCredentials: String, encodedPolicy: String, - policySignature: String) = { + policySignature: String): Map[String, String] = { val fields = Map( - "x-amz-algorithm" -> "AWS4-HMAC-SHA256", - "x-amz-credential" -> signingCredentials, - "x-amz-date" -> timeStamp, - "policy" -> encodedPolicy, - "x-amz-signature" -> policySignature, - "acl" -> uploadParameters.acl, - "key" -> uploadParameters.objectKey, - "x-amz-meta-original-filename" -> "${filename}" + "x-amz-algorithm" -> "AWS4-HMAC-SHA256", + "x-amz-credential" -> signingCredentials, + "x-amz-date" -> timeStamp, + "policy" -> encodedPolicy, + "x-amz-signature" -> policySignature, + "acl" -> uploadParameters.acl, + "key" -> uploadParameters.objectKey, + "x-amz-meta-original-filename" -> "${filename}", + "x-amz-meta-upscan-initiate-response" -> currentTime().toString ) val sessionCredentials = securityToken.map(t => Map("x-amz-security-token" -> t)).getOrElse(Map.empty) @@ -81,7 +82,6 @@ class S3UploadFormGenerator( val contentTypeField = uploadParameters.expectedContentType.map(contentType => "Content-Type" -> contentType) fields ++ sessionCredentials ++ metadataFields ++ contentTypeField - } private def buildPolicy( @@ -92,9 +92,11 @@ class S3UploadFormGenerator( val securityTokenJson = securityToken.map(t => Json.obj("x-amz-security-token" -> t)).toList - val metadataJson = uploadParameters.additionalMetadata.map { + val metadataJson: Seq[JsValue] = uploadParameters.additionalMetadata.map { case (k, v) => Json.obj(s"x-amz-meta-$k" -> v) - }.toSeq :+ Json.arr("starts-with", "$x-amz-meta-original-filename", "") + }.toSeq :+ + Json.arr("starts-with", "$x-amz-meta-original-filename", "") :+ + Json.arr("starts-with", "$x-amz-meta-upscan-initiate-response", "") val contentTypeConstraintJson = uploadParameters.expectedContentType.map(contentType => Json.obj("Content-Type" -> contentType)) diff --git a/app/connectors/s3/S3UploadFormGeneratorProvider.scala b/app/connectors/s3/S3UploadFormGeneratorProvider.scala index 0a2e446..963e453 100644 --- a/app/connectors/s3/S3UploadFormGeneratorProvider.scala +++ b/app/connectors/s3/S3UploadFormGeneratorProvider.scala @@ -1,22 +1,24 @@ package connectors.s3 import config.ServiceConfiguration -import java.time.Instant +import java.time.{Clock, Instant} import domain.UploadFormGenerator import javax.inject.{Inject, Provider, Singleton} @Singleton -class S3UploadFormGeneratorProvider @Inject()(configuration: ServiceConfiguration) +class S3UploadFormGeneratorProvider @Inject()(configuration: ServiceConfiguration, clock: Clock) extends Provider[UploadFormGenerator] { import configuration._ + private val tick: () => Instant = () => Instant.now(clock) + override def get() = new S3UploadFormGenerator( AwsCredentials(accessKeyId, secretAccessKey, sessionToken), regionName = region, - currentTime = Instant.now + currentTime = tick ) } diff --git a/app/controllers/PrepareUploadController.scala b/app/controllers/PrepareUploadController.scala index 447f70b..aa50664 100644 --- a/app/controllers/PrepareUploadController.scala +++ b/app/controllers/PrepareUploadController.scala @@ -4,22 +4,25 @@ import config.ServiceConfiguration import domain._ import javax.inject.{Inject, Singleton} import java.net.URL +import java.time.{Clock, Instant} + import play.api.Logger import play.api.data.validation.ValidationError import play.api.libs.functional.syntax._ import play.api.libs.json.Reads._ import play.api.libs.json.{JsPath, Json, Writes, _} import play.api.mvc.{Action, Result} + import scala.concurrent.Future import scala.util.{Failure, Success, Try} import uk.gov.hmrc.play.bootstrap.controller.BaseController - import utils.UserAgentFilter @Singleton class PrepareUploadController @Inject()( prepareUploadService: PrepareUploadService, - override val configuration: ServiceConfiguration) + override val configuration: ServiceConfiguration, + clock: Clock) extends BaseController with UserAgentFilter { @@ -46,6 +49,8 @@ class PrepareUploadController @Inject()( def prepareUpload(): Action[JsValue] = Action.async(parse.json) { implicit request => + val receivedAt = Instant.now(clock) + onlyAllowedServices[JsValue] { (_, consumingService) => withJsonBody[UploadSettings] { (fileUploadDetails: UploadSettings) => @@ -56,7 +61,7 @@ class PrepareUploadController @Inject()( val sessionId = hc(request).sessionId.map(_.value).getOrElse("n/a") val requestId = hc(request).requestId.map(_.value).getOrElse("n/a") val result: PreparedUpload = - prepareUploadService.prepareUpload(fileUploadDetails, consumingService, requestId, sessionId) + prepareUploadService.prepareUpload(fileUploadDetails, consumingService, requestId, sessionId, receivedAt) Future.successful(Ok(Json.toJson(result))) } diff --git a/app/domain/PrepareUploadService.scala b/app/domain/PrepareUploadService.scala index 1a3fccf..999e4a4 100644 --- a/app/domain/PrepareUploadService.scala +++ b/app/domain/PrepareUploadService.scala @@ -6,23 +6,23 @@ import java.util.UUID import com.kenshoo.play.metrics.Metrics import config.ServiceConfiguration import javax.inject.{Inject, Singleton} - import org.slf4j.MDC import play.api.Logger + @Singleton class PrepareUploadService @Inject()( postSigner: UploadFormGenerator, configuration: ServiceConfiguration, metrics: Metrics) { - def prepareUpload(settings: UploadSettings, consumingService: String, requestId : String, sessionId: String): PreparedUpload = { + def prepareUpload(settings: UploadSettings, consumingService: String, requestId : String, sessionId: String, receivedAt: Instant): PreparedUpload = { val reference = generateReference() - val expiration = Instant.now().plus(configuration.fileExpirationPeriod) + val expiration = receivedAt.plus(configuration.fileExpirationPeriod) val result = PreparedUpload( reference = reference, - uploadRequest = generatePost(reference.value, expiration, settings, consumingService, requestId, sessionId)) + uploadRequest = generatePost(reference.value, expiration, settings, consumingService, requestId, sessionId, receivedAt)) try { MDC.put("file-reference", reference.value) @@ -45,7 +45,8 @@ class PrepareUploadService @Inject()( settings: UploadSettings, consumingService: String, requestId: String, - sessionId: String): UploadFormTemplate = { + sessionId: String, + receivedAt: Instant): UploadFormTemplate = { val minFileSize = settings.minimumFileSize.getOrElse(0) val maxFileSize = settings.maximumFileSize.getOrElse(globalFileSizeLimit) @@ -63,7 +64,8 @@ class PrepareUploadService @Inject()( "callback-url" -> settings.callbackUrl, "consuming-service" -> consumingService, "session-id" -> sessionId, - "request-id" -> requestId + "request-id" -> requestId, + "upscan-initiate-received" -> receivedAt.toString ), contentLengthRange = ContentLengthRange(minFileSize, maxFileSize), expectedContentType = settings.expectedContentType diff --git a/conf/application-json-logger.xml b/conf/application-json-logger.xml index cc0ebb4..5d7f9dc 100644 --- a/conf/application-json-logger.xml +++ b/conf/application-json-logger.xml @@ -9,7 +9,7 @@ - + diff --git a/test/connectors/s3/S3UploadFormGeneratorSpec.scala b/test/connectors/s3/S3UploadFormGeneratorSpec.scala index 838c502..8dc681e 100644 --- a/test/connectors/s3/S3UploadFormGeneratorSpec.scala +++ b/test/connectors/s3/S3UploadFormGeneratorSpec.scala @@ -10,8 +10,6 @@ import org.scalatest.mockito.MockitoSugar import org.scalatest.{GivenWhenThen, Matchers, WordSpec} import play.api.libs.json.{JsArray, JsValue, Json} -import scala.util.Try - class S3UploadFormGeneratorSpec extends WordSpec with GivenWhenThen with Matchers with MockitoSugar { "S3UploadFormGenerator" should { diff --git a/test/controllers/PrepareUploadControllerSpec.scala b/test/controllers/PrepareUploadControllerSpec.scala index 656e11f..2033a16 100644 --- a/test/controllers/PrepareUploadControllerSpec.scala +++ b/test/controllers/PrepareUploadControllerSpec.scala @@ -1,5 +1,7 @@ package controllers +import java.time.Clock + import akka.actor.ActorSystem import akka.stream.ActorMaterializer import config.ServiceConfiguration @@ -27,13 +29,15 @@ class PrepareUploadControllerSpec extends UnitSpec with Matchers with GivenWhenT implicit val timeout: akka.util.Timeout = 10 seconds + private val clock: Clock = Clock.systemDefaultZone() + "PrepareUploadController" should { val config = mock[ServiceConfiguration] Mockito.when(config.allowedUserAgents).thenReturn(List("VALID-AGENT")) Mockito.when(config.allowedCallbackProtocols).thenReturn(List("https")) "build and return upload URL if valid request with all data" in { - val controller = new PrepareUploadController(prepareUploadService, config) + val controller = new PrepareUploadController(prepareUploadService, config, clock) Given("there is a valid upload request with all data") @@ -68,7 +72,7 @@ class PrepareUploadControllerSpec extends UnitSpec with Matchers with GivenWhenT } "build and return upload URL if valid request with minimal data including session id and request id" in { - val controller = new PrepareUploadController(prepareUploadService, config) + val controller = new PrepareUploadController(prepareUploadService, config, clock) Given("there is a valid upload request with minimal data") @@ -96,7 +100,7 @@ class PrepareUploadControllerSpec extends UnitSpec with Matchers with GivenWhenT } "build and return upload URL if valid request with minimal data excluding session id and request id" in { - val controller = new PrepareUploadController(prepareUploadService, config) + val controller = new PrepareUploadController(prepareUploadService, config, clock) Given("there is a valid upload request with minimal data") @@ -125,7 +129,7 @@ class PrepareUploadControllerSpec extends UnitSpec with Matchers with GivenWhenT "return a bad request error if invalid request - wrong structure" in { - val controller = new PrepareUploadController(prepareUploadService, config) + val controller = new PrepareUploadController(prepareUploadService, config, clock) Given("there is an invalid upload request") @@ -143,7 +147,7 @@ class PrepareUploadControllerSpec extends UnitSpec with Matchers with GivenWhenT } "return a bad request error if invalid request - incorrect maximum file size " in { - val controller = new PrepareUploadController(prepareUploadService, config) + val controller = new PrepareUploadController(prepareUploadService, config, clock) Given("there is an invalid upload request") @@ -163,7 +167,7 @@ class PrepareUploadControllerSpec extends UnitSpec with Matchers with GivenWhenT } "return okay if service is allowed on whitelist " in { - val controller = new PrepareUploadController(prepareUploadService, config) + val controller = new PrepareUploadController(prepareUploadService, config, clock) Given("there is a valid upload request from a whitelisted service") @@ -200,7 +204,7 @@ class PrepareUploadControllerSpec extends UnitSpec with Matchers with GivenWhenT } "return a forbidden error if service is not whitelisted " in { - val controller = new PrepareUploadController(prepareUploadService, config) + val controller = new PrepareUploadController(prepareUploadService, config, clock) Given("there is a valid upload request from a non-whitelisted service") @@ -228,7 +232,7 @@ class PrepareUploadControllerSpec extends UnitSpec with Matchers with GivenWhenT } "allow https callback urls" in { - val controller = new PrepareUploadController(prepareUploadService, config) + val controller = new PrepareUploadController(prepareUploadService, config, clock) val result = controller.withAllowedCallbackProtocol("https://my.callback.url") { Future.successful(Ok) @@ -238,7 +242,7 @@ class PrepareUploadControllerSpec extends UnitSpec with Matchers with GivenWhenT } "disallow http callback urls" in { - val controller = new PrepareUploadController(prepareUploadService, config) + val controller = new PrepareUploadController(prepareUploadService, config, clock) val result = controller.withAllowedCallbackProtocol("http://my.callback.url") { Future.failed(new RuntimeException("This block should not have been invoked.")) @@ -249,7 +253,7 @@ class PrepareUploadControllerSpec extends UnitSpec with Matchers with GivenWhenT } "disallow invalidly formatted callback urls" in { - val controller = new PrepareUploadController(prepareUploadService, config) + val controller = new PrepareUploadController(prepareUploadService, config, clock) val result = controller.withAllowedCallbackProtocol("123") { Future.failed(new RuntimeException("This block should not have been invoked.")) @@ -263,7 +267,7 @@ class PrepareUploadControllerSpec extends UnitSpec with Matchers with GivenWhenT val service = mock[PrepareUploadService] Mockito.when(service.globalFileSizeLimit).thenReturn(1024) Mockito - .when(service.prepareUpload(any(), any(), any(), any())) + .when(service.prepareUpload(any(), any(), any(), any(), any())) .thenAnswer(new Answer[PreparedUpload]() { override def answer(invocationOnMock: InvocationOnMock): PreparedUpload = { val settings = invocationOnMock.getArgument[UploadSettings](0) diff --git a/test/domain/PrepareUploadServiceSpec.scala b/test/domain/PrepareUploadServiceSpec.scala index ebdf483..ba91baa 100644 --- a/test/domain/PrepareUploadServiceSpec.scala +++ b/test/domain/PrepareUploadServiceSpec.scala @@ -1,6 +1,7 @@ package domain import java.time +import java.time.Instant import com.codahale.metrics.MetricRegistry import com.kenshoo.play.metrics.Metrics @@ -54,6 +55,8 @@ class PrepareUploadServiceSpec extends UnitSpec with Matchers with GivenWhenThen override def buildEndpoint(bucketName: String): String = s"$bucketName.s3" } + val receivedAt: Instant = Instant.now() + "S3 Upload Service" should { def service(metrics: Metrics) = new PrepareUploadService(s3PostSigner, serviceConfiguration, metrics) @@ -74,7 +77,7 @@ class PrepareUploadServiceSpec extends UnitSpec with Matchers with GivenWhenThen When("we setup the upload") - val result = service(metrics).prepareUpload(uploadSettings, "PrepareUploadServiceSpec", "some-request-id", "some-session-id") + val result = service(metrics).prepareUpload(uploadSettings, "PrepareUploadServiceSpec", "some-request-id", "some-session-id", receivedAt) Then("proper upload request form definition should be returned") @@ -85,10 +88,11 @@ class PrepareUploadServiceSpec extends UnitSpec with Matchers with GivenWhenThen "x-amz-meta-callback-url" -> callbackUrl, "x-amz-meta-consuming-service" -> "PrepareUploadServiceSpec", "x-amz-meta-session-id" -> "some-session-id", - "x-amz-meta-request-id" -> "some-request-id", + "x-amz-meta-request-id" -> "some-request-id", "minSize" -> "0", "maxSize" -> "1024", - "Content-Type" -> "application/xml" + "Content-Type" -> "application/xml", + "x-amz-meta-upscan-initiate-received" -> receivedAt.toString ) And("uploadInitiated counter has been incremented") @@ -113,7 +117,7 @@ class PrepareUploadServiceSpec extends UnitSpec with Matchers with GivenWhenThen When("we setup the upload") - val result = service(metrics).prepareUpload(uploadSettings, "PrepareUploadServiceSpec", "some-request-id", "some-session-id") + val result = service(metrics).prepareUpload(uploadSettings, "PrepareUploadServiceSpec", "some-request-id", "some-session-id", receivedAt) Then("upload request should contain requested min/max size") @@ -140,7 +144,7 @@ class PrepareUploadServiceSpec extends UnitSpec with Matchers with GivenWhenThen Then("an exception should be thrown") val thrown = the[IllegalArgumentException] thrownBy service(metrics) - .prepareUpload(uploadSettings, "PrepareUploadServiceSpec", "some-request-id", "some-session-id") + .prepareUpload(uploadSettings, "PrepareUploadServiceSpec", "some-request-id", "some-session-id", receivedAt) thrown.getMessage should include("Minimum file size is less than 0") metrics.defaultRegistry.counter("uploadInitiated").getCount shouldBe 0 @@ -166,7 +170,7 @@ class PrepareUploadServiceSpec extends UnitSpec with Matchers with GivenWhenThen Then("an exception should be thrown") val thrown = the[IllegalArgumentException] thrownBy service(metrics) - .prepareUpload(uploadSettings, "PrepareUploadServiceSpec", "some-request-id", "some-session-id") + .prepareUpload(uploadSettings, "PrepareUploadServiceSpec", "some-request-id", "some-session-id", receivedAt) thrown.getMessage should include("Maximum file size is greater than global maximum file size") metrics.defaultRegistry.counter("uploadInitiated").getCount shouldBe 0 @@ -191,7 +195,7 @@ class PrepareUploadServiceSpec extends UnitSpec with Matchers with GivenWhenThen Then("an exception should be thrown") val thrown = the[IllegalArgumentException] thrownBy service(metrics) - .prepareUpload(uploadSettings, "PrepareUploadServiceSpec", "some-request-id", "some-session-id") + .prepareUpload(uploadSettings, "PrepareUploadServiceSpec", "some-request-id", "some-session-id", receivedAt) thrown.getMessage should include("Minimum file size is greater than maximum file size") metrics.defaultRegistry.counter("uploadInitiated").getCount shouldBe 0 From 6f1db3344d5b2e9c07762f33de0f55d4ba8d5b14 Mon Sep 17 00:00:00 2001 From: Tomasz Rosiek <7059542+tomasz-rosiek@users.noreply.github.com> Date: Mon, 18 Feb 2019 13:02:05 +0000 Subject: [PATCH 08/96] PLATOPS-1850 Upgraded dependencies --- project/MicroServiceBuild.scala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/project/MicroServiceBuild.scala b/project/MicroServiceBuild.scala index 7d08445..b55e34d 100644 --- a/project/MicroServiceBuild.scala +++ b/project/MicroServiceBuild.scala @@ -12,8 +12,8 @@ private object AppDependencies { import play.core.PlayVersion val compile = Seq( - "uk.gov.hmrc" %% "bootstrap-play-25" % "3.15.0", - "com.amazonaws" % "aws-java-sdk-s3" % "1.11.261", + "uk.gov.hmrc" %% "bootstrap-play-25" % "4.9.0", + "com.amazonaws" % "aws-java-sdk-s3" % "1.11.500", "com.typesafe.akka" %% "akka-stream" % "2.5.6" ) @@ -23,8 +23,8 @@ private object AppDependencies { } private def commonTestDependencies(scope: String) = Seq( - "uk.gov.hmrc" %% "hmrctest" % "3.2.0" % scope, - "uk.gov.hmrc" %% "http-verbs-test" % "1.2.0" % scope, + "uk.gov.hmrc" %% "hmrctest" % "3.3.0" % scope, + "uk.gov.hmrc" %% "http-verbs-test" % "1.3.0" % scope, "org.scalatest" %% "scalatest" % "2.2.6" % scope, "org.pegdown" % "pegdown" % "1.6.0" % scope, "com.typesafe.play" %% "play-test" % PlayVersion.current % scope, From 9744358c823e0f816a3529d0dad25b854aab434d Mon Sep 17 00:00:00 2001 From: Tomasz Rosiek <7059542+tomasz-rosiek@users.noreply.github.com> Date: Fri, 1 Mar 2019 15:26:49 +0000 Subject: [PATCH 09/96] PLATOPS-BAU: Enforce repo yaml privacy key. --- repository.yaml | 1 + 1 file changed, 1 insertion(+) create mode 100644 repository.yaml diff --git a/repository.yaml b/repository.yaml new file mode 100644 index 0000000..f7c6ae2 --- /dev/null +++ b/repository.yaml @@ -0,0 +1 @@ +repoVisibility: public_0C3F0CE3E6E6448FAD341E7BFA50FCD333E06A20CFF05FCACE61154DDBBADF71 From 5f67e45564b2b2f90d9f3cab63d3e0d45133481e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Miguel?= <474643+jm2dev@users.noreply.github.com> Date: Tue, 9 Apr 2019 16:34:21 +0100 Subject: [PATCH 10/96] PLATOPS-1908 sbt plugin upgrade. --- project/plugins.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/plugins.sbt b/project/plugins.sbt index c111e29..cd7f59f 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -5,7 +5,7 @@ resolvers += Resolver.bintrayRepo("hmrc", "releases") addSbtPlugin("com.github.gseitz" % "sbt-release" % "0.8.5") -addSbtPlugin("com.typesafe.play" % "sbt-plugin" % "2.5.12") +addSbtPlugin("com.typesafe.play" % "sbt-plugin" % "2.5.19") addSbtPlugin("uk.gov.hmrc" % "sbt-auto-build" % "1.13.0") From 532ec7abe40d0d4372cee77025fd3aa5eaf322c3 Mon Sep 17 00:00:00 2001 From: Christopher Turner <23338096+christopherjturner@users.noreply.github.com> Date: Tue, 16 Apr 2019 13:35:53 +0100 Subject: [PATCH 11/96] BDOG-189 added support for success redirect --- app/connectors/s3/S3UploadFormGenerator.scala | 4 +- app/controllers/PrepareUploadController.scala | 15 +++---- app/domain/UploadFormGenerator.scala | 3 +- app/domain/model.scala | 3 +- .../s3/S3UploadFormGeneratorSpec.scala | 33 +++++++++++++++ .../PrepareUploadControllerSpec.scala | 41 ++++++++++++++++++- 6 files changed, 88 insertions(+), 11 deletions(-) diff --git a/app/connectors/s3/S3UploadFormGenerator.scala b/app/connectors/s3/S3UploadFormGenerator.scala index 87bc5a8..3260f5a 100644 --- a/app/connectors/s3/S3UploadFormGenerator.scala +++ b/app/connectors/s3/S3UploadFormGenerator.scala @@ -81,7 +81,9 @@ class S3UploadFormGenerator( val contentTypeField = uploadParameters.expectedContentType.map(contentType => "Content-Type" -> contentType) - fields ++ sessionCredentials ++ metadataFields ++ contentTypeField + val successRedirect = uploadParameters.successRedirect.map(url => Map("success_action_redirect" -> url)).getOrElse(Map.empty) + + fields ++ sessionCredentials ++ metadataFields ++ contentTypeField ++ successRedirect } private def buildPolicy( diff --git a/app/controllers/PrepareUploadController.scala b/app/controllers/PrepareUploadController.scala index aa50664..5c405e0 100644 --- a/app/controllers/PrepareUploadController.scala +++ b/app/controllers/PrepareUploadController.scala @@ -30,13 +30,14 @@ class PrepareUploadController @Inject()( (JsPath \ "callbackUrl").read[String] and (JsPath \ "minimumFileSize").readNullable[Int](min(0)) and (JsPath \ "maximumFileSize").readNullable[Int](min(0) keepAnd max(prepareUploadService.globalFileSizeLimit + 1)) and - (JsPath \ "expectedContentType").readNullable[String] + (JsPath \ "expectedContentType").readNullable[String] and + (JsPath \ "successRedirect").readNullable[String] + )(UploadSettings.apply _) .filter(ValidationError("Maximum file size must be equal or greater than minimum file size"))( settings => settings.minimumFileSize.getOrElse(0) <= settings.maximumFileSize.getOrElse( - prepareUploadService.globalFileSizeLimit) - ) + prepareUploadService.globalFileSizeLimit)) implicit val uploadFormTemplateWrites: Writes[UploadFormTemplate] = Json.writes[UploadFormTemplate] @@ -53,15 +54,15 @@ class PrepareUploadController @Inject()( onlyAllowedServices[JsValue] { (_, consumingService) => - withJsonBody[UploadSettings] { (fileUploadDetails: UploadSettings) => + withJsonBody[UploadSettings] { (uploadSettings: UploadSettings) => - withAllowedCallbackProtocol(fileUploadDetails.callbackUrl) { - Logger.debug(s"Processing request: [$fileUploadDetails].") + withAllowedCallbackProtocol(uploadSettings.callbackUrl) { + Logger.debug(s"Processing request: [$uploadSettings].") val sessionId = hc(request).sessionId.map(_.value).getOrElse("n/a") val requestId = hc(request).requestId.map(_.value).getOrElse("n/a") val result: PreparedUpload = - prepareUploadService.prepareUpload(fileUploadDetails, consumingService, requestId, sessionId, receivedAt) + prepareUploadService.prepareUpload(uploadSettings, consumingService, requestId, sessionId, receivedAt) Future.successful(Ok(Json.toJson(result))) } diff --git a/app/domain/UploadFormGenerator.scala b/app/domain/UploadFormGenerator.scala index 7e156d4..5970d94 100644 --- a/app/domain/UploadFormGenerator.scala +++ b/app/domain/UploadFormGenerator.scala @@ -17,5 +17,6 @@ case class UploadParameters( acl: String, additionalMetadata: Map[String, String], contentLengthRange: ContentLengthRange, - expectedContentType: Option[String] + expectedContentType: Option[String], + successRedirect: Option[String] = None ) diff --git a/app/domain/model.scala b/app/domain/model.scala index f198939..dac749c 100644 --- a/app/domain/model.scala +++ b/app/domain/model.scala @@ -4,7 +4,8 @@ case class UploadSettings( callbackUrl: String, minimumFileSize: Option[Int], maximumFileSize: Option[Int], - expectedContentType: Option[String]) + expectedContentType: Option[String], + successRedirect: Option[String] = None) case class UploadFormTemplate(href: String, fields: Map[String, String]) diff --git a/test/connectors/s3/S3UploadFormGeneratorSpec.scala b/test/connectors/s3/S3UploadFormGeneratorSpec.scala index 8dc681e..61a7836 100644 --- a/test/connectors/s3/S3UploadFormGeneratorSpec.scala +++ b/test/connectors/s3/S3UploadFormGeneratorSpec.scala @@ -88,6 +88,39 @@ class S3UploadFormGeneratorSpec extends WordSpec with GivenWhenThen with Matcher result("x-amz-security-token") shouldBe "session-token" result("x-amz-meta-original-filename") shouldBe "${filename}" } + + "generate a signed link with a redirect" in { + + Given("there is a properly configured form generator with AWS credentials") + val credentials = AwsCredentials("accessKeyId", "secretKey", Some("session-token")) + val regionName = "us-east-1" + val currentTime = () => Instant.parse("1997-07-16T19:20:30Z") + val policySigner = mock[PolicySigner] + val testSignature = "test-signature" + + when(policySigner.signPolicy(any(), any(), any(), any())).thenReturn(testSignature) + + val generator = new S3UploadFormGenerator(credentials, regionName, currentTime, policySigner) + + And("there are valid upload parameters") + val expirationTimestamp = "1997-07-16T19:20:40Z" + val uploadParameters = + UploadParameters( + expirationDateTime = Instant.parse(expirationTimestamp), + bucketName = "test-bucket", + objectKey = "test-key", + acl = "private", + additionalMetadata = Map("key1" -> "value1"), + contentLengthRange = ContentLengthRange(0, 1024), + expectedContentType = Some("application/xml"), + successRedirect = Some("http://test.server/success") + ) + + When("form fields are generated") + val result = generator.generateFormFields(uploadParameters) + + result("success_action_redirect") shouldBe "http://test.server/success" + } } def decodePolicyFormResult(base64EncodedPolicy: String): JsValue = { diff --git a/test/controllers/PrepareUploadControllerSpec.scala b/test/controllers/PrepareUploadControllerSpec.scala index 2033a16..4ffbc7e 100644 --- a/test/controllers/PrepareUploadControllerSpec.scala +++ b/test/controllers/PrepareUploadControllerSpec.scala @@ -71,6 +71,43 @@ class PrepareUploadControllerSpec extends UnitSpec with Matchers with GivenWhenT )) } + "build and return upload URL if valid request with redirect on success url" in { + val controller = new PrepareUploadController(prepareUploadService, config, clock) + + Given("there is a valid upload request with all data") + + val request: FakeRequest[JsValue] = FakeRequest() + .withHeaders(("User-Agent", "VALID-AGENT"), ("x-session-id", "some-session-id"), ("x-request-id", "some-request-id")) + .withBody( + Json.obj( + "id" -> "1", + "callbackUrl" -> "https://www.example.com", + "successRedirect" -> "https://www.example.com/nextpage", + "minimumFileSize" -> 0, + "maximumFileSize" -> 1024)) + + When("upload initiation has been requested") + + val result = controller.prepareUpload()(request) + + Then("service returns valid response with reference and template of upload form") + + withClue(Helpers.contentAsString(result)) { status(result) shouldBe 200 } + val json = Helpers.contentAsJson(result) + json shouldBe Json.obj( + "reference" -> "TEST", + "uploadRequest" -> Json.obj( + "href" -> "https://www.example.com", + "fields" -> Json.obj( + "minFileSize" -> "0", + "maxFileSize" -> "1024", + "sessionId" -> "some-session-id", + "requestId" -> "some-request-id", + "success_action_redirect" -> "https://www.example.com/nextpage" + ) + )) + } + "build and return upload URL if valid request with minimal data including session id and request id" in { val controller = new PrepareUploadController(prepareUploadService, config, clock) @@ -281,7 +318,9 @@ class PrepareUploadControllerSpec extends UnitSpec with Matchers with GivenWhenT settings.minimumFileSize.map(s => Map("minFileSize" -> s.toString).head) ++ settings.maximumFileSize.map(s => Map("maxFileSize" -> s.toString).head) ++ Map("sessionId" -> sessionId) ++ - Map("requestId" -> requestId) + Map("requestId" -> requestId) ++ + settings.successRedirect.map(url => Map("success_action_redirect" -> url)).getOrElse(Map.empty) + ) ) } From eaac9cdc0b29eb694dc950e9a7713ff7d4fdf32f Mon Sep 17 00:00:00 2001 From: Tomasz Rosiek <7059542+tomasz-rosiek@users.noreply.github.com> Date: Wed, 17 Apr 2019 09:26:59 +0100 Subject: [PATCH 12/96] PLATOPS-BAU Fixed documentation of outputs returned by upscan-notify in case virus has been found --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 9485394..4763dc6 100644 --- a/README.md +++ b/README.md @@ -204,7 +204,7 @@ of the file, not to the name (if user uploads PDF document named `data.png`, it The list of failure reasons is as follows: -- `QUARANTINED` - the file has failed virus scanning +- `QUARANTINE` - the file has failed virus scanning - `REJECTED` - the file is not of an allowed file type - `UNKNOWN` - there is some other problem with the file @@ -216,7 +216,7 @@ These reasons form one of the following JSON responses sent to the callback URL: "reference" : "11370e18-6e24-453e-b45a-76d3e32ea33d", "fileStatus" : "FAILED", "failureDetails": { - "failureReason": "QUARANTINED", + "failureReason": "QUARANTINE", "message": "e.g. This file has a virus" } } From 67d49db348f2b6388222a8c39757112089f995d0 Mon Sep 17 00:00:00 2001 From: Christopher Turner <23338096+christopherjturner@users.noreply.github.com> Date: Thu, 18 Apr 2019 10:13:24 +0100 Subject: [PATCH 13/96] BDOG-189 updated readme parameter list --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 9485394..eacb68b 100644 --- a/README.md +++ b/README.md @@ -100,7 +100,7 @@ Meaning of parameters: |callbackUrl |Url that will be called to report the outcome of file checking and upload, including retrieval details if successful. Notification format is detailed further down in this file. Must be https.| yes| |minimumFileSize|Minimum file size (in Bytes). Default is 0.|no| |maximumFileSize|Maximum file size (in Bytes). Cannot be greater than 100MB. Default is 100MB.|no| - +|successRedirect|Url to redirect to after file has been successfully uploaded.|no| The request has to include the following HTTP headers: | Header name|Description|Required| From 6cb65477f9bc472f54cdbcf37a86f1d66f52dacc Mon Sep 17 00:00:00 2001 From: Christopher Turner <23338096+christopherjturner@users.noreply.github.com> Date: Thu, 18 Apr 2019 16:14:51 +0100 Subject: [PATCH 14/96] BDOG-189 fixed tests --- app/domain/PrepareUploadService.scala | 3 +- app/domain/UploadFormGenerator.scala | 2 +- .../s3/S3UploadFormGeneratorSpec.scala | 3 +- test/domain/PrepareUploadServiceSpec.scala | 45 ++++++++++++++++++- 4 files changed, 49 insertions(+), 4 deletions(-) diff --git a/app/domain/PrepareUploadService.scala b/app/domain/PrepareUploadService.scala index 999e4a4..45e98ef 100644 --- a/app/domain/PrepareUploadService.scala +++ b/app/domain/PrepareUploadService.scala @@ -68,7 +68,8 @@ class PrepareUploadService @Inject()( "upscan-initiate-received" -> receivedAt.toString ), contentLengthRange = ContentLengthRange(minFileSize, maxFileSize), - expectedContentType = settings.expectedContentType + expectedContentType = settings.expectedContentType, + successRedirect = settings.successRedirect ) val form = postSigner.generateFormFields(uploadParameters) diff --git a/app/domain/UploadFormGenerator.scala b/app/domain/UploadFormGenerator.scala index 5970d94..4dfa7b2 100644 --- a/app/domain/UploadFormGenerator.scala +++ b/app/domain/UploadFormGenerator.scala @@ -18,5 +18,5 @@ case class UploadParameters( additionalMetadata: Map[String, String], contentLengthRange: ContentLengthRange, expectedContentType: Option[String], - successRedirect: Option[String] = None + successRedirect: Option[String] ) diff --git a/test/connectors/s3/S3UploadFormGeneratorSpec.scala b/test/connectors/s3/S3UploadFormGeneratorSpec.scala index 61a7836..b049872 100644 --- a/test/connectors/s3/S3UploadFormGeneratorSpec.scala +++ b/test/connectors/s3/S3UploadFormGeneratorSpec.scala @@ -38,7 +38,8 @@ class S3UploadFormGeneratorSpec extends WordSpec with GivenWhenThen with Matcher acl = "private", additionalMetadata = Map("key1" -> "value1"), contentLengthRange = ContentLengthRange(0, 1024), - expectedContentType = Some("application/xml") + expectedContentType = Some("application/xml"), + None ) When("form fields are generated") diff --git a/test/domain/PrepareUploadServiceSpec.scala b/test/domain/PrepareUploadServiceSpec.scala index ba91baa..621fe02 100644 --- a/test/domain/PrepareUploadServiceSpec.scala +++ b/test/domain/PrepareUploadServiceSpec.scala @@ -50,7 +50,8 @@ class PrepareUploadServiceSpec extends UnitSpec with Matchers with GivenWhenThen uploadParameters.additionalMetadata.map { case (k, v) => s"x-amz-meta-$k" -> v } ++ uploadParameters.expectedContentType.map { contentType => "Content-Type" -> contentType - } + } ++ + uploadParameters.successRedirect.map { "success_redirect_url" -> _} override def buildEndpoint(bucketName: String): String = s"$bucketName.s3" } @@ -201,6 +202,48 @@ class PrepareUploadServiceSpec extends UnitSpec with Matchers with GivenWhenThen metrics.defaultRegistry.counter("uploadInitiated").getCount shouldBe 0 } + "create post form that allows to upload the file with redirect on success" in { + + val metrics = metricsStub() + + Given("there are have valid upload settings") + + val callbackUrl = "http://www.callback.com" + + val uploadSettings = UploadSettings( + callbackUrl = callbackUrl, + minimumFileSize = None, + maximumFileSize = None, + expectedContentType = Some("application/xml"), + successRedirect = Some("https://new.service/page1") + ) + + When("we setup the upload") + + val result = service(metrics).prepareUpload(uploadSettings, "PrepareUploadServiceSpec", "some-request-id", "some-session-id", receivedAt) + + Then("proper upload request form definition should be returned") + + result.uploadRequest.href shouldBe s"${serviceConfiguration.inboundBucketName}.s3" + result.uploadRequest.fields shouldBe Map( + "bucket" -> serviceConfiguration.inboundBucketName, + "key" -> result.reference.value, + "x-amz-meta-callback-url" -> callbackUrl, + "x-amz-meta-consuming-service" -> "PrepareUploadServiceSpec", + "x-amz-meta-session-id" -> "some-session-id", + "x-amz-meta-request-id" -> "some-request-id", + "minSize" -> "0", + "maxSize" -> "1024", + "Content-Type" -> "application/xml", + "x-amz-meta-upscan-initiate-received" -> receivedAt.toString, + "success_redirect_url" -> "https://new.service/page1" + ) + + And("uploadInitiated counter has been incremented") + metrics.defaultRegistry.counter("uploadInitiated").getCount shouldBe 1 + + } + } } From acd000de021f79ad6ee667c52b041cae1400cffa Mon Sep 17 00:00:00 2001 From: Christopher Turner <23338096+christopherjturner@users.noreply.github.com> Date: Wed, 24 Apr 2019 14:23:12 +0100 Subject: [PATCH 15/96] BDOG-189 refactored building of map --- app/connectors/s3/S3UploadFormGenerator.scala | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/app/connectors/s3/S3UploadFormGenerator.scala b/app/connectors/s3/S3UploadFormGenerator.scala index 3260f5a..ea44e32 100644 --- a/app/connectors/s3/S3UploadFormGenerator.scala +++ b/app/connectors/s3/S3UploadFormGenerator.scala @@ -72,18 +72,16 @@ class S3UploadFormGenerator( "x-amz-meta-upscan-initiate-response" -> currentTime().toString ) - val sessionCredentials = securityToken.map(t => Map("x-amz-security-token" -> t)).getOrElse(Map.empty) - val metadataFields = uploadParameters.additionalMetadata.map { case (metadataKey, value) => s"x-amz-meta-$metadataKey" -> value } - val contentTypeField = uploadParameters.expectedContentType.map(contentType => "Content-Type" -> contentType) - - val successRedirect = uploadParameters.successRedirect.map(url => Map("success_action_redirect" -> url)).getOrElse(Map.empty) + val sessionCredentials = securityToken.map(v => Map("x-amz-security-token" -> v)).getOrElse(Map.empty) + val contentTypeField = uploadParameters.expectedContentType.map(v => Map("Content-Type" -> v)).getOrElse(Map.empty) + val successRedirect = uploadParameters.successRedirect.map(v => Map("success_action_redirect" -> v)).getOrElse(Map.empty) - fields ++ sessionCredentials ++ metadataFields ++ contentTypeField ++ successRedirect + fields ++ metadataFields ++ sessionCredentials ++ contentTypeField ++ successRedirect } private def buildPolicy( From 96a539cca3550fe54218323c2de4976f30374ddd Mon Sep 17 00:00:00 2001 From: Christopher Turner <23338096+christopherjturner@users.noreply.github.com> Date: Tue, 7 May 2019 15:21:44 +0100 Subject: [PATCH 16/96] BDOG-189 updated policy generation to include success_redirects --- app/connectors/s3/S3UploadFormGenerator.scala | 3 +++ .../s3/S3UploadFormGeneratorSpec.scala | 19 ++++++++++--------- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/app/connectors/s3/S3UploadFormGenerator.scala b/app/connectors/s3/S3UploadFormGenerator.scala index ea44e32..dbd2891 100644 --- a/app/connectors/s3/S3UploadFormGenerator.scala +++ b/app/connectors/s3/S3UploadFormGenerator.scala @@ -101,6 +101,8 @@ class S3UploadFormGenerator( val contentTypeConstraintJson = uploadParameters.expectedContentType.map(contentType => Json.obj("Content-Type" -> contentType)) + val successRedirectConstraint = uploadParameters.successRedirect.map(redirect => Json.obj("success_action_redirect" -> redirect)) + val policyDocument = Json.obj( "expiration" -> ISO_INSTANT.format(uploadParameters.expirationDateTime), "conditions" -> JsArray( @@ -118,6 +120,7 @@ class S3UploadFormGenerator( ) ++ securityTokenJson ++ metadataJson ++ contentTypeConstraintJson + ++ successRedirectConstraint ) ) diff --git a/test/connectors/s3/S3UploadFormGeneratorSpec.scala b/test/connectors/s3/S3UploadFormGeneratorSpec.scala index b049872..80af554 100644 --- a/test/connectors/s3/S3UploadFormGeneratorSpec.scala +++ b/test/connectors/s3/S3UploadFormGeneratorSpec.scala @@ -39,7 +39,7 @@ class S3UploadFormGeneratorSpec extends WordSpec with GivenWhenThen with Matcher additionalMetadata = Map("key1" -> "value1"), contentLengthRange = ContentLengthRange(0, 1024), expectedContentType = Some("application/xml"), - None + successRedirect = Some("http://test.com/abc") ) When("form fields are generated") @@ -52,14 +52,15 @@ class S3UploadFormGeneratorSpec extends WordSpec with GivenWhenThen with Matcher And("policy contains proper conditions") - ((policy \ "conditions").get \\ "acl").head.as[String] shouldBe "private" - ((policy \ "conditions").get \\ "bucket").head.as[String] shouldBe "test-bucket" - ((policy \ "conditions").get \\ "key").head.as[String] shouldBe "test-key" - ((policy \ "conditions").get \\ "x-amz-algorithm").head.as[String] shouldBe "AWS4-HMAC-SHA256" - ((policy \ "conditions").get \\ "x-amz-date").head.as[String] shouldBe "19970716T192030Z" - ((policy \ "conditions").get \\ "x-amz-security-token").head.as[String] shouldBe "session-token" - ((policy \ "conditions").get \\ "x-amz-meta-key1").head.as[String] shouldBe "value1" - ((policy \ "conditions").get \\ "Content-Type").head.as[String] shouldBe "application/xml" + ((policy \ "conditions").get \\ "acl").head.as[String] shouldBe "private" + ((policy \ "conditions").get \\ "bucket").head.as[String] shouldBe "test-bucket" + ((policy \ "conditions").get \\ "key").head.as[String] shouldBe "test-key" + ((policy \ "conditions").get \\ "x-amz-algorithm").head.as[String] shouldBe "AWS4-HMAC-SHA256" + ((policy \ "conditions").get \\ "x-amz-date").head.as[String] shouldBe "19970716T192030Z" + ((policy \ "conditions").get \\ "x-amz-security-token").head.as[String] shouldBe "session-token" + ((policy \ "conditions").get \\ "x-amz-meta-key1").head.as[String] shouldBe "value1" + ((policy \ "conditions").get \\ "Content-Type").head.as[String] shouldBe "application/xml" + ((policy \ "conditions").get \\ "success_action_redirect").head.as[String] shouldBe "http://test.com/abc" val conditions = (policy \ "conditions").as[JsArray].value val arrayConditions: Seq[Seq[JsValue]] = conditions.flatMap(_.asOpt[JsArray].map(_.value)) From 71d0549e900c617b6be282e65e626adbd6655214 Mon Sep 17 00:00:00 2001 From: chotai Date: Wed, 29 May 2019 23:12:03 +0100 Subject: [PATCH 17/96] BDOG-188 Refactor classes to model & services packages --- app/connectors/model/ContentLengthRange.scala | 3 + .../model/UploadFormGenerator.scala | 7 + .../model/UploadParameters.scala} | 11 +- app/connectors/s3/S3Module.scala | 2 +- app/connectors/s3/S3UploadFormGenerator.scala | 12 +- .../s3/S3UploadFormGeneratorProvider.scala | 2 +- app/controllers/PrepareUploadController.scala | 40 +++--- .../model/PrepareUploadRequestV1.scala | 8 ++ .../model/PreparedUploadResponse.scala | 3 + app/controllers/model/Reference.scala | 3 + .../model/UploadFormTemplate.scala | 3 + app/domain/model.scala | 14 -- .../PrepareUploadService.scala | 28 ++-- .../s3/S3UploadFormGeneratorSpec.scala | 2 +- .../PrepareUploadControllerSpec.scala | 125 ++++++++++-------- .../PrepareUploadServiceSpec.scala | 66 ++++----- 16 files changed, 184 insertions(+), 145 deletions(-) create mode 100644 app/connectors/model/ContentLengthRange.scala create mode 100644 app/connectors/model/UploadFormGenerator.scala rename app/{domain/UploadFormGenerator.scala => connectors/model/UploadParameters.scala} (57%) create mode 100644 app/controllers/model/PrepareUploadRequestV1.scala create mode 100644 app/controllers/model/PreparedUploadResponse.scala create mode 100644 app/controllers/model/Reference.scala create mode 100644 app/controllers/model/UploadFormTemplate.scala delete mode 100644 app/domain/model.scala rename app/{domain => services}/PrepareUploadService.scala (72%) rename test/{domain => model}/PrepareUploadServiceSpec.scala (76%) diff --git a/app/connectors/model/ContentLengthRange.scala b/app/connectors/model/ContentLengthRange.scala new file mode 100644 index 0000000..81a5523 --- /dev/null +++ b/app/connectors/model/ContentLengthRange.scala @@ -0,0 +1,3 @@ +package connectors.model + +case class ContentLengthRange(min: Int, max: Int) diff --git a/app/connectors/model/UploadFormGenerator.scala b/app/connectors/model/UploadFormGenerator.scala new file mode 100644 index 0000000..f4e2b34 --- /dev/null +++ b/app/connectors/model/UploadFormGenerator.scala @@ -0,0 +1,7 @@ +package connectors.model + +trait UploadFormGenerator { + def buildEndpoint(bucketName: String): String + + def generateFormFields(uploadParameters: UploadParameters): Map[String, String] +} diff --git a/app/domain/UploadFormGenerator.scala b/app/connectors/model/UploadParameters.scala similarity index 57% rename from app/domain/UploadFormGenerator.scala rename to app/connectors/model/UploadParameters.scala index 4dfa7b2..46484f4 100644 --- a/app/domain/UploadFormGenerator.scala +++ b/app/connectors/model/UploadParameters.scala @@ -1,15 +1,6 @@ -package domain - +package connectors.model import java.time.Instant -trait UploadFormGenerator { - def buildEndpoint(bucketName: String): String - - def generateFormFields(uploadParameters: UploadParameters): Map[String, String] -} - -case class ContentLengthRange(min: Int, max: Int) - case class UploadParameters( expirationDateTime: Instant, bucketName: String, diff --git a/app/connectors/s3/S3Module.scala b/app/connectors/s3/S3Module.scala index 201a16d..6edf981 100644 --- a/app/connectors/s3/S3Module.scala +++ b/app/connectors/s3/S3Module.scala @@ -1,6 +1,6 @@ package connectors.s3 -import domain.UploadFormGenerator +import connectors.model.UploadFormGenerator import play.api.inject.{Binding, Module} import play.api.{Configuration, Environment} diff --git a/app/connectors/s3/S3UploadFormGenerator.scala b/app/connectors/s3/S3UploadFormGenerator.scala index dbd2891..3c3630e 100644 --- a/app/connectors/s3/S3UploadFormGenerator.scala +++ b/app/connectors/s3/S3UploadFormGenerator.scala @@ -4,7 +4,7 @@ import java.time.format.DateTimeFormatter import java.time.format.DateTimeFormatter.ISO_INSTANT import java.time.{Instant, ZoneOffset} -import domain.{UploadFormGenerator, UploadParameters} +import connectors.model.{UploadFormGenerator, UploadParameters} import play.api.libs.json.{JsArray, JsValue, Json} final case class AwsCredentials( @@ -77,11 +77,12 @@ class S3UploadFormGenerator( case (metadataKey, value) => s"x-amz-meta-$metadataKey" -> value } - val sessionCredentials = securityToken.map(v => Map("x-amz-security-token" -> v)).getOrElse(Map.empty) + val sessionCredentials = securityToken.map(v => Map("x-amz-security-token" -> v)).getOrElse(Map.empty) val contentTypeField = uploadParameters.expectedContentType.map(v => Map("Content-Type" -> v)).getOrElse(Map.empty) - val successRedirect = uploadParameters.successRedirect.map(v => Map("success_action_redirect" -> v)).getOrElse(Map.empty) + val successRedirect = + uploadParameters.successRedirect.map(v => Map("success_action_redirect" -> v)).getOrElse(Map.empty) - fields ++ metadataFields ++ sessionCredentials ++ contentTypeField ++ successRedirect + fields ++ metadataFields ++ sessionCredentials ++ contentTypeField ++ successRedirect } private def buildPolicy( @@ -101,7 +102,8 @@ class S3UploadFormGenerator( val contentTypeConstraintJson = uploadParameters.expectedContentType.map(contentType => Json.obj("Content-Type" -> contentType)) - val successRedirectConstraint = uploadParameters.successRedirect.map(redirect => Json.obj("success_action_redirect" -> redirect)) + val successRedirectConstraint = + uploadParameters.successRedirect.map(redirect => Json.obj("success_action_redirect" -> redirect)) val policyDocument = Json.obj( "expiration" -> ISO_INSTANT.format(uploadParameters.expirationDateTime), diff --git a/app/connectors/s3/S3UploadFormGeneratorProvider.scala b/app/connectors/s3/S3UploadFormGeneratorProvider.scala index 963e453..5c6087f 100644 --- a/app/connectors/s3/S3UploadFormGeneratorProvider.scala +++ b/app/connectors/s3/S3UploadFormGeneratorProvider.scala @@ -3,7 +3,7 @@ package connectors.s3 import config.ServiceConfiguration import java.time.{Clock, Instant} -import domain.UploadFormGenerator +import connectors.model.UploadFormGenerator import javax.inject.{Inject, Provider, Singleton} @Singleton diff --git a/app/controllers/PrepareUploadController.scala b/app/controllers/PrepareUploadController.scala index 5c405e0..eb4a49a 100644 --- a/app/controllers/PrepareUploadController.scala +++ b/app/controllers/PrepareUploadController.scala @@ -1,22 +1,23 @@ package controllers -import config.ServiceConfiguration -import domain._ -import javax.inject.{Inject, Singleton} import java.net.URL import java.time.{Clock, Instant} +import config.ServiceConfiguration +import controllers.model.{PrepareUploadRequestV1, PreparedUploadResponse, UploadFormTemplate} +import javax.inject.{Inject, Singleton} import play.api.Logger import play.api.data.validation.ValidationError import play.api.libs.functional.syntax._ import play.api.libs.json.Reads._ import play.api.libs.json.{JsPath, Json, Writes, _} import play.api.mvc.{Action, Result} +import services.PrepareUploadService +import uk.gov.hmrc.play.bootstrap.controller.BaseController +import utils.UserAgentFilter import scala.concurrent.Future import scala.util.{Failure, Success, Try} -import uk.gov.hmrc.play.bootstrap.controller.BaseController -import utils.UserAgentFilter @Singleton class PrepareUploadController @Inject()( @@ -26,23 +27,21 @@ class PrepareUploadController @Inject()( extends BaseController with UserAgentFilter { - implicit val uploadSettingsReads: Reads[UploadSettings] = ( + implicit val uploadSettingsReads: Reads[PrepareUploadRequestV1] = ( (JsPath \ "callbackUrl").read[String] and (JsPath \ "minimumFileSize").readNullable[Int](min(0)) and (JsPath \ "maximumFileSize").readNullable[Int](min(0) keepAnd max(prepareUploadService.globalFileSizeLimit + 1)) and (JsPath \ "expectedContentType").readNullable[String] and (JsPath \ "successRedirect").readNullable[String] - - )(UploadSettings.apply _) - .filter(ValidationError("Maximum file size must be equal or greater than minimum file size"))( - settings => - settings.minimumFileSize.getOrElse(0) <= settings.maximumFileSize.getOrElse( - prepareUploadService.globalFileSizeLimit)) + )(PrepareUploadRequestV1.apply _) + .filter(ValidationError("Maximum file size must be equal or greater than minimum file size"))(settings => + settings.minimumFileSize.getOrElse(0) <= settings.maximumFileSize.getOrElse( + prepareUploadService.globalFileSizeLimit)) implicit val uploadFormTemplateWrites: Writes[UploadFormTemplate] = Json.writes[UploadFormTemplate] - implicit val preparedUploadWrites: Writes[PreparedUpload] = new Writes[PreparedUpload] { - def writes(preparedUpload: PreparedUpload) = Json.obj( + implicit val preparedUploadWrites: Writes[PreparedUploadResponse] = new Writes[PreparedUploadResponse] { + def writes(preparedUpload: PreparedUploadResponse): JsValue = Json.obj( "reference" -> preparedUpload.reference.value, "uploadRequest" -> preparedUpload.uploadRequest ) @@ -53,15 +52,13 @@ class PrepareUploadController @Inject()( val receivedAt = Instant.now(clock) onlyAllowedServices[JsValue] { (_, consumingService) => - - withJsonBody[UploadSettings] { (uploadSettings: UploadSettings) => - + withJsonBody[PrepareUploadRequestV1] { uploadSettings: PrepareUploadRequestV1 => withAllowedCallbackProtocol(uploadSettings.callbackUrl) { Logger.debug(s"Processing request: [$uploadSettings].") val sessionId = hc(request).sessionId.map(_.value).getOrElse("n/a") val requestId = hc(request).requestId.map(_.value).getOrElse("n/a") - val result: PreparedUpload = + val result: PreparedUploadResponse = prepareUploadService.prepareUpload(uploadSettings, consumingService, requestId, sessionId, receivedAt) Future.successful(Ok(Json.toJson(result))) @@ -70,8 +67,8 @@ class PrepareUploadController @Inject()( } } - private[controllers] def withAllowedCallbackProtocol[A](callbackUrl: String) - (block: => Future[Result]): Future[Result]= { + private[controllers] def withAllowedCallbackProtocol[A](callbackUrl: String)( + block: => Future[Result]): Future[Result] = { val allowedCallbackProtocols: Seq[String] = configuration.allowedCallbackProtocols @@ -84,7 +81,8 @@ class PrepareUploadController @Inject()( case Success(false) => { Logger.warn(s"Invalid callback url protocol: [$callbackUrl].") - Future.successful(BadRequest(s"Invalid callback url protocol: [$callbackUrl]. Protocol must be in: [${allowedCallbackProtocols.mkString(",")}].")) + Future.successful(BadRequest( + s"Invalid callback url protocol: [$callbackUrl]. Protocol must be in: [${allowedCallbackProtocols.mkString(",")}].")) } case Failure(e) => { Logger.warn(s"Invalid callback url format: [$callbackUrl].") diff --git a/app/controllers/model/PrepareUploadRequestV1.scala b/app/controllers/model/PrepareUploadRequestV1.scala new file mode 100644 index 0000000..7fbb225 --- /dev/null +++ b/app/controllers/model/PrepareUploadRequestV1.scala @@ -0,0 +1,8 @@ +package controllers.model + +case class PrepareUploadRequestV1( + callbackUrl: String, + minimumFileSize: Option[Int], + maximumFileSize: Option[Int], + expectedContentType: Option[String], + successRedirect: Option[String] = None) diff --git a/app/controllers/model/PreparedUploadResponse.scala b/app/controllers/model/PreparedUploadResponse.scala new file mode 100644 index 0000000..f6c17e8 --- /dev/null +++ b/app/controllers/model/PreparedUploadResponse.scala @@ -0,0 +1,3 @@ +package controllers.model + +case class PreparedUploadResponse(reference: Reference, uploadRequest: UploadFormTemplate) diff --git a/app/controllers/model/Reference.scala b/app/controllers/model/Reference.scala new file mode 100644 index 0000000..b0bbb2e --- /dev/null +++ b/app/controllers/model/Reference.scala @@ -0,0 +1,3 @@ +package controllers.model + +case class Reference(value: String) diff --git a/app/controllers/model/UploadFormTemplate.scala b/app/controllers/model/UploadFormTemplate.scala new file mode 100644 index 0000000..3eb3d44 --- /dev/null +++ b/app/controllers/model/UploadFormTemplate.scala @@ -0,0 +1,3 @@ +package controllers.model + +case class UploadFormTemplate(href: String, fields: Map[String, String]) diff --git a/app/domain/model.scala b/app/domain/model.scala deleted file mode 100644 index dac749c..0000000 --- a/app/domain/model.scala +++ /dev/null @@ -1,14 +0,0 @@ -package domain - -case class UploadSettings( - callbackUrl: String, - minimumFileSize: Option[Int], - maximumFileSize: Option[Int], - expectedContentType: Option[String], - successRedirect: Option[String] = None) - -case class UploadFormTemplate(href: String, fields: Map[String, String]) - -case class Reference(value: String) - -case class PreparedUpload(reference: Reference, uploadRequest: UploadFormTemplate) diff --git a/app/domain/PrepareUploadService.scala b/app/services/PrepareUploadService.scala similarity index 72% rename from app/domain/PrepareUploadService.scala rename to app/services/PrepareUploadService.scala index 45e98ef..03708cc 100644 --- a/app/domain/PrepareUploadService.scala +++ b/app/services/PrepareUploadService.scala @@ -1,10 +1,12 @@ -package domain +package services import java.time.Instant import java.util.UUID import com.kenshoo.play.metrics.Metrics import config.ServiceConfiguration +import connectors.model.{ContentLengthRange, UploadFormGenerator, UploadParameters} +import controllers.model.{PrepareUploadRequestV1, PreparedUploadResponse, Reference, UploadFormTemplate} import javax.inject.{Inject, Singleton} import org.slf4j.MDC import play.api.Logger @@ -15,14 +17,20 @@ class PrepareUploadService @Inject()( configuration: ServiceConfiguration, metrics: Metrics) { - def prepareUpload(settings: UploadSettings, consumingService: String, requestId : String, sessionId: String, receivedAt: Instant): PreparedUpload = { + def prepareUpload( + settings: PrepareUploadRequestV1, + consumingService: String, + requestId: String, + sessionId: String, + receivedAt: Instant): PreparedUploadResponse = { val reference = generateReference() val expiration = receivedAt.plus(configuration.fileExpirationPeriod) val result = - PreparedUpload( - reference = reference, - uploadRequest = generatePost(reference.value, expiration, settings, consumingService, requestId, sessionId, receivedAt)) + controllers.model.PreparedUploadResponse( + reference = reference, + uploadRequest = + generatePost(reference.value, expiration, settings, consumingService, requestId, sessionId, receivedAt)) try { MDC.put("file-reference", reference.value) @@ -42,7 +50,7 @@ class PrepareUploadService @Inject()( private def generatePost( key: String, expiration: Instant, - settings: UploadSettings, + settings: PrepareUploadRequestV1, consumingService: String, requestId: String, sessionId: String, @@ -61,10 +69,10 @@ class PrepareUploadService @Inject()( objectKey = key, acl = "private", additionalMetadata = Map( - "callback-url" -> settings.callbackUrl, - "consuming-service" -> consumingService, - "session-id" -> sessionId, - "request-id" -> requestId, + "callback-url" -> settings.callbackUrl, + "consuming-service" -> consumingService, + "session-id" -> sessionId, + "request-id" -> requestId, "upscan-initiate-received" -> receivedAt.toString ), contentLengthRange = ContentLengthRange(minFileSize, maxFileSize), diff --git a/test/connectors/s3/S3UploadFormGeneratorSpec.scala b/test/connectors/s3/S3UploadFormGeneratorSpec.scala index 80af554..0e127ad 100644 --- a/test/connectors/s3/S3UploadFormGeneratorSpec.scala +++ b/test/connectors/s3/S3UploadFormGeneratorSpec.scala @@ -3,7 +3,7 @@ package connectors.s3 import java.time.Instant import java.util.Base64 -import domain.{ContentLengthRange, UploadParameters} +import connectors.model.{ContentLengthRange, UploadParameters} import org.mockito.ArgumentMatchers.any import org.mockito.Mockito.{verify, when} import org.scalatest.mockito.MockitoSugar diff --git a/test/controllers/PrepareUploadControllerSpec.scala b/test/controllers/PrepareUploadControllerSpec.scala index 4ffbc7e..6437a5c 100644 --- a/test/controllers/PrepareUploadControllerSpec.scala +++ b/test/controllers/PrepareUploadControllerSpec.scala @@ -5,7 +5,7 @@ import java.time.Clock import akka.actor.ActorSystem import akka.stream.ActorMaterializer import config.ServiceConfiguration -import domain._ +import controllers.model.{PrepareUploadRequestV1, PreparedUploadResponse, Reference, UploadFormTemplate} import org.mockito.ArgumentMatchers.any import org.mockito.Mockito import org.mockito.invocation.InvocationOnMock @@ -13,19 +13,21 @@ import org.mockito.stubbing.Answer import org.scalatest.mockito.MockitoSugar import org.scalatest.{GivenWhenThen, Matchers} import play.api.libs.json.{JsValue, Json} -import play.api.test.{FakeRequest, Helpers} +import play.api.mvc.Results.Ok import play.api.test.Helpers.contentAsString +import play.api.test.{FakeRequest, Helpers} +import services.PrepareUploadService import uk.gov.hmrc.play.test.UnitSpec -import play.api.mvc.Results.Ok import scala.concurrent.Future import scala.concurrent.duration._ +import scala.language.postfixOps class PrepareUploadControllerSpec extends UnitSpec with Matchers with GivenWhenThen with MockitoSugar { - implicit val actorSystem = ActorSystem() + implicit val actorSystem: ActorSystem = ActorSystem() - implicit val materializer = ActorMaterializer() + implicit val materializer: ActorMaterializer = ActorMaterializer() implicit val timeout: akka.util.Timeout = 10 seconds @@ -42,7 +44,10 @@ class PrepareUploadControllerSpec extends UnitSpec with Matchers with GivenWhenT Given("there is a valid upload request with all data") val request: FakeRequest[JsValue] = FakeRequest() - .withHeaders(("User-Agent", "VALID-AGENT"), ("x-session-id", "some-session-id"), ("x-request-id", "some-request-id")) + .withHeaders( + ("User-Agent", "VALID-AGENT"), + ("x-session-id", "some-session-id"), + ("x-request-id", "some-request-id")) .withBody( Json.obj( "id" -> "1", @@ -65,10 +70,11 @@ class PrepareUploadControllerSpec extends UnitSpec with Matchers with GivenWhenT "fields" -> Json.obj( "minFileSize" -> "0", "maxFileSize" -> "1024", - "sessionId" -> "some-session-id", - "requestId" -> "some-request-id" + "sessionId" -> "some-session-id", + "requestId" -> "some-request-id" ) - )) + ) + ) } "build and return upload URL if valid request with redirect on success url" in { @@ -77,7 +83,10 @@ class PrepareUploadControllerSpec extends UnitSpec with Matchers with GivenWhenT Given("there is a valid upload request with all data") val request: FakeRequest[JsValue] = FakeRequest() - .withHeaders(("User-Agent", "VALID-AGENT"), ("x-session-id", "some-session-id"), ("x-request-id", "some-request-id")) + .withHeaders( + ("User-Agent", "VALID-AGENT"), + ("x-session-id", "some-session-id"), + ("x-request-id", "some-request-id")) .withBody( Json.obj( "id" -> "1", @@ -99,13 +108,14 @@ class PrepareUploadControllerSpec extends UnitSpec with Matchers with GivenWhenT "uploadRequest" -> Json.obj( "href" -> "https://www.example.com", "fields" -> Json.obj( - "minFileSize" -> "0", - "maxFileSize" -> "1024", - "sessionId" -> "some-session-id", - "requestId" -> "some-request-id", + "minFileSize" -> "0", + "maxFileSize" -> "1024", + "sessionId" -> "some-session-id", + "requestId" -> "some-request-id", "success_action_redirect" -> "https://www.example.com/nextpage" ) - )) + ) + ) } "build and return upload URL if valid request with minimal data including session id and request id" in { @@ -114,7 +124,10 @@ class PrepareUploadControllerSpec extends UnitSpec with Matchers with GivenWhenT Given("there is a valid upload request with minimal data") val request: FakeRequest[JsValue] = FakeRequest() - .withHeaders(("User-Agent", "VALID-AGENT"), ("x-session-id", "some-session-id"), ("x-request-id", "some-request-id")) + .withHeaders( + ("User-Agent", "VALID-AGENT"), + ("x-session-id", "some-session-id"), + ("x-request-id", "some-request-id")) .withBody(Json.obj("callbackUrl" -> "https://www.example.com")) When("upload initiation has been requested") @@ -128,12 +141,13 @@ class PrepareUploadControllerSpec extends UnitSpec with Matchers with GivenWhenT json shouldBe Json.obj( "reference" -> "TEST", "uploadRequest" -> Json.obj( - "href" -> "https://www.example.com", + "href" -> "https://www.example.com", "fields" -> Json.obj( "sessionId" -> "some-session-id", "requestId" -> "some-request-id" ) - )) + ) + ) } "build and return upload URL if valid request with minimal data excluding session id and request id" in { @@ -156,7 +170,7 @@ class PrepareUploadControllerSpec extends UnitSpec with Matchers with GivenWhenT json shouldBe Json.obj( "reference" -> "TEST", "uploadRequest" -> Json.obj( - "href" -> "https://www.example.com", + "href" -> "https://www.example.com", "fields" -> Json.obj( "sessionId" -> "n/a", "requestId" -> "n/a" @@ -164,14 +178,16 @@ class PrepareUploadControllerSpec extends UnitSpec with Matchers with GivenWhenT )) } - "return a bad request error if invalid request - wrong structure" in { val controller = new PrepareUploadController(prepareUploadService, config, clock) Given("there is an invalid upload request") val request: FakeRequest[JsValue] = FakeRequest() - .withHeaders(("User-Agent", "VALID-AGENT"), ("x-session-id", "some-session-id"), ("x-request-id", "some-request-id")) + .withHeaders( + ("User-Agent", "VALID-AGENT"), + ("x-session-id", "some-session-id"), + ("x-request-id", "some-request-id")) .withBody(Json.obj("invalid" -> "body")) When("upload initiation has been requested") @@ -209,15 +225,18 @@ class PrepareUploadControllerSpec extends UnitSpec with Matchers with GivenWhenT Given("there is a valid upload request from a whitelisted service") val request: FakeRequest[JsValue] = FakeRequest() - .withHeaders(("User-Agent", "VALID-AGENT"), ("x-session-id", "some-session-id"), ("x-request-id", "some-request-id")) - .withBody( - Json.obj( - "id" -> "1", - "callbackUrl" -> "https://www.example.com", - "minimumFileSize" -> 0, - "maximumFileSize" -> 1024, - "session-id" -> "some-session-id", - "request-id" -> "some-request-id")) + .withHeaders( + ("User-Agent", "VALID-AGENT"), + ("x-session-id", "some-session-id"), + ("x-request-id", "some-request-id")) + .withBody(Json.obj( + "id" -> "1", + "callbackUrl" -> "https://www.example.com", + "minimumFileSize" -> 0, + "maximumFileSize" -> 1024, + "session-id" -> "some-session-id", + "request-id" -> "some-request-id" + )) When("upload initiation has been requested") @@ -234,10 +253,11 @@ class PrepareUploadControllerSpec extends UnitSpec with Matchers with GivenWhenT "fields" -> Json.obj( "minFileSize" -> "0", "maxFileSize" -> "1024", - "sessionId" -> "some-session-id", - "requestId" -> "some-request-id" + "sessionId" -> "some-session-id", + "requestId" -> "some-request-id" ) - )) + ) + ) } "return a forbidden error if service is not whitelisted " in { @@ -246,16 +266,18 @@ class PrepareUploadControllerSpec extends UnitSpec with Matchers with GivenWhenT Given("there is a valid upload request from a non-whitelisted service") val request: FakeRequest[JsValue] = FakeRequest() - .withHeaders(("User-Agent", "INVALID-AGENT"), ("x-session-id", "some-session-id"), ("x-request-id", "some-request-id")) - .withBody( - Json.obj( - "id" -> "1", - "callbackUrl" -> "http://www.example.com", - "minimumFileSize" -> 0, - "maximumFileSize" -> 1024, - "session-id" -> "some-session-id", - "request-id" -> "some-request-id" - )) + .withHeaders( + ("User-Agent", "INVALID-AGENT"), + ("x-session-id", "some-session-id"), + ("x-request-id", "some-request-id")) + .withBody(Json.obj( + "id" -> "1", + "callbackUrl" -> "http://www.example.com", + "minimumFileSize" -> 0, + "maximumFileSize" -> 1024, + "session-id" -> "some-session-id", + "request-id" -> "some-request-id" + )) When("upload initiation has been requested") @@ -285,7 +307,7 @@ class PrepareUploadControllerSpec extends UnitSpec with Matchers with GivenWhenT Future.failed(new RuntimeException("This block should not have been invoked.")) } - status(result) shouldBe 400 + status(result) shouldBe 400 contentAsString(result) should include("Invalid callback url protocol") } @@ -295,22 +317,22 @@ class PrepareUploadControllerSpec extends UnitSpec with Matchers with GivenWhenT val result = controller.withAllowedCallbackProtocol("123") { Future.failed(new RuntimeException("This block should not have been invoked.")) } - status(result) shouldBe 400 + status(result) shouldBe 400 contentAsString(result) should include("Invalid callback url format") } } - def prepareUploadService = { + def prepareUploadService: PrepareUploadService = { val service = mock[PrepareUploadService] Mockito.when(service.globalFileSizeLimit).thenReturn(1024) Mockito .when(service.prepareUpload(any(), any(), any(), any(), any())) - .thenAnswer(new Answer[PreparedUpload]() { - override def answer(invocationOnMock: InvocationOnMock): PreparedUpload = { - val settings = invocationOnMock.getArgument[UploadSettings](0) + .thenAnswer(new Answer[PreparedUploadResponse]() { + override def answer(invocationOnMock: InvocationOnMock): PreparedUploadResponse = { + val settings = invocationOnMock.getArgument[PrepareUploadRequestV1](0) val requestId = invocationOnMock.getArgument[String](2) val sessionId = invocationOnMock.getArgument[String](3) - PreparedUpload( + PreparedUploadResponse( Reference("TEST"), UploadFormTemplate( settings.callbackUrl, @@ -319,8 +341,7 @@ class PrepareUploadControllerSpec extends UnitSpec with Matchers with GivenWhenT settings.maximumFileSize.map(s => Map("maxFileSize" -> s.toString).head) ++ Map("sessionId" -> sessionId) ++ Map("requestId" -> requestId) ++ - settings.successRedirect.map(url => Map("success_action_redirect" -> url)).getOrElse(Map.empty) - + settings.successRedirect.map(url => Map("success_action_redirect" -> url)).getOrElse(Map.empty) ) ) } diff --git a/test/domain/PrepareUploadServiceSpec.scala b/test/model/PrepareUploadServiceSpec.scala similarity index 76% rename from test/domain/PrepareUploadServiceSpec.scala rename to test/model/PrepareUploadServiceSpec.scala index 621fe02..4f41d4b 100644 --- a/test/domain/PrepareUploadServiceSpec.scala +++ b/test/model/PrepareUploadServiceSpec.scala @@ -1,4 +1,4 @@ -package domain +package model import java.time import java.time.Instant @@ -6,7 +6,10 @@ import java.time.Instant import com.codahale.metrics.MetricRegistry import com.kenshoo.play.metrics.Metrics import config.ServiceConfiguration +import connectors.model.{UploadFormGenerator, UploadParameters} +import controllers.model.PrepareUploadRequestV1 import org.scalatest.{GivenWhenThen, Matchers} +import services.PrepareUploadService import uk.gov.hmrc.play.test.UnitSpec class PrepareUploadServiceSpec extends UnitSpec with Matchers with GivenWhenThen { @@ -51,7 +54,7 @@ class PrepareUploadServiceSpec extends UnitSpec with Matchers with GivenWhenThen uploadParameters.expectedContentType.map { contentType => "Content-Type" -> contentType } ++ - uploadParameters.successRedirect.map { "success_redirect_url" -> _} + uploadParameters.successRedirect.map { "success_redirect_url" -> _ } override def buildEndpoint(bucketName: String): String = s"$bucketName.s3" } @@ -70,7 +73,7 @@ class PrepareUploadServiceSpec extends UnitSpec with Matchers with GivenWhenThen val callbackUrl = "http://www.callback.com" - val uploadSettings = UploadSettings( + val uploadSettings = PrepareUploadRequestV1( callbackUrl = callbackUrl, minimumFileSize = None, maximumFileSize = None, @@ -78,21 +81,22 @@ class PrepareUploadServiceSpec extends UnitSpec with Matchers with GivenWhenThen When("we setup the upload") - val result = service(metrics).prepareUpload(uploadSettings, "PrepareUploadServiceSpec", "some-request-id", "some-session-id", receivedAt) + val result = service(metrics) + .prepareUpload(uploadSettings, "PrepareUploadServiceSpec", "some-request-id", "some-session-id", receivedAt) Then("proper upload request form definition should be returned") result.uploadRequest.href shouldBe s"${serviceConfiguration.inboundBucketName}.s3" result.uploadRequest.fields shouldBe Map( - "bucket" -> serviceConfiguration.inboundBucketName, - "key" -> result.reference.value, - "x-amz-meta-callback-url" -> callbackUrl, - "x-amz-meta-consuming-service" -> "PrepareUploadServiceSpec", - "x-amz-meta-session-id" -> "some-session-id", - "x-amz-meta-request-id" -> "some-request-id", - "minSize" -> "0", - "maxSize" -> "1024", - "Content-Type" -> "application/xml", + "bucket" -> serviceConfiguration.inboundBucketName, + "key" -> result.reference.value, + "x-amz-meta-callback-url" -> callbackUrl, + "x-amz-meta-consuming-service" -> "PrepareUploadServiceSpec", + "x-amz-meta-session-id" -> "some-session-id", + "x-amz-meta-request-id" -> "some-request-id", + "minSize" -> "0", + "maxSize" -> "1024", + "Content-Type" -> "application/xml", "x-amz-meta-upscan-initiate-received" -> receivedAt.toString ) @@ -110,7 +114,7 @@ class PrepareUploadServiceSpec extends UnitSpec with Matchers with GivenWhenThen val callbackUrl = "http://www.callback.com" val uploadSettings = - UploadSettings( + PrepareUploadRequestV1( callbackUrl = callbackUrl, minimumFileSize = Some(100), maximumFileSize = Some(200), @@ -118,7 +122,8 @@ class PrepareUploadServiceSpec extends UnitSpec with Matchers with GivenWhenThen When("we setup the upload") - val result = service(metrics).prepareUpload(uploadSettings, "PrepareUploadServiceSpec", "some-request-id", "some-session-id", receivedAt) + val result = service(metrics) + .prepareUpload(uploadSettings, "PrepareUploadServiceSpec", "some-request-id", "some-session-id", receivedAt) Then("upload request should contain requested min/max size") @@ -135,7 +140,7 @@ class PrepareUploadServiceSpec extends UnitSpec with Matchers with GivenWhenThen val callbackUrl = "http://www.callback.com" val uploadSettings = - UploadSettings( + PrepareUploadRequestV1( callbackUrl = callbackUrl, minimumFileSize = Some(-1), maximumFileSize = Some(1024), @@ -161,7 +166,7 @@ class PrepareUploadServiceSpec extends UnitSpec with Matchers with GivenWhenThen val callbackUrl = "http://www.callback.com" val uploadSettings = - UploadSettings( + PrepareUploadRequestV1( callbackUrl = callbackUrl, minimumFileSize = Some(0), maximumFileSize = Some(1025), @@ -186,7 +191,7 @@ class PrepareUploadServiceSpec extends UnitSpec with Matchers with GivenWhenThen val callbackUrl = "http://www.callback.com" val uploadSettings = - UploadSettings( + PrepareUploadRequestV1( callbackUrl = callbackUrl, minimumFileSize = Some(1024), maximumFileSize = Some(0), @@ -210,7 +215,7 @@ class PrepareUploadServiceSpec extends UnitSpec with Matchers with GivenWhenThen val callbackUrl = "http://www.callback.com" - val uploadSettings = UploadSettings( + val uploadSettings = PrepareUploadRequestV1( callbackUrl = callbackUrl, minimumFileSize = None, maximumFileSize = None, @@ -220,23 +225,24 @@ class PrepareUploadServiceSpec extends UnitSpec with Matchers with GivenWhenThen When("we setup the upload") - val result = service(metrics).prepareUpload(uploadSettings, "PrepareUploadServiceSpec", "some-request-id", "some-session-id", receivedAt) + val result = service(metrics) + .prepareUpload(uploadSettings, "PrepareUploadServiceSpec", "some-request-id", "some-session-id", receivedAt) Then("proper upload request form definition should be returned") result.uploadRequest.href shouldBe s"${serviceConfiguration.inboundBucketName}.s3" result.uploadRequest.fields shouldBe Map( - "bucket" -> serviceConfiguration.inboundBucketName, - "key" -> result.reference.value, - "x-amz-meta-callback-url" -> callbackUrl, - "x-amz-meta-consuming-service" -> "PrepareUploadServiceSpec", - "x-amz-meta-session-id" -> "some-session-id", - "x-amz-meta-request-id" -> "some-request-id", - "minSize" -> "0", - "maxSize" -> "1024", - "Content-Type" -> "application/xml", + "bucket" -> serviceConfiguration.inboundBucketName, + "key" -> result.reference.value, + "x-amz-meta-callback-url" -> callbackUrl, + "x-amz-meta-consuming-service" -> "PrepareUploadServiceSpec", + "x-amz-meta-session-id" -> "some-session-id", + "x-amz-meta-request-id" -> "some-request-id", + "minSize" -> "0", + "maxSize" -> "1024", + "Content-Type" -> "application/xml", "x-amz-meta-upscan-initiate-received" -> receivedAt.toString, - "success_redirect_url" -> "https://new.service/page1" + "success_redirect_url" -> "https://new.service/page1" ) And("uploadInitiated counter has been incremented") From c6a1cecea4c0a915d5081d3953886d8ab12665ff Mon Sep 17 00:00:00 2001 From: chotai Date: Thu, 30 May 2019 00:59:11 +0100 Subject: [PATCH 18/96] BDOG-188 Rename request to PrepareUploadController from 'settings' --- app/controllers/PrepareUploadController.scala | 9 +++++---- project/MicroService.scala | 18 +++++++++--------- .../PrepareUploadControllerSpec.scala | 10 +++++----- test/model/PrepareUploadServiceSpec.scala | 12 ++++++------ 4 files changed, 25 insertions(+), 24 deletions(-) diff --git a/app/controllers/PrepareUploadController.scala b/app/controllers/PrepareUploadController.scala index eb4a49a..34a70a2 100644 --- a/app/controllers/PrepareUploadController.scala +++ b/app/controllers/PrepareUploadController.scala @@ -52,14 +52,15 @@ class PrepareUploadController @Inject()( val receivedAt = Instant.now(clock) onlyAllowedServices[JsValue] { (_, consumingService) => - withJsonBody[PrepareUploadRequestV1] { uploadSettings: PrepareUploadRequestV1 => - withAllowedCallbackProtocol(uploadSettings.callbackUrl) { - Logger.debug(s"Processing request: [$uploadSettings].") + withJsonBody[PrepareUploadRequestV1] { prepareUploadRequest: PrepareUploadRequestV1 => + withAllowedCallbackProtocol(prepareUploadRequest.callbackUrl) { + Logger.debug(s"Processing request: [$prepareUploadRequest].") val sessionId = hc(request).sessionId.map(_.value).getOrElse("n/a") val requestId = hc(request).requestId.map(_.value).getOrElse("n/a") val result: PreparedUploadResponse = - prepareUploadService.prepareUpload(uploadSettings, consumingService, requestId, sessionId, receivedAt) + prepareUploadService + .prepareUpload(prepareUploadRequest, consumingService, requestId, sessionId, receivedAt) Future.successful(Ok(Json.toJson(result))) } diff --git a/project/MicroService.scala b/project/MicroService.scala index 73e19e6..0a6238a 100644 --- a/project/MicroService.scala +++ b/project/MicroService.scala @@ -4,28 +4,28 @@ import play.sbt.routes.RoutesKeys.routesGenerator import sbt.Keys._ import sbt.Tests.{Group, SubProcess} import sbt._ -import uk.gov.hmrc.SbtArtifactory -import uk.gov.hmrc.sbtdistributables.SbtDistributablesPlugin -import uk.gov.hmrc.versioning.SbtGitVersioning import uk.gov.hmrc.sbtdistributables.SbtDistributablesPlugin import uk.gov.hmrc.versioning.SbtGitVersioning import uk.gov.hmrc.versioning.SbtGitVersioning.autoImport.majorVersion trait MicroService { - import uk.gov.hmrc._ - import uk.gov.hmrc.DefaultBuildSettings.{addTestReportOption, scalaSettings, defaultSettings, targetJvm} import TestPhases._ import scoverage.ScoverageKeys - import uk.gov.hmrc.sbtdistributables.SbtDistributablesPlugin._ + import uk.gov.hmrc.DefaultBuildSettings.{addTestReportOption, defaultSettings, scalaSettings, targetJvm} + import uk.gov.hmrc._ val appName: String lazy val appDependencies: Seq[ModuleID] = ??? lazy val plugins: Seq[Plugins] = Seq( - play.sbt.PlayScala, SbtAutoBuildPlugin, SbtGitVersioning, SbtDistributablesPlugin, SbtArtifactory + play.sbt.PlayScala, + SbtAutoBuildPlugin, + SbtGitVersioning, + SbtDistributablesPlugin, + SbtArtifactory ) - lazy val playSettings: Seq[Setting[_]] = Seq.empty + lazy val playSettings: Seq[Setting[_]] = Seq.empty routesGenerator := InjectedRoutesGenerator @@ -88,6 +88,6 @@ private object TestPhases { def oneForkedJvmPerTest(tests: Seq[TestDefinition]) = tests map { test => - new Group(test.name, Seq(test), SubProcess(ForkOptions(runJVMOptions = Seq("-Dtest.name=" + test.name)))) + Group(test.name, Seq(test), SubProcess(ForkOptions(runJVMOptions = Seq("-Dtest.name=" + test.name)))) } } diff --git a/test/controllers/PrepareUploadControllerSpec.scala b/test/controllers/PrepareUploadControllerSpec.scala index 6437a5c..baf44b5 100644 --- a/test/controllers/PrepareUploadControllerSpec.scala +++ b/test/controllers/PrepareUploadControllerSpec.scala @@ -329,19 +329,19 @@ class PrepareUploadControllerSpec extends UnitSpec with Matchers with GivenWhenT .when(service.prepareUpload(any(), any(), any(), any(), any())) .thenAnswer(new Answer[PreparedUploadResponse]() { override def answer(invocationOnMock: InvocationOnMock): PreparedUploadResponse = { - val settings = invocationOnMock.getArgument[PrepareUploadRequestV1](0) + val request = invocationOnMock.getArgument[PrepareUploadRequestV1](0) val requestId = invocationOnMock.getArgument[String](2) val sessionId = invocationOnMock.getArgument[String](3) PreparedUploadResponse( Reference("TEST"), UploadFormTemplate( - settings.callbackUrl, + request.callbackUrl, Map.empty ++ - settings.minimumFileSize.map(s => Map("minFileSize" -> s.toString).head) ++ - settings.maximumFileSize.map(s => Map("maxFileSize" -> s.toString).head) ++ + request.minimumFileSize.map(s => Map("minFileSize" -> s.toString).head) ++ + request.maximumFileSize.map(s => Map("maxFileSize" -> s.toString).head) ++ Map("sessionId" -> sessionId) ++ Map("requestId" -> requestId) ++ - settings.successRedirect.map(url => Map("success_action_redirect" -> url)).getOrElse(Map.empty) + request.successRedirect.map(url => Map("success_action_redirect" -> url)).getOrElse(Map.empty) ) ) } diff --git a/test/model/PrepareUploadServiceSpec.scala b/test/model/PrepareUploadServiceSpec.scala index 4f41d4b..e4fa5d9 100644 --- a/test/model/PrepareUploadServiceSpec.scala +++ b/test/model/PrepareUploadServiceSpec.scala @@ -61,7 +61,7 @@ class PrepareUploadServiceSpec extends UnitSpec with Matchers with GivenWhenThen val receivedAt: Instant = Instant.now() - "S3 Upload Service" should { + "S3 Prepare Upload Service" should { def service(metrics: Metrics) = new PrepareUploadService(s3PostSigner, serviceConfiguration, metrics) @@ -69,7 +69,7 @@ class PrepareUploadServiceSpec extends UnitSpec with Matchers with GivenWhenThen val metrics = metricsStub() - Given("there are have valid upload settings") + Given("there are valid upload settings") val callbackUrl = "http://www.callback.com" @@ -109,7 +109,7 @@ class PrepareUploadServiceSpec extends UnitSpec with Matchers with GivenWhenThen val metrics = metricsStub() - Given("there are have valid upload settings with size limits") + Given("there are valid upload settings with size limits") val callbackUrl = "http://www.callback.com" @@ -159,7 +159,7 @@ class PrepareUploadServiceSpec extends UnitSpec with Matchers with GivenWhenThen "fail when maximum file size is greater than global limit" in { - Given("there are upload settings with maximum file size greater than global limit") + Given("there upload settings with maximum file size greater than global limit") val metrics = metricsStub() @@ -184,7 +184,7 @@ class PrepareUploadServiceSpec extends UnitSpec with Matchers with GivenWhenThen "fail when minimum file size is greater than maximum file size" in { - Given("there are upload settings with minimum file size greater than maximum size ") + Given("there are upload settings with minimum file size greater than maximum size") val metrics = metricsStub() @@ -211,7 +211,7 @@ class PrepareUploadServiceSpec extends UnitSpec with Matchers with GivenWhenThen val metrics = metricsStub() - Given("there are have valid upload settings") + Given("there are valid upload settings") val callbackUrl = "http://www.callback.com" From 38e4f63b6c3f72854867b6a558f20448717ea431 Mon Sep 17 00:00:00 2001 From: chotai Date: Thu, 30 May 2019 01:04:59 +0100 Subject: [PATCH 19/96] BDOG-188 Move json serde to corresponding case classes --- app/controllers/PrepareUploadController.scala | 30 ++++--------------- .../model/PrepareUploadRequestV1.scala | 19 ++++++++++++ .../model/PreparedUploadResponse.scala | 11 +++++++ app/controllers/model/Reference.scala | 8 +++++ .../model/UploadFormTemplate.scala | 10 +++++++ app/services/PrepareUploadService.scala | 2 +- 6 files changed, 54 insertions(+), 26 deletions(-) diff --git a/app/controllers/PrepareUploadController.scala b/app/controllers/PrepareUploadController.scala index 34a70a2..b075edc 100644 --- a/app/controllers/PrepareUploadController.scala +++ b/app/controllers/PrepareUploadController.scala @@ -4,13 +4,10 @@ import java.net.URL import java.time.{Clock, Instant} import config.ServiceConfiguration -import controllers.model.{PrepareUploadRequestV1, PreparedUploadResponse, UploadFormTemplate} +import controllers.model.{PrepareUploadRequestV1, PreparedUploadResponse} import javax.inject.{Inject, Singleton} import play.api.Logger -import play.api.data.validation.ValidationError -import play.api.libs.functional.syntax._ -import play.api.libs.json.Reads._ -import play.api.libs.json.{JsPath, Json, Writes, _} +import play.api.libs.json.{Json, _} import play.api.mvc.{Action, Result} import services.PrepareUploadService import uk.gov.hmrc.play.bootstrap.controller.BaseController @@ -27,25 +24,8 @@ class PrepareUploadController @Inject()( extends BaseController with UserAgentFilter { - implicit val uploadSettingsReads: Reads[PrepareUploadRequestV1] = ( - (JsPath \ "callbackUrl").read[String] and - (JsPath \ "minimumFileSize").readNullable[Int](min(0)) and - (JsPath \ "maximumFileSize").readNullable[Int](min(0) keepAnd max(prepareUploadService.globalFileSizeLimit + 1)) and - (JsPath \ "expectedContentType").readNullable[String] and - (JsPath \ "successRedirect").readNullable[String] - )(PrepareUploadRequestV1.apply _) - .filter(ValidationError("Maximum file size must be equal or greater than minimum file size"))(settings => - settings.minimumFileSize.getOrElse(0) <= settings.maximumFileSize.getOrElse( - prepareUploadService.globalFileSizeLimit)) - - implicit val uploadFormTemplateWrites: Writes[UploadFormTemplate] = Json.writes[UploadFormTemplate] - - implicit val preparedUploadWrites: Writes[PreparedUploadResponse] = new Writes[PreparedUploadResponse] { - def writes(preparedUpload: PreparedUploadResponse): JsValue = Json.obj( - "reference" -> preparedUpload.reference.value, - "uploadRequest" -> preparedUpload.uploadRequest - ) - } + implicit val prepareUploadRequestReads: Reads[PrepareUploadRequestV1] = + PrepareUploadRequestV1.reads(prepareUploadService.globalFileSizeLimit) def prepareUpload(): Action[JsValue] = Action.async(parse.json) { implicit request => @@ -62,7 +42,7 @@ class PrepareUploadController @Inject()( prepareUploadService .prepareUpload(prepareUploadRequest, consumingService, requestId, sessionId, receivedAt) - Future.successful(Ok(Json.toJson(result))) + Future.successful(Ok(Json.toJson(result)(PreparedUploadResponse.writes))) } } } diff --git a/app/controllers/model/PrepareUploadRequestV1.scala b/app/controllers/model/PrepareUploadRequestV1.scala index 7fbb225..a46a824 100644 --- a/app/controllers/model/PrepareUploadRequestV1.scala +++ b/app/controllers/model/PrepareUploadRequestV1.scala @@ -1,4 +1,8 @@ package controllers.model +import play.api.data.validation.ValidationError +import play.api.libs.json.{JsPath, Reads} +import play.api.libs.json.Reads.{max, min} +import play.api.libs.functional.syntax._ case class PrepareUploadRequestV1( callbackUrl: String, @@ -6,3 +10,18 @@ case class PrepareUploadRequestV1( maximumFileSize: Option[Int], expectedContentType: Option[String], successRedirect: Option[String] = None) + +object PrepareUploadRequestV1 { + + def reads(maxFileSize: Int): Reads[PrepareUploadRequestV1] = + ( + (JsPath \ "callbackUrl").read[String] and + (JsPath \ "minimumFileSize").readNullable[Int](min(0)) and + (JsPath \ "maximumFileSize").readNullable[Int](min(0) keepAnd max(maxFileSize + 1)) and + (JsPath \ "expectedContentType").readNullable[String] and + (JsPath \ "successRedirect").readNullable[String] + )(PrepareUploadRequestV1.apply _) + .filter(ValidationError("Maximum file size must be equal or greater than minimum file size"))(request => + request.minimumFileSize.getOrElse(0) <= request.maximumFileSize.getOrElse(maxFileSize)) + +} diff --git a/app/controllers/model/PreparedUploadResponse.scala b/app/controllers/model/PreparedUploadResponse.scala index f6c17e8..4261fc7 100644 --- a/app/controllers/model/PreparedUploadResponse.scala +++ b/app/controllers/model/PreparedUploadResponse.scala @@ -1,3 +1,14 @@ package controllers.model +import play.api.libs.functional.syntax._ +import play.api.libs.json.{OWrites, __} + case class PreparedUploadResponse(reference: Reference, uploadRequest: UploadFormTemplate) + +object PreparedUploadResponse { + + val writes: OWrites[PreparedUploadResponse] = + ((__ \ "reference").write(Reference.writes) + ~ (__ \ "uploadRequest").write(UploadFormTemplate.writes))(unlift(PreparedUploadResponse.unapply)) + +} diff --git a/app/controllers/model/Reference.scala b/app/controllers/model/Reference.scala index b0bbb2e..8328ad7 100644 --- a/app/controllers/model/Reference.scala +++ b/app/controllers/model/Reference.scala @@ -1,3 +1,11 @@ package controllers.model +import play.api.libs.json.{JsString, JsValue, Writes} case class Reference(value: String) + +object Reference { + + val writes: Writes[Reference] = new Writes[Reference] { + override def writes(o: Reference): JsValue = JsString(o.value) + } +} diff --git a/app/controllers/model/UploadFormTemplate.scala b/app/controllers/model/UploadFormTemplate.scala index 3eb3d44..9cbf3e8 100644 --- a/app/controllers/model/UploadFormTemplate.scala +++ b/app/controllers/model/UploadFormTemplate.scala @@ -1,3 +1,13 @@ package controllers.model +import play.api.libs.functional.syntax._ +import play.api.libs.json.{OWrites, __} case class UploadFormTemplate(href: String, fields: Map[String, String]) + +object UploadFormTemplate { + + val writes: OWrites[UploadFormTemplate] = + ((__ \ "href").write[String] + ~ (__ \ "fields").write[Map[String, String]])(unlift(UploadFormTemplate.unapply)) + +} diff --git a/app/services/PrepareUploadService.scala b/app/services/PrepareUploadService.scala index 03708cc..cdeea53 100644 --- a/app/services/PrepareUploadService.scala +++ b/app/services/PrepareUploadService.scala @@ -27,7 +27,7 @@ class PrepareUploadService @Inject()( val expiration = receivedAt.plus(configuration.fileExpirationPeriod) val result = - controllers.model.PreparedUploadResponse( + PreparedUploadResponse( reference = reference, uploadRequest = generatePost(reference.value, expiration, settings, consumingService, requestId, sessionId, receivedAt)) From 610e08728e37988e69bc5e93ff8dfdce35a4febd Mon Sep 17 00:00:00 2001 From: chotai Date: Thu, 30 May 2019 01:28:53 +0100 Subject: [PATCH 20/96] BDOG-188 Decouple request to service to arguement to PrepareUploadService --- app/controllers/PrepareUploadController.scala | 7 ++- .../model/PrepareUploadRequestV1.scala | 14 ++++- app/services/PrepareUploadService.scala | 7 ++- app/services/model/UploadSettings.scala | 8 +++ .../PrepareUploadControllerSpec.scala | 11 ++-- test/model/PrepareUploadServiceSpec.scala | 57 ++++++++++--------- 6 files changed, 66 insertions(+), 38 deletions(-) create mode 100644 app/services/model/UploadSettings.scala diff --git a/app/controllers/PrepareUploadController.scala b/app/controllers/PrepareUploadController.scala index b075edc..e7e9d76 100644 --- a/app/controllers/PrepareUploadController.scala +++ b/app/controllers/PrepareUploadController.scala @@ -40,7 +40,12 @@ class PrepareUploadController @Inject()( val requestId = hc(request).requestId.map(_.value).getOrElse("n/a") val result: PreparedUploadResponse = prepareUploadService - .prepareUpload(prepareUploadRequest, consumingService, requestId, sessionId, receivedAt) + .prepareUpload( + prepareUploadRequest.toUploadSettings, + consumingService, + requestId, + sessionId, + receivedAt) Future.successful(Ok(Json.toJson(result)(PreparedUploadResponse.writes))) } diff --git a/app/controllers/model/PrepareUploadRequestV1.scala b/app/controllers/model/PrepareUploadRequestV1.scala index a46a824..2012770 100644 --- a/app/controllers/model/PrepareUploadRequestV1.scala +++ b/app/controllers/model/PrepareUploadRequestV1.scala @@ -3,16 +3,28 @@ import play.api.data.validation.ValidationError import play.api.libs.json.{JsPath, Reads} import play.api.libs.json.Reads.{max, min} import play.api.libs.functional.syntax._ +import services.model.UploadSettings case class PrepareUploadRequestV1( callbackUrl: String, minimumFileSize: Option[Int], maximumFileSize: Option[Int], expectedContentType: Option[String], - successRedirect: Option[String] = None) + successRedirect: Option[String]) object PrepareUploadRequestV1 { + implicit class PrepareUploadRequestV1Ops(request: PrepareUploadRequestV1) { + + def toUploadSettings: UploadSettings = UploadSettings( + callbackUrl = request.callbackUrl, + minimumFileSize = request.minimumFileSize, + maximumFileSize = request.maximumFileSize, + expectedContentType = request.expectedContentType, + successRedirect = request.successRedirect + ) + } + def reads(maxFileSize: Int): Reads[PrepareUploadRequestV1] = ( (JsPath \ "callbackUrl").read[String] and diff --git a/app/services/PrepareUploadService.scala b/app/services/PrepareUploadService.scala index cdeea53..8f9a8b9 100644 --- a/app/services/PrepareUploadService.scala +++ b/app/services/PrepareUploadService.scala @@ -6,10 +6,11 @@ import java.util.UUID import com.kenshoo.play.metrics.Metrics import config.ServiceConfiguration import connectors.model.{ContentLengthRange, UploadFormGenerator, UploadParameters} -import controllers.model.{PrepareUploadRequestV1, PreparedUploadResponse, Reference, UploadFormTemplate} +import controllers.model.{PreparedUploadResponse, Reference, UploadFormTemplate} import javax.inject.{Inject, Singleton} import org.slf4j.MDC import play.api.Logger +import services.model.UploadSettings @Singleton class PrepareUploadService @Inject()( @@ -18,7 +19,7 @@ class PrepareUploadService @Inject()( metrics: Metrics) { def prepareUpload( - settings: PrepareUploadRequestV1, + settings: UploadSettings, consumingService: String, requestId: String, sessionId: String, @@ -50,7 +51,7 @@ class PrepareUploadService @Inject()( private def generatePost( key: String, expiration: Instant, - settings: PrepareUploadRequestV1, + settings: UploadSettings, consumingService: String, requestId: String, sessionId: String, diff --git a/app/services/model/UploadSettings.scala b/app/services/model/UploadSettings.scala new file mode 100644 index 0000000..2c067b2 --- /dev/null +++ b/app/services/model/UploadSettings.scala @@ -0,0 +1,8 @@ +package services.model + +case class UploadSettings( + callbackUrl: String, + minimumFileSize: Option[Int], + maximumFileSize: Option[Int], + expectedContentType: Option[String], + successRedirect: Option[String]) diff --git a/test/controllers/PrepareUploadControllerSpec.scala b/test/controllers/PrepareUploadControllerSpec.scala index baf44b5..1762698 100644 --- a/test/controllers/PrepareUploadControllerSpec.scala +++ b/test/controllers/PrepareUploadControllerSpec.scala @@ -17,6 +17,7 @@ import play.api.mvc.Results.Ok import play.api.test.Helpers.contentAsString import play.api.test.{FakeRequest, Helpers} import services.PrepareUploadService +import services.model.UploadSettings import uk.gov.hmrc.play.test.UnitSpec import scala.concurrent.Future @@ -329,19 +330,19 @@ class PrepareUploadControllerSpec extends UnitSpec with Matchers with GivenWhenT .when(service.prepareUpload(any(), any(), any(), any(), any())) .thenAnswer(new Answer[PreparedUploadResponse]() { override def answer(invocationOnMock: InvocationOnMock): PreparedUploadResponse = { - val request = invocationOnMock.getArgument[PrepareUploadRequestV1](0) + val settings = invocationOnMock.getArgument[UploadSettings](0) val requestId = invocationOnMock.getArgument[String](2) val sessionId = invocationOnMock.getArgument[String](3) PreparedUploadResponse( Reference("TEST"), UploadFormTemplate( - request.callbackUrl, + settings.callbackUrl, Map.empty ++ - request.minimumFileSize.map(s => Map("minFileSize" -> s.toString).head) ++ - request.maximumFileSize.map(s => Map("maxFileSize" -> s.toString).head) ++ + settings.minimumFileSize.map(s => Map("minFileSize" -> s.toString).head) ++ + settings.maximumFileSize.map(s => Map("maxFileSize" -> s.toString).head) ++ Map("sessionId" -> sessionId) ++ Map("requestId" -> requestId) ++ - request.successRedirect.map(url => Map("success_action_redirect" -> url)).getOrElse(Map.empty) + settings.successRedirect.map(url => Map("success_action_redirect" -> url)).getOrElse(Map.empty) ) ) } diff --git a/test/model/PrepareUploadServiceSpec.scala b/test/model/PrepareUploadServiceSpec.scala index e4fa5d9..dd81143 100644 --- a/test/model/PrepareUploadServiceSpec.scala +++ b/test/model/PrepareUploadServiceSpec.scala @@ -7,9 +7,9 @@ import com.codahale.metrics.MetricRegistry import com.kenshoo.play.metrics.Metrics import config.ServiceConfiguration import connectors.model.{UploadFormGenerator, UploadParameters} -import controllers.model.PrepareUploadRequestV1 import org.scalatest.{GivenWhenThen, Matchers} import services.PrepareUploadService +import services.model.UploadSettings import uk.gov.hmrc.play.test.UnitSpec class PrepareUploadServiceSpec extends UnitSpec with Matchers with GivenWhenThen { @@ -73,11 +73,12 @@ class PrepareUploadServiceSpec extends UnitSpec with Matchers with GivenWhenThen val callbackUrl = "http://www.callback.com" - val uploadSettings = PrepareUploadRequestV1( + val uploadSettings = UploadSettings( callbackUrl = callbackUrl, minimumFileSize = None, maximumFileSize = None, - expectedContentType = Some("application/xml")) + expectedContentType = Some("application/xml"), + successRedirect = None) When("we setup the upload") @@ -113,12 +114,12 @@ class PrepareUploadServiceSpec extends UnitSpec with Matchers with GivenWhenThen val callbackUrl = "http://www.callback.com" - val uploadSettings = - PrepareUploadRequestV1( - callbackUrl = callbackUrl, - minimumFileSize = Some(100), - maximumFileSize = Some(200), - expectedContentType = None) + val uploadSettings = UploadSettings( + callbackUrl = callbackUrl, + minimumFileSize = Some(100), + maximumFileSize = Some(200), + expectedContentType = None, + successRedirect = None) When("we setup the upload") @@ -139,12 +140,12 @@ class PrepareUploadServiceSpec extends UnitSpec with Matchers with GivenWhenThen val callbackUrl = "http://www.callback.com" - val uploadSettings = - PrepareUploadRequestV1( - callbackUrl = callbackUrl, - minimumFileSize = Some(-1), - maximumFileSize = Some(1024), - expectedContentType = None) + val uploadSettings = UploadSettings( + callbackUrl = callbackUrl, + minimumFileSize = Some(-1), + maximumFileSize = Some(1024), + expectedContentType = None, + successRedirect = None) When("we setup the upload") Then("an exception should be thrown") @@ -165,12 +166,12 @@ class PrepareUploadServiceSpec extends UnitSpec with Matchers with GivenWhenThen val callbackUrl = "http://www.callback.com" - val uploadSettings = - PrepareUploadRequestV1( - callbackUrl = callbackUrl, - minimumFileSize = Some(0), - maximumFileSize = Some(1025), - expectedContentType = None) + val uploadSettings = UploadSettings( + callbackUrl = callbackUrl, + minimumFileSize = Some(0), + maximumFileSize = Some(1025), + expectedContentType = None, + successRedirect = None) When("we setup the upload") Then("an exception should be thrown") @@ -190,12 +191,12 @@ class PrepareUploadServiceSpec extends UnitSpec with Matchers with GivenWhenThen val callbackUrl = "http://www.callback.com" - val uploadSettings = - PrepareUploadRequestV1( - callbackUrl = callbackUrl, - minimumFileSize = Some(1024), - maximumFileSize = Some(0), - expectedContentType = None) + val uploadSettings = UploadSettings( + callbackUrl = callbackUrl, + minimumFileSize = Some(1024), + maximumFileSize = Some(0), + expectedContentType = None, + successRedirect = None) When("we setup the upload") Then("an exception should be thrown") @@ -215,7 +216,7 @@ class PrepareUploadServiceSpec extends UnitSpec with Matchers with GivenWhenThen val callbackUrl = "http://www.callback.com" - val uploadSettings = PrepareUploadRequestV1( + val uploadSettings = UploadSettings( callbackUrl = callbackUrl, minimumFileSize = None, maximumFileSize = None, From d9c02a8e80549014a96649f405aba5ae73de2b77 Mon Sep 17 00:00:00 2001 From: chotai Date: Thu, 30 May 2019 01:53:21 +0100 Subject: [PATCH 21/96] BDOG-188 Update UploadSettings with optional errorRedirect field --- app/connectors/model/AwsCredentials.scala | 3 ++ app/connectors/model/UploadParameters.scala | 3 +- app/connectors/s3/PolicySigner.scala | 1 + app/connectors/s3/S3UploadFormGenerator.scala | 8 +--- .../s3/S3UploadFormGeneratorProvider.scala | 2 +- .../model/PrepareUploadRequestV1.scala | 3 +- app/services/PrepareUploadService.scala | 3 +- app/services/model/UploadSettings.scala | 3 +- test/connectors/s3/PolicySignerSpec.scala | 1 + .../s3/S3UploadFormGeneratorSpec.scala | 46 +++++++++---------- test/model/PrepareUploadServiceSpec.scala | 18 +++++--- 11 files changed, 50 insertions(+), 41 deletions(-) create mode 100644 app/connectors/model/AwsCredentials.scala diff --git a/app/connectors/model/AwsCredentials.scala b/app/connectors/model/AwsCredentials.scala new file mode 100644 index 0000000..7652a7c --- /dev/null +++ b/app/connectors/model/AwsCredentials.scala @@ -0,0 +1,3 @@ +package connectors.model + +final case class AwsCredentials(accessKeyId: String, secretKey: String, sessionToken: Option[String]) diff --git a/app/connectors/model/UploadParameters.scala b/app/connectors/model/UploadParameters.scala index 46484f4..af261a8 100644 --- a/app/connectors/model/UploadParameters.scala +++ b/app/connectors/model/UploadParameters.scala @@ -9,5 +9,6 @@ case class UploadParameters( additionalMetadata: Map[String, String], contentLengthRange: ContentLengthRange, expectedContentType: Option[String], - successRedirect: Option[String] + successRedirect: Option[String], + errorRedirect: Option[String] ) diff --git a/app/connectors/s3/PolicySigner.scala b/app/connectors/s3/PolicySigner.scala index 1adcdbc..a1e7a84 100644 --- a/app/connectors/s3/PolicySigner.scala +++ b/app/connectors/s3/PolicySigner.scala @@ -3,6 +3,7 @@ package connectors.s3 import java.nio.charset.Charset import com.amazonaws.util.{BinaryUtils, StringUtils} +import connectors.model.AwsCredentials import javax.crypto.Mac import javax.crypto.spec.SecretKeySpec diff --git a/app/connectors/s3/S3UploadFormGenerator.scala b/app/connectors/s3/S3UploadFormGenerator.scala index 3c3630e..befff2a 100644 --- a/app/connectors/s3/S3UploadFormGenerator.scala +++ b/app/connectors/s3/S3UploadFormGenerator.scala @@ -4,15 +4,9 @@ import java.time.format.DateTimeFormatter import java.time.format.DateTimeFormatter.ISO_INSTANT import java.time.{Instant, ZoneOffset} -import connectors.model.{UploadFormGenerator, UploadParameters} +import connectors.model.{AwsCredentials, UploadFormGenerator, UploadParameters} import play.api.libs.json.{JsArray, JsValue, Json} -final case class AwsCredentials( - accessKeyId: String, - secretKey: String, - sessionToken: Option[String] -) - class S3UploadFormGenerator( credentials: AwsCredentials, regionName: String, diff --git a/app/connectors/s3/S3UploadFormGeneratorProvider.scala b/app/connectors/s3/S3UploadFormGeneratorProvider.scala index 5c6087f..b9d378a 100644 --- a/app/connectors/s3/S3UploadFormGeneratorProvider.scala +++ b/app/connectors/s3/S3UploadFormGeneratorProvider.scala @@ -3,7 +3,7 @@ package connectors.s3 import config.ServiceConfiguration import java.time.{Clock, Instant} -import connectors.model.UploadFormGenerator +import connectors.model.{AwsCredentials, UploadFormGenerator} import javax.inject.{Inject, Provider, Singleton} @Singleton diff --git a/app/controllers/model/PrepareUploadRequestV1.scala b/app/controllers/model/PrepareUploadRequestV1.scala index 2012770..bc23a2f 100644 --- a/app/controllers/model/PrepareUploadRequestV1.scala +++ b/app/controllers/model/PrepareUploadRequestV1.scala @@ -21,7 +21,8 @@ object PrepareUploadRequestV1 { minimumFileSize = request.minimumFileSize, maximumFileSize = request.maximumFileSize, expectedContentType = request.expectedContentType, - successRedirect = request.successRedirect + successRedirect = request.successRedirect, + errorRedirect = None ) } diff --git a/app/services/PrepareUploadService.scala b/app/services/PrepareUploadService.scala index 8f9a8b9..f936ed1 100644 --- a/app/services/PrepareUploadService.scala +++ b/app/services/PrepareUploadService.scala @@ -78,7 +78,8 @@ class PrepareUploadService @Inject()( ), contentLengthRange = ContentLengthRange(minFileSize, maxFileSize), expectedContentType = settings.expectedContentType, - successRedirect = settings.successRedirect + successRedirect = settings.successRedirect, + errorRedirect = settings.errorRedirect ) val form = postSigner.generateFormFields(uploadParameters) diff --git a/app/services/model/UploadSettings.scala b/app/services/model/UploadSettings.scala index 2c067b2..089d7a2 100644 --- a/app/services/model/UploadSettings.scala +++ b/app/services/model/UploadSettings.scala @@ -5,4 +5,5 @@ case class UploadSettings( minimumFileSize: Option[Int], maximumFileSize: Option[Int], expectedContentType: Option[String], - successRedirect: Option[String]) + successRedirect: Option[String], + errorRedirect: Option[String]) diff --git a/test/connectors/s3/PolicySignerSpec.scala b/test/connectors/s3/PolicySignerSpec.scala index 528f621..f0073f4 100644 --- a/test/connectors/s3/PolicySignerSpec.scala +++ b/test/connectors/s3/PolicySignerSpec.scala @@ -1,5 +1,6 @@ package connectors.s3 +import connectors.model.AwsCredentials import org.scalatest.{Matchers, WordSpec} class PolicySignerSpec extends WordSpec with Matchers { diff --git a/test/connectors/s3/S3UploadFormGeneratorSpec.scala b/test/connectors/s3/S3UploadFormGeneratorSpec.scala index 0e127ad..3f129ac 100644 --- a/test/connectors/s3/S3UploadFormGeneratorSpec.scala +++ b/test/connectors/s3/S3UploadFormGeneratorSpec.scala @@ -3,7 +3,7 @@ package connectors.s3 import java.time.Instant import java.util.Base64 -import connectors.model.{ContentLengthRange, UploadParameters} +import connectors.model.{AwsCredentials, ContentLengthRange, UploadParameters} import org.mockito.ArgumentMatchers.any import org.mockito.Mockito.{verify, when} import org.scalatest.mockito.MockitoSugar @@ -30,17 +30,17 @@ class S3UploadFormGeneratorSpec extends WordSpec with GivenWhenThen with Matcher And("there are valid upload parameters") val expirationTimestamp = "1997-07-16T19:20:40Z" - val uploadParameters = - UploadParameters( - expirationDateTime = Instant.parse(expirationTimestamp), - bucketName = "test-bucket", - objectKey = "test-key", - acl = "private", - additionalMetadata = Map("key1" -> "value1"), - contentLengthRange = ContentLengthRange(0, 1024), - expectedContentType = Some("application/xml"), - successRedirect = Some("http://test.com/abc") - ) + val uploadParameters = UploadParameters( + expirationDateTime = Instant.parse(expirationTimestamp), + bucketName = "test-bucket", + objectKey = "test-key", + acl = "private", + additionalMetadata = Map("key1" -> "value1"), + contentLengthRange = ContentLengthRange(0, 1024), + expectedContentType = Some("application/xml"), + successRedirect = Some("http://test.com/abc"), + errorRedirect = None + ) When("form fields are generated") val result = generator.generateFormFields(uploadParameters) @@ -106,17 +106,17 @@ class S3UploadFormGeneratorSpec extends WordSpec with GivenWhenThen with Matcher And("there are valid upload parameters") val expirationTimestamp = "1997-07-16T19:20:40Z" - val uploadParameters = - UploadParameters( - expirationDateTime = Instant.parse(expirationTimestamp), - bucketName = "test-bucket", - objectKey = "test-key", - acl = "private", - additionalMetadata = Map("key1" -> "value1"), - contentLengthRange = ContentLengthRange(0, 1024), - expectedContentType = Some("application/xml"), - successRedirect = Some("http://test.server/success") - ) + val uploadParameters = UploadParameters( + expirationDateTime = Instant.parse(expirationTimestamp), + bucketName = "test-bucket", + objectKey = "test-key", + acl = "private", + additionalMetadata = Map("key1" -> "value1"), + contentLengthRange = ContentLengthRange(0, 1024), + expectedContentType = Some("application/xml"), + successRedirect = Some("http://test.server/success"), + errorRedirect = None + ) When("form fields are generated") val result = generator.generateFormFields(uploadParameters) diff --git a/test/model/PrepareUploadServiceSpec.scala b/test/model/PrepareUploadServiceSpec.scala index dd81143..c80430a 100644 --- a/test/model/PrepareUploadServiceSpec.scala +++ b/test/model/PrepareUploadServiceSpec.scala @@ -78,7 +78,8 @@ class PrepareUploadServiceSpec extends UnitSpec with Matchers with GivenWhenThen minimumFileSize = None, maximumFileSize = None, expectedContentType = Some("application/xml"), - successRedirect = None) + successRedirect = None, + errorRedirect = None) When("we setup the upload") @@ -119,7 +120,8 @@ class PrepareUploadServiceSpec extends UnitSpec with Matchers with GivenWhenThen minimumFileSize = Some(100), maximumFileSize = Some(200), expectedContentType = None, - successRedirect = None) + successRedirect = None, + errorRedirect = None) When("we setup the upload") @@ -145,7 +147,8 @@ class PrepareUploadServiceSpec extends UnitSpec with Matchers with GivenWhenThen minimumFileSize = Some(-1), maximumFileSize = Some(1024), expectedContentType = None, - successRedirect = None) + successRedirect = None, + errorRedirect = None) When("we setup the upload") Then("an exception should be thrown") @@ -171,7 +174,8 @@ class PrepareUploadServiceSpec extends UnitSpec with Matchers with GivenWhenThen minimumFileSize = Some(0), maximumFileSize = Some(1025), expectedContentType = None, - successRedirect = None) + successRedirect = None, + errorRedirect = None) When("we setup the upload") Then("an exception should be thrown") @@ -196,7 +200,8 @@ class PrepareUploadServiceSpec extends UnitSpec with Matchers with GivenWhenThen minimumFileSize = Some(1024), maximumFileSize = Some(0), expectedContentType = None, - successRedirect = None) + successRedirect = None, + errorRedirect = None) When("we setup the upload") Then("an exception should be thrown") @@ -221,7 +226,8 @@ class PrepareUploadServiceSpec extends UnitSpec with Matchers with GivenWhenThen minimumFileSize = None, maximumFileSize = None, expectedContentType = Some("application/xml"), - successRedirect = Some("https://new.service/page1") + successRedirect = Some("https://new.service/page1"), + errorRedirect = None ) When("we setup the upload") From 138c11a62888215f1d90dee71ce1e5ec9be4692d Mon Sep 17 00:00:00 2001 From: chotai Date: Thu, 30 May 2019 02:00:03 +0100 Subject: [PATCH 22/96] BDOG-188 Update S3 Upload Form Generator form fields and policy with errorRedirect --- app/connectors/s3/S3UploadFormGenerator.scala | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/app/connectors/s3/S3UploadFormGenerator.scala b/app/connectors/s3/S3UploadFormGenerator.scala index befff2a..f68bc60 100644 --- a/app/connectors/s3/S3UploadFormGenerator.scala +++ b/app/connectors/s3/S3UploadFormGenerator.scala @@ -76,7 +76,10 @@ class S3UploadFormGenerator( val successRedirect = uploadParameters.successRedirect.map(v => Map("success_action_redirect" -> v)).getOrElse(Map.empty) - fields ++ metadataFields ++ sessionCredentials ++ contentTypeField ++ successRedirect + val errorRedirect = + uploadParameters.errorRedirect.map(v => Map("error_action_redirect" -> v)).getOrElse(Map.empty) + + fields ++ metadataFields ++ sessionCredentials ++ contentTypeField ++ successRedirect ++ errorRedirect } private def buildPolicy( @@ -99,6 +102,9 @@ class S3UploadFormGenerator( val successRedirectConstraint = uploadParameters.successRedirect.map(redirect => Json.obj("success_action_redirect" -> redirect)) + val errorRedirectConstraint = + uploadParameters.errorRedirect.map(redirect => Json.obj("error_action_redirect" -> redirect)) + val policyDocument = Json.obj( "expiration" -> ISO_INSTANT.format(uploadParameters.expirationDateTime), "conditions" -> JsArray( @@ -117,6 +123,7 @@ class S3UploadFormGenerator( ++ metadataJson ++ contentTypeConstraintJson ++ successRedirectConstraint + ++ errorRedirectConstraint ) ) From f672486a1560d6f70c061e6e46169f4b2df59140 Mon Sep 17 00:00:00 2001 From: chotai Date: Thu, 30 May 2019 02:14:25 +0100 Subject: [PATCH 23/96] BDOG-188 Add error redirect test for S3 Upload Form Generator --- .../s3/S3UploadFormGeneratorSpec.scala | 42 +++++++++++++++++-- 1 file changed, 38 insertions(+), 4 deletions(-) diff --git a/test/connectors/s3/S3UploadFormGeneratorSpec.scala b/test/connectors/s3/S3UploadFormGeneratorSpec.scala index 3f129ac..dc1d9c7 100644 --- a/test/connectors/s3/S3UploadFormGeneratorSpec.scala +++ b/test/connectors/s3/S3UploadFormGeneratorSpec.scala @@ -39,7 +39,7 @@ class S3UploadFormGeneratorSpec extends WordSpec with GivenWhenThen with Matcher contentLengthRange = ContentLengthRange(0, 1024), expectedContentType = Some("application/xml"), successRedirect = Some("http://test.com/abc"), - errorRedirect = None + errorRedirect = Some("http://test.com/error") ) When("form fields are generated") @@ -61,6 +61,7 @@ class S3UploadFormGeneratorSpec extends WordSpec with GivenWhenThen with Matcher ((policy \ "conditions").get \\ "x-amz-meta-key1").head.as[String] shouldBe "value1" ((policy \ "conditions").get \\ "Content-Type").head.as[String] shouldBe "application/xml" ((policy \ "conditions").get \\ "success_action_redirect").head.as[String] shouldBe "http://test.com/abc" + ((policy \ "conditions").get \\ "error_action_redirect").head.as[String] shouldBe "http://test.com/error" val conditions = (policy \ "conditions").as[JsArray].value val arrayConditions: Seq[Seq[JsValue]] = conditions.flatMap(_.asOpt[JsArray].map(_.value)) @@ -73,8 +74,8 @@ class S3UploadFormGeneratorSpec extends WordSpec with GivenWhenThen with Matcher And("policy contains proper filename constraint") val filenameMetadataCondition = arrayConditions.toList.find(_.toIndexedSeq(1).asOpt[String].contains("$x-amz-meta-original-filename")) - filenameMetadataCondition.get(0).as[String] shouldBe "starts-with" - filenameMetadataCondition.get(2).as[String] shouldBe "" + filenameMetadataCondition.get.head.as[String] shouldBe "starts-with" + filenameMetadataCondition.get(2).as[String] shouldBe "" And("policy's signature is correct") verify(policySigner).signPolicy(credentials, "19970716", regionName, result("policy")) @@ -91,7 +92,7 @@ class S3UploadFormGeneratorSpec extends WordSpec with GivenWhenThen with Matcher result("x-amz-meta-original-filename") shouldBe "${filename}" } - "generate a signed link with a redirect" in { + "generate a signed link with a success redirect" in { Given("there is a properly configured form generator with AWS credentials") val credentials = AwsCredentials("accessKeyId", "secretKey", Some("session-token")) @@ -123,6 +124,39 @@ class S3UploadFormGeneratorSpec extends WordSpec with GivenWhenThen with Matcher result("success_action_redirect") shouldBe "http://test.server/success" } + + "generate a signed link with a error redirect" in { + + Given("there is a properly configured form generator with AWS credentials") + val credentials = AwsCredentials("accessKeyId", "secretKey", Some("session-token")) + val regionName = "us-east-1" + val currentTime = () => Instant.parse("1997-07-16T19:20:30Z") + val policySigner = mock[PolicySigner] + val testSignature = "test-signature" + + when(policySigner.signPolicy(any(), any(), any(), any())).thenReturn(testSignature) + + val generator = new S3UploadFormGenerator(credentials, regionName, currentTime, policySigner) + + And("there are valid upload parameters") + val expirationTimestamp = "1997-07-16T19:20:40Z" + val uploadParameters = UploadParameters( + expirationDateTime = Instant.parse(expirationTimestamp), + bucketName = "test-bucket", + objectKey = "test-key", + acl = "private", + additionalMetadata = Map("key1" -> "value1"), + contentLengthRange = ContentLengthRange(0, 1024), + expectedContentType = Some("application/xml"), + successRedirect = None, + errorRedirect = Some("http://test.server/error") + ) + + When("form fields are generated") + val result = generator.generateFormFields(uploadParameters) + + result("error_action_redirect") shouldBe "http://test.server/error" + } } def decodePolicyFormResult(base64EncodedPolicy: String): JsValue = { From a1eb50a855fc6c1a4b853ff625b7f6d4308c6a4d Mon Sep 17 00:00:00 2001 From: chotai Date: Thu, 30 May 2019 02:15:01 +0100 Subject: [PATCH 24/96] BDOG-188 Add error redirect test for Prepare Service Upload --- test/model/PrepareUploadServiceSpec.scala | 47 ++++++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) diff --git a/test/model/PrepareUploadServiceSpec.scala b/test/model/PrepareUploadServiceSpec.scala index c80430a..8bcbc4a 100644 --- a/test/model/PrepareUploadServiceSpec.scala +++ b/test/model/PrepareUploadServiceSpec.scala @@ -54,7 +54,8 @@ class PrepareUploadServiceSpec extends UnitSpec with Matchers with GivenWhenThen uploadParameters.expectedContentType.map { contentType => "Content-Type" -> contentType } ++ - uploadParameters.successRedirect.map { "success_redirect_url" -> _ } + uploadParameters.successRedirect.map { "success_redirect_url" -> _ } ++ + uploadParameters.errorRedirect.map { "error_redirect_url" -> _ } override def buildEndpoint(bucketName: String): String = s"$bucketName.s3" } @@ -257,6 +258,50 @@ class PrepareUploadServiceSpec extends UnitSpec with Matchers with GivenWhenThen } + "create post form that allows to upload the file with redirect on error" in { + + val metrics = metricsStub() + + Given("there are valid upload settings") + + val callbackUrl = "http://www.callback.com" + + val uploadSettings = UploadSettings( + callbackUrl = callbackUrl, + minimumFileSize = None, + maximumFileSize = None, + expectedContentType = Some("application/xml"), + successRedirect = None, + errorRedirect = Some("https://new.service/error") + ) + + When("we setup the upload") + + val result = service(metrics) + .prepareUpload(uploadSettings, "PrepareUploadServiceSpec", "some-request-id", "some-session-id", receivedAt) + + Then("proper upload request form definition should be returned") + + result.uploadRequest.href shouldBe s"${serviceConfiguration.inboundBucketName}.s3" + result.uploadRequest.fields shouldBe Map( + "bucket" -> serviceConfiguration.inboundBucketName, + "key" -> result.reference.value, + "x-amz-meta-callback-url" -> callbackUrl, + "x-amz-meta-consuming-service" -> "PrepareUploadServiceSpec", + "x-amz-meta-session-id" -> "some-session-id", + "x-amz-meta-request-id" -> "some-request-id", + "minSize" -> "0", + "maxSize" -> "1024", + "Content-Type" -> "application/xml", + "x-amz-meta-upscan-initiate-received" -> receivedAt.toString, + "error_redirect_url" -> "https://new.service/error" + ) + + And("uploadInitiated counter has been incremented") + metrics.defaultRegistry.counter("uploadInitiated").getCount shouldBe 1 + + } + } } From 27438e9f2cb2a6e7e119d5cb49356eaff4f082d6 Mon Sep 17 00:00:00 2001 From: chotai Date: Thu, 30 May 2019 09:03:36 +0100 Subject: [PATCH 25/96] BDOG-188 Add upscan initiate V2 --- app/controllers/PrepareUploadController.scala | 15 +++++-- app/controllers/model/PrepareUpload.scala | 7 ++++ .../model/PrepareUploadRequestV1.scala | 34 ++++++++-------- .../model/PrepareUploadRequestV2.scala | 39 +++++++++++++++++++ conf/app.routes | 3 +- .../PrepareUploadControllerSpec.scala | 18 ++++----- 6 files changed, 85 insertions(+), 31 deletions(-) create mode 100644 app/controllers/model/PrepareUpload.scala create mode 100644 app/controllers/model/PrepareUploadRequestV2.scala diff --git a/app/controllers/PrepareUploadController.scala b/app/controllers/PrepareUploadController.scala index e7e9d76..7078b7b 100644 --- a/app/controllers/PrepareUploadController.scala +++ b/app/controllers/PrepareUploadController.scala @@ -4,7 +4,7 @@ import java.net.URL import java.time.{Clock, Instant} import config.ServiceConfiguration -import controllers.model.{PrepareUploadRequestV1, PreparedUploadResponse} +import controllers.model.{PrepareUpload, PrepareUploadRequestV1, PrepareUploadRequestV2, PreparedUploadResponse} import javax.inject.{Inject, Singleton} import play.api.Logger import play.api.libs.json.{Json, _} @@ -27,12 +27,15 @@ class PrepareUploadController @Inject()( implicit val prepareUploadRequestReads: Reads[PrepareUploadRequestV1] = PrepareUploadRequestV1.reads(prepareUploadService.globalFileSizeLimit) - def prepareUpload(): Action[JsValue] = + implicit val prepareUploadRequestV2Reads: Reads[PrepareUploadRequestV2] = + PrepareUploadRequestV2.reads(prepareUploadService.globalFileSizeLimit) + + private def prepareUpload[T <: PrepareUpload]()(implicit reads: Reads[T], manifest: Manifest[T]): Action[JsValue] = Action.async(parse.json) { implicit request => val receivedAt = Instant.now(clock) onlyAllowedServices[JsValue] { (_, consumingService) => - withJsonBody[PrepareUploadRequestV1] { prepareUploadRequest: PrepareUploadRequestV1 => + withJsonBody[T] { prepareUploadRequest: T => withAllowedCallbackProtocol(prepareUploadRequest.callbackUrl) { Logger.debug(s"Processing request: [$prepareUploadRequest].") @@ -53,6 +56,12 @@ class PrepareUploadController @Inject()( } } + def prepareUploadV1(): Action[JsValue] = + prepareUpload[PrepareUploadRequestV1]() + + def prepareUploadV2(): Action[JsValue] = + prepareUpload[PrepareUploadRequestV2]() + private[controllers] def withAllowedCallbackProtocol[A](callbackUrl: String)( block: => Future[Result]): Future[Result] = { diff --git a/app/controllers/model/PrepareUpload.scala b/app/controllers/model/PrepareUpload.scala new file mode 100644 index 0000000..255704b --- /dev/null +++ b/app/controllers/model/PrepareUpload.scala @@ -0,0 +1,7 @@ +package controllers.model +import services.model.UploadSettings + +trait PrepareUpload { + def callbackUrl: String + def toUploadSettings: UploadSettings +} diff --git a/app/controllers/model/PrepareUploadRequestV1.scala b/app/controllers/model/PrepareUploadRequestV1.scala index bc23a2f..ab343ac 100644 --- a/app/controllers/model/PrepareUploadRequestV1.scala +++ b/app/controllers/model/PrepareUploadRequestV1.scala @@ -11,29 +11,27 @@ case class PrepareUploadRequestV1( maximumFileSize: Option[Int], expectedContentType: Option[String], successRedirect: Option[String]) + extends PrepareUpload { -object PrepareUploadRequestV1 { + def toUploadSettings: UploadSettings = UploadSettings( + callbackUrl = callbackUrl, + minimumFileSize = minimumFileSize, + maximumFileSize = maximumFileSize, + expectedContentType = expectedContentType, + successRedirect = successRedirect, + errorRedirect = None + ) - implicit class PrepareUploadRequestV1Ops(request: PrepareUploadRequestV1) { +} - def toUploadSettings: UploadSettings = UploadSettings( - callbackUrl = request.callbackUrl, - minimumFileSize = request.minimumFileSize, - maximumFileSize = request.maximumFileSize, - expectedContentType = request.expectedContentType, - successRedirect = request.successRedirect, - errorRedirect = None - ) - } +object PrepareUploadRequestV1 { def reads(maxFileSize: Int): Reads[PrepareUploadRequestV1] = - ( - (JsPath \ "callbackUrl").read[String] and - (JsPath \ "minimumFileSize").readNullable[Int](min(0)) and - (JsPath \ "maximumFileSize").readNullable[Int](min(0) keepAnd max(maxFileSize + 1)) and - (JsPath \ "expectedContentType").readNullable[String] and - (JsPath \ "successRedirect").readNullable[String] - )(PrepareUploadRequestV1.apply _) + ((JsPath \ "callbackUrl").read[String] and + (JsPath \ "minimumFileSize").readNullable[Int](min(0)) and + (JsPath \ "maximumFileSize").readNullable[Int](min(0) keepAnd max(maxFileSize + 1)) and + (JsPath \ "expectedContentType").readNullable[String] and + (JsPath \ "successRedirect").readNullable[String])(PrepareUploadRequestV1.apply _) .filter(ValidationError("Maximum file size must be equal or greater than minimum file size"))(request => request.minimumFileSize.getOrElse(0) <= request.maximumFileSize.getOrElse(maxFileSize)) diff --git a/app/controllers/model/PrepareUploadRequestV2.scala b/app/controllers/model/PrepareUploadRequestV2.scala new file mode 100644 index 0000000..828a906 --- /dev/null +++ b/app/controllers/model/PrepareUploadRequestV2.scala @@ -0,0 +1,39 @@ +package controllers.model +import play.api.data.validation.ValidationError +import play.api.libs.json.{JsPath, Reads} +import play.api.libs.json.Reads.{max, min} +import services.model.UploadSettings +import play.api.libs.functional.syntax._ + +case class PrepareUploadRequestV2( + callbackUrl: String, + successRedirect: String, + errorRedirect: String, + minimumFileSize: Option[Int], + maximumFileSize: Option[Int], + expectedContentType: Option[String]) + extends PrepareUpload { + + def toUploadSettings: UploadSettings = UploadSettings( + callbackUrl = callbackUrl, + minimumFileSize = minimumFileSize, + maximumFileSize = maximumFileSize, + expectedContentType = expectedContentType, + successRedirect = Some(successRedirect), + errorRedirect = Some(errorRedirect) + ) +} + +object PrepareUploadRequestV2 { + + def reads(maxFileSize: Int): Reads[PrepareUploadRequestV2] = + ((JsPath \ "callbackUrl").read[String] and + (JsPath \ "successRedirect").read[String] and + (JsPath \ "errorRedirect").read[String] and + (JsPath \ "minimumFileSize").readNullable[Int](min(0)) and + (JsPath \ "maximumFileSize").readNullable[Int](min(0) keepAnd max(maxFileSize + 1)) and + (JsPath \ "expectedContentType").readNullable[String])(PrepareUploadRequestV2.apply _) + .filter(ValidationError("Maximum file size must be equal or greater than minimum file size"))(request => + request.minimumFileSize.getOrElse(0) <= request.maximumFileSize.getOrElse(maxFileSize)) + +} diff --git a/conf/app.routes b/conf/app.routes index 9f94448..6e7f283 100644 --- a/conf/app.routes +++ b/conf/app.routes @@ -1 +1,2 @@ -POST /initiate @controllers.PrepareUploadController.prepareUpload +POST /initiate @controllers.PrepareUploadController.prepareUploadV1 +POST /v2/initiate @controllers.PrepareUploadController.prepareUploadV2 diff --git a/test/controllers/PrepareUploadControllerSpec.scala b/test/controllers/PrepareUploadControllerSpec.scala index 1762698..2f8d87b 100644 --- a/test/controllers/PrepareUploadControllerSpec.scala +++ b/test/controllers/PrepareUploadControllerSpec.scala @@ -5,7 +5,7 @@ import java.time.Clock import akka.actor.ActorSystem import akka.stream.ActorMaterializer import config.ServiceConfiguration -import controllers.model.{PrepareUploadRequestV1, PreparedUploadResponse, Reference, UploadFormTemplate} +import controllers.model.{PreparedUploadResponse, Reference, UploadFormTemplate} import org.mockito.ArgumentMatchers.any import org.mockito.Mockito import org.mockito.invocation.InvocationOnMock @@ -58,7 +58,7 @@ class PrepareUploadControllerSpec extends UnitSpec with Matchers with GivenWhenT When("upload initiation has been requested") - val result = controller.prepareUpload()(request) + val result = controller.prepareUploadV1()(request) Then("service returns valid response with reference and template of upload form") @@ -98,7 +98,7 @@ class PrepareUploadControllerSpec extends UnitSpec with Matchers with GivenWhenT When("upload initiation has been requested") - val result = controller.prepareUpload()(request) + val result = controller.prepareUploadV1()(request) Then("service returns valid response with reference and template of upload form") @@ -133,7 +133,7 @@ class PrepareUploadControllerSpec extends UnitSpec with Matchers with GivenWhenT When("upload initiation has been requested") - val result = controller.prepareUpload()(request) + val result = controller.prepareUploadV1()(request) Then("service returns valid response with reference and template of upload form") @@ -162,7 +162,7 @@ class PrepareUploadControllerSpec extends UnitSpec with Matchers with GivenWhenT When("upload initiation has been requested") - val result = controller.prepareUpload()(request) + val result = controller.prepareUploadV1()(request) Then("service returns valid response with reference and template of upload form") @@ -193,7 +193,7 @@ class PrepareUploadControllerSpec extends UnitSpec with Matchers with GivenWhenT When("upload initiation has been requested") - val result = controller.prepareUpload()(request) + val result = controller.prepareUploadV1()(request) Then("service returns error response") @@ -211,7 +211,7 @@ class PrepareUploadControllerSpec extends UnitSpec with Matchers with GivenWhenT When("upload initiation has been requested") - val result = controller.prepareUpload()(request) + val result = controller.prepareUploadV1()(request) Then("service returns error response") @@ -241,7 +241,7 @@ class PrepareUploadControllerSpec extends UnitSpec with Matchers with GivenWhenT When("upload initiation has been requested") - val result = controller.prepareUpload()(request) + val result = controller.prepareUploadV1()(request) Then("service returns error response") @@ -282,7 +282,7 @@ class PrepareUploadControllerSpec extends UnitSpec with Matchers with GivenWhenT When("upload initiation has been requested") - val result = controller.prepareUpload()(request) + val result = controller.prepareUploadV1()(request) Then("service returns error response") From e5dbffdd31bc3082b0cf946c99375bcac1a17b41 Mon Sep 17 00:00:00 2001 From: chotai Date: Thu, 30 May 2019 18:18:38 +0100 Subject: [PATCH 26/96] BDOG-188 Add PrepareUploadController tests --- .../PrepareUploadControllerISpec.scala | 41 ++++++--- .../PrepareUploadControllerSpec.scala | 89 +++++++++++-------- 2 files changed, 83 insertions(+), 47 deletions(-) diff --git a/it/controllers/PrepareUploadControllerISpec.scala b/it/controllers/PrepareUploadControllerISpec.scala index 2d50e24..5d65260 100644 --- a/it/controllers/PrepareUploadControllerISpec.scala +++ b/it/controllers/PrepareUploadControllerISpec.scala @@ -5,7 +5,7 @@ import org.scalatestplus.play.guice.GuiceOneAppPerSuite import play.api.Application import play.api.http.HeaderNames.USER_AGENT import play.api.inject.guice.GuiceApplicationBuilder -import play.api.libs.json.Json +import play.api.libs.json.{JsValue, Json} import play.api.test.Helpers._ import play.api.test.{FakeHeaders, FakeRequest} import uk.gov.hmrc.http.HeaderNames.xSessionId @@ -18,7 +18,7 @@ class PrepareUploadControllerISpec extends UnitSpec with GuiceOneAppPerSuite wit ) .build() - "PrepareUploadController" should { + "PrepareUploadController prepareUploadV1" should { val postBodyJson = Json.parse(""" |{ | "callbackUrl": "https://some-url/callback", @@ -28,11 +28,33 @@ class PrepareUploadControllerISpec extends UnitSpec with GuiceOneAppPerSuite wit |} """.stripMargin) + behave like prepareUploadTests(postBodyJson, "/upscan/initiate") + } + + "PrepareUploadController prepareUploadV2" should { + val postBodyJson = Json.parse(""" + |{ + | "callbackUrl": "https://some-url/callback", + | "successRedirect": "https://some-url/success", + | "errorRedirect": "https://some-url/error", + | "minimumFileSize" : 0, + | "maximumFileSize" : 1024, + | "expectedMimeType": "application/xml" + |} + """.stripMargin) + + behave like prepareUploadTests(postBodyJson, "/upscan/v2/initiate") + } + + private def prepareUploadTests( // scalastyle:ignore + postBodyJson: JsValue, + uri: String): Unit = { + "include x-amz-meta-consuming-service in the response" in { Given("a request containing a USER_AGENT header contained in the configuration of allowedUserAgents") val initiateRequest = FakeRequest( POST, - "/upscan/initiate", + uri, FakeHeaders(Seq((USER_AGENT, "PrepareUploadControllerISpec"), (xSessionId, "some-session-id"))), postBodyJson) @@ -53,7 +75,7 @@ class PrepareUploadControllerISpec extends UnitSpec with GuiceOneAppPerSuite wit Given("a request containing a x-session-id header") val initiateRequest = FakeRequest( POST, - "/upscan/initiate", + uri, FakeHeaders(Seq((USER_AGENT, "PrepareUploadControllerISpec"), (xSessionId, "some-session-id"))), postBodyJson) @@ -72,11 +94,8 @@ class PrepareUploadControllerISpec extends UnitSpec with GuiceOneAppPerSuite wit "set a default x-amz-meta-session-id in the response if no session id passed in" in { Given("a request containing a x-session-id header") - val initiateRequest = FakeRequest( - POST, - "/upscan/initiate", - FakeHeaders(Seq((USER_AGENT, "PrepareUploadControllerISpec"))), - postBodyJson) + val initiateRequest = + FakeRequest(POST, uri, FakeHeaders(Seq((USER_AGENT, "PrepareUploadControllerISpec"))), postBodyJson) When("a request is posted to the /initiate endpoint") val initiateResponse = route(app, initiateRequest).get @@ -94,7 +113,7 @@ class PrepareUploadControllerISpec extends UnitSpec with GuiceOneAppPerSuite wit "reject requests which do not include a valid USER_AGENT header" in { Given("a request containing a USER_AGENT header not contained in the configuration of allowedUserAgents") val initiateRequest = - FakeRequest(POST, "/upscan/initiate", FakeHeaders(Seq((USER_AGENT, "SomeInvalidUserAgent"))), postBodyJson) + FakeRequest(POST, uri, FakeHeaders(Seq((USER_AGENT, "SomeInvalidUserAgent"))), postBodyJson) When("a request is posted to the /initiate endpoint") val initiateResponse = route(app, initiateRequest).get @@ -106,7 +125,7 @@ class PrepareUploadControllerISpec extends UnitSpec with GuiceOneAppPerSuite wit "reject requests which do not include a USER_AGENT header" in { Given("a request not containing a USER_AGENT header") val initiateRequest = - FakeRequest(POST, "/upscan/initiate", FakeHeaders(Seq((xSessionId, "some-session-id"))), postBodyJson) + FakeRequest(POST, uri, FakeHeaders(Seq((xSessionId, "some-session-id"))), postBodyJson) When("a request is posted to the /initiate endpoint") val initiateResponse = route(app, initiateRequest).get diff --git a/test/controllers/PrepareUploadControllerSpec.scala b/test/controllers/PrepareUploadControllerSpec.scala index 2f8d87b..b122009 100644 --- a/test/controllers/PrepareUploadControllerSpec.scala +++ b/test/controllers/PrepareUploadControllerSpec.scala @@ -12,7 +12,8 @@ import org.mockito.invocation.InvocationOnMock import org.mockito.stubbing.Answer import org.scalatest.mockito.MockitoSugar import org.scalatest.{GivenWhenThen, Matchers} -import play.api.libs.json.{JsValue, Json} +import play.api.libs.json.{JsObject, JsValue, Json} +import play.api.mvc.Action import play.api.mvc.Results.Ok import play.api.test.Helpers.contentAsString import play.api.test.{FakeRequest, Helpers} @@ -34,7 +35,28 @@ class PrepareUploadControllerSpec extends UnitSpec with Matchers with GivenWhenT private val clock: Clock = Clock.systemDefaultZone() - "PrepareUploadController" should { + "PaymentController prepareUploadV1" should { + + behave like prapareUploadTests(_.prepareUploadV1()) + } + + "PaymentController prepareUploadV2" should { + + val extraRequestFields = Json + .obj("successRedirect" -> "https://www.example.com/nextpage", "errorRedirect" -> "https://www.example.com/error") + + val extraResponseFields = Json.obj( + "success_action_redirect" -> "https://www.example.com/nextpage", + "error_action_redirect" -> "https://www.example.com/error") + + behave like prapareUploadTests(_.prepareUploadV2(), extraRequestFields, extraResponseFields) + } + + private def prapareUploadTests( // scalastyle:ignore + prepareUploadAction: PrepareUploadController => Action[JsValue], + extraRequestFields: JsObject = JsObject(Seq()), + extraResponseFields: JsObject = JsObject(Seq())) { + val config = mock[ServiceConfiguration] Mockito.when(config.allowedUserAgents).thenReturn(List("VALID-AGENT")) Mockito.when(config.allowedCallbackProtocols).thenReturn(List("https")) @@ -54,11 +76,11 @@ class PrepareUploadControllerSpec extends UnitSpec with Matchers with GivenWhenT "id" -> "1", "callbackUrl" -> "https://www.example.com", "minimumFileSize" -> 0, - "maximumFileSize" -> 1024)) + "maximumFileSize" -> 1024) ++ extraRequestFields) When("upload initiation has been requested") - val result = controller.prepareUploadV1()(request) + val result = prepareUploadAction(controller)(request) Then("service returns valid response with reference and template of upload form") @@ -68,12 +90,11 @@ class PrepareUploadControllerSpec extends UnitSpec with Matchers with GivenWhenT "reference" -> "TEST", "uploadRequest" -> Json.obj( "href" -> "https://www.example.com", - "fields" -> Json.obj( + "fields" -> (Json.obj( "minFileSize" -> "0", "maxFileSize" -> "1024", "sessionId" -> "some-session-id", - "requestId" -> "some-request-id" - ) + "requestId" -> "some-request-id") ++ extraResponseFields) ) ) } @@ -94,11 +115,11 @@ class PrepareUploadControllerSpec extends UnitSpec with Matchers with GivenWhenT "callbackUrl" -> "https://www.example.com", "successRedirect" -> "https://www.example.com/nextpage", "minimumFileSize" -> 0, - "maximumFileSize" -> 1024)) + "maximumFileSize" -> 1024) ++ extraRequestFields) When("upload initiation has been requested") - val result = controller.prepareUploadV1()(request) + val result = prepareUploadAction(controller)(request) Then("service returns valid response with reference and template of upload form") @@ -108,13 +129,13 @@ class PrepareUploadControllerSpec extends UnitSpec with Matchers with GivenWhenT "reference" -> "TEST", "uploadRequest" -> Json.obj( "href" -> "https://www.example.com", - "fields" -> Json.obj( + "fields" -> (Json.obj( "minFileSize" -> "0", "maxFileSize" -> "1024", "sessionId" -> "some-session-id", "requestId" -> "some-request-id", "success_action_redirect" -> "https://www.example.com/nextpage" - ) + ) ++ extraResponseFields) ) ) } @@ -129,11 +150,11 @@ class PrepareUploadControllerSpec extends UnitSpec with Matchers with GivenWhenT ("User-Agent", "VALID-AGENT"), ("x-session-id", "some-session-id"), ("x-request-id", "some-request-id")) - .withBody(Json.obj("callbackUrl" -> "https://www.example.com")) + .withBody(Json.obj("callbackUrl" -> "https://www.example.com") ++ extraRequestFields) When("upload initiation has been requested") - val result = controller.prepareUploadV1()(request) + val result = prepareUploadAction(controller)(request) Then("service returns valid response with reference and template of upload form") @@ -143,10 +164,8 @@ class PrepareUploadControllerSpec extends UnitSpec with Matchers with GivenWhenT "reference" -> "TEST", "uploadRequest" -> Json.obj( "href" -> "https://www.example.com", - "fields" -> Json.obj( - "sessionId" -> "some-session-id", - "requestId" -> "some-request-id" - ) + "fields" -> (Json + .obj("sessionId" -> "some-session-id", "requestId" -> "some-request-id") ++ extraResponseFields) ) ) } @@ -158,11 +177,11 @@ class PrepareUploadControllerSpec extends UnitSpec with Matchers with GivenWhenT val request: FakeRequest[JsValue] = FakeRequest() .withHeaders(("User-Agent", "VALID-AGENT")) - .withBody(Json.obj("callbackUrl" -> "https://www.example.com")) + .withBody(Json.obj("callbackUrl" -> "https://www.example.com") ++ extraRequestFields) When("upload initiation has been requested") - val result = controller.prepareUploadV1()(request) + val result = prepareUploadAction(controller)(request) Then("service returns valid response with reference and template of upload form") @@ -171,12 +190,10 @@ class PrepareUploadControllerSpec extends UnitSpec with Matchers with GivenWhenT json shouldBe Json.obj( "reference" -> "TEST", "uploadRequest" -> Json.obj( - "href" -> "https://www.example.com", - "fields" -> Json.obj( - "sessionId" -> "n/a", - "requestId" -> "n/a" - ) - )) + "href" -> "https://www.example.com", + "fields" -> (Json.obj("sessionId" -> "n/a", "requestId" -> "n/a") ++ extraResponseFields) + ) + ) } "return a bad request error if invalid request - wrong structure" in { @@ -193,7 +210,7 @@ class PrepareUploadControllerSpec extends UnitSpec with Matchers with GivenWhenT When("upload initiation has been requested") - val result = controller.prepareUploadV1()(request) + val result = prepareUploadAction(controller)(request) Then("service returns error response") @@ -211,7 +228,7 @@ class PrepareUploadControllerSpec extends UnitSpec with Matchers with GivenWhenT When("upload initiation has been requested") - val result = controller.prepareUploadV1()(request) + val result = prepareUploadAction(controller)(request) Then("service returns error response") @@ -237,11 +254,11 @@ class PrepareUploadControllerSpec extends UnitSpec with Matchers with GivenWhenT "maximumFileSize" -> 1024, "session-id" -> "some-session-id", "request-id" -> "some-request-id" - )) + ) ++ extraRequestFields) When("upload initiation has been requested") - val result = controller.prepareUploadV1()(request) + val result = prepareUploadAction(controller)(request) Then("service returns error response") @@ -251,12 +268,11 @@ class PrepareUploadControllerSpec extends UnitSpec with Matchers with GivenWhenT "reference" -> "TEST", "uploadRequest" -> Json.obj( "href" -> "https://www.example.com", - "fields" -> Json.obj( + "fields" -> (Json.obj( "minFileSize" -> "0", "maxFileSize" -> "1024", "sessionId" -> "some-session-id", - "requestId" -> "some-request-id" - ) + "requestId" -> "some-request-id") ++ extraResponseFields) ) ) } @@ -278,11 +294,11 @@ class PrepareUploadControllerSpec extends UnitSpec with Matchers with GivenWhenT "maximumFileSize" -> 1024, "session-id" -> "some-session-id", "request-id" -> "some-request-id" - )) + ) ++ extraRequestFields) When("upload initiation has been requested") - val result = controller.prepareUploadV1()(request) + val result = prepareUploadAction(controller)(request) Then("service returns error response") @@ -323,7 +339,7 @@ class PrepareUploadControllerSpec extends UnitSpec with Matchers with GivenWhenT } } - def prepareUploadService: PrepareUploadService = { + private def prepareUploadService: PrepareUploadService = { val service = mock[PrepareUploadService] Mockito.when(service.globalFileSizeLimit).thenReturn(1024) Mockito @@ -342,7 +358,8 @@ class PrepareUploadControllerSpec extends UnitSpec with Matchers with GivenWhenT settings.maximumFileSize.map(s => Map("maxFileSize" -> s.toString).head) ++ Map("sessionId" -> sessionId) ++ Map("requestId" -> requestId) ++ - settings.successRedirect.map(url => Map("success_action_redirect" -> url)).getOrElse(Map.empty) + settings.successRedirect.map(url => Map("success_action_redirect" -> url)).getOrElse(Map.empty) ++ + settings.errorRedirect.map(url => Map("error_action_redirect" -> url)).getOrElse(Map.empty) ) ) } From b00d3f4eb3a4b10935beb9b5a031aeab84fecdc4 Mon Sep 17 00:00:00 2001 From: chotai Date: Fri, 31 May 2019 00:39:47 +0100 Subject: [PATCH 27/96] BDOG-188 Update README.md with documentation for v2/upscan/initiate --- README.md | 123 ++++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 111 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 36a0ec2..e290877 100644 --- a/README.md +++ b/README.md @@ -75,33 +75,126 @@ Configuration of these values is here (https://github.com/hmrc/upscan-infrastruc ### Requesting a URL to upload to -Assuming the consuming service is whitelisted, it makes a POST request to the `/upscan/initiate` endpoint. This request includes details about the expected upload, specifically the callback URL and optional constraints on size. - -The callback will be made from inside the MDTP environment. Hence, the callback URL should comprise the MDTP internal callback address and not the public domain address. +Assuming the consuming service is whitelisted, it makes a POST request to `/upscan/initiate` or `upscan/v2/initiate`. The service must provide a callbackUrl for asynchronous notification of the outcome of a upload. The callback will be made from inside the MDTP environment. Hence, the callback URL should comprise the MDTP internal callback address and not the public domain address. `upscan/v2/initiate` additionally requires a successRedirect and errorRedirect url. See next section for specifics. **Note:** `callbackUrl` must use the `https` protocol. (Although this rule is relaxed when testing locally with [upscan-stub](https://github.com/hmrc/upscan-stub) rather than [upscan-initiate](https://github.com/hmrc/upscan-initiate). In this stubbed scenario a `callbackUrl` referring to localhost may still specific `http` as the protocol.) -Here is an example of the request body: +Session-ID / Request-ID headers will be used to link the file with user's journey. + +*Note:* If you are using `[http-verbs](https://github.com/hmrc/http-verbs)` to call Upscan, all the headers will be set automatically +(See: [HttpVerb.scala](https://github.com/hmrc/http-verbs/blob/2807dc65f64009bd7ce1f14b38b356e06dd23512/src/main/scala/uk/gov/hmrc/http/HttpVerb.scala#L53)) + +The service replies with a pre-filled template for the upload of the file. +The JSON response also contains a globally unique file reference of the upload. This reference can be used by the Upscan service team to view the progress and result of the journey through the different Upscan components. The consuming service can use this reference to correlate the upload request with a successfully uploaded file. + + +### POST upscan/v2/initiate + +The S3 rest api is able to redirect on successful uploads. It does not support redirects on errors however. Version 2 of `upscan/initiate` returns the url [upscan-upload-proxy](https://github.com/hmrc/upscan-upload-proxy) that sits in front of S3 to gracefully handle S3 errors. + +Example `upscan/v2/initiate` request: ```json { "callbackUrl": "https://myservice.com/callback", + "successRedirect": "https://myservice.com/nextPage", + "errorRedirect": "https://myservice.com/errorPage", "minimumFileSize" : 0, "maximumFileSize" : 1024 } ``` -Meaning of parameters: +#### HTTP Headers: + +| Header name|Description|Required| +|--------------|-----------|--------| +| User-Agent | Identifier of the service that calls upscan | yes | +| X-Session-ID | Identifier of the user's session | no | +| X-Request-ID | Identifier of the user's request | no | + +#### Body parameters: | Parameter name|Description|Required| |--------------|-----------|--------| |callbackUrl |Url that will be called to report the outcome of file checking and upload, including retrieval details if successful. Notification format is detailed further down in this file. Must be https.| yes| +|successRedirect|Url to redirect to after file has been successfully uploaded.|yes| +|errorRedirect|Url to redirect to if error encountered during upload.|yes| |minimumFileSize|Minimum file size (in Bytes). Default is 0.|no| |maximumFileSize|Maximum file size (in Bytes). Cannot be greater than 100MB. Default is 100MB.|no| -|successRedirect|Url to redirect to after file has been successfully uploaded.|no| -The request has to include the following HTTP headers: + +Example response + +```json +{ + "reference": "11370e18-6e24-453e-b45a-76d3e32ea33d", + "uploadRequest": { + "href": "https://xxxx/upscan-upload-proxy/bucketName", + "fields": { + "Content-Type": "application/xml", + "acl": "private", + "key": "xxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx", + "policy": "xxxxxxxx==", + "x-amz-algorithm": "AWS4-HMAC-SHA256", + "x-amz-credential": "ASIAxxxxxxxxx/20180202/eu-west-2/s3/aws4_request", + "x-amz-date": "yyyyMMddThhmmssZ", + "x-amz-meta-callback-url": "https://myservice.com/callback", + "x-amz-signature": "xxxx", + "success_action_redirect": "https://myservice.com/nextPage", + "error_action_redirect": "https://myservice.com/errorPage" + } + } +} +``` + +*Note:* We recommend that the response fields are not hardcoded as these are subject to change + +The `href` in the response is for a proxy that sits in front of S3 to handle error responses. + +S3 will return errors in `application/xml` in the following format: + +```xml + + + NoSuchKey + The resource you requested does not exist + /mybucket/myfoto.jpg + 4442587FB7D0A2F9 + +``` + +Example [upscan-upload-proxy](https://github.com/hmrc/upscan-upload-proxy) error redirect response: + +Redirect Response: + +``` +HTTP Response Code: 303 +Header ("Location" -> "https://myservice.com/errorPage?errorCode=NoSuchKey&errorMessage=The resource you requested does not exist&errorResource=/mybucket/myfoto.jpg$errorRequestId=4442587FB7D0A2F9") +``` + +Json Error Response (can not redirect): + +```json +{"message":"Bad request"} +``` + + +[[Back to the top]](#top) + +### POST upscan/initiate + +Example request: + +```json +{ + "callbackUrl": "https://myservice.com/callback", + "minimumFileSize" : 0, + "maximumFileSize" : 1024 +} +``` + +#### HTTP Headers: | Header name|Description|Required| |--------------|-----------|--------| @@ -109,13 +202,18 @@ The request has to include the following HTTP headers: | X-Session-ID | Identifier of the user's session | no | | X-Request-ID | Identifier of the user's request | no | -Session-ID / Request-ID headers will be used to link the file with user's journey. +#### Body parameters: + +| Parameter name|Description|Required| +|--------------|-----------|--------| +|callbackUrl |Url that will be called to report the outcome of file checking and upload, including retrieval details if successful. Notification format is detailed further down in this file. Must be https.| yes| +|minimumFileSize|Minimum file size (in Bytes). Default is 0.|no| +|maximumFileSize|Maximum file size (in Bytes). Cannot be greater than 100MB. Default is 100MB.|no| +|successRedirect|Url to redirect to after file has been successfully uploaded.|no| -*Note:* If you are using `[http-verbs](https://github.com/hmrc/http-verbs)` to call Upscan, all the headers will be set automatically -(See: [HttpVerb.scala](https://github.com/hmrc/http-verbs/blob/2807dc65f64009bd7ce1f14b38b356e06dd23512/src/main/scala/uk/gov/hmrc/http/HttpVerb.scala#L53)) -The service replies with a pre-filled template for the upload of the file (described below). -The JSON response also contains a globally unique file reference of the upload. This reference can be used by the Upscan service team to view the progress and result of the journey through the different Upscan components. The consuming service can use this reference to correlate the upload request with a successfully uploaded file. + +Example Response: ```json { @@ -136,6 +234,7 @@ The JSON response also contains a globally unique file reference of the upload. } } ``` + [[Back to the top]](#top) ### The file upload From 1b9dde9d65c7337898895ee9fd34279608907545 Mon Sep 17 00:00:00 2001 From: chotai Date: Wed, 5 Jun 2019 23:07:29 +0100 Subject: [PATCH 28/96] BDOG-188 Return upscan-upload-proxy url in v2/initiate href response --- app/config/ServiceConfiguration.scala | 31 +++++++----- .../model/UploadFormGenerator.scala | 2 - app/connectors/s3/S3UploadFormGenerator.scala | 2 - app/controllers/PrepareUploadController.scala | 22 +++++---- app/controllers/model/PrepareUpload.scala | 2 +- .../model/PrepareUploadRequestV1.scala | 3 +- .../model/PrepareUploadRequestV2.scala | 3 +- app/services/PrepareUploadService.scala | 2 +- app/services/model/UploadSettings.scala | 1 + conf/application.conf | 2 + .../PrepareUploadControllerISpec.scala | 35 ++++++++++++-- test/model/PrepareUploadServiceSpec.scala | 47 +++++++++++++------ 12 files changed, 104 insertions(+), 48 deletions(-) diff --git a/app/config/ServiceConfiguration.scala b/app/config/ServiceConfiguration.scala index ce4fbb1..05e25c8 100644 --- a/app/config/ServiceConfiguration.scala +++ b/app/config/ServiceConfiguration.scala @@ -9,6 +9,7 @@ import play.api.Configuration trait ServiceConfiguration { def region: String + def uploadProxyUrl: String def inboundBucketName: String def sessionToken: Option[String] def accessKeyId: String @@ -22,23 +23,25 @@ trait ServiceConfiguration { class PlayBasedServiceConfiguration @Inject()(configuration: Configuration) extends ServiceConfiguration { - override def region = getRequired(configuration.getString(_), "aws.s3.region") + override def region: String = getRequired(configuration.getString(_), "aws.s3.region") - override def inboundBucketName = getRequired(configuration.getString(_), "aws.s3.bucket.inbound") + override def uploadProxyUrl: String = getRequired(configuration.getString(_), "uploadProxy.url") - override def fileExpirationPeriod = + override def inboundBucketName: String = getRequired(configuration.getString(_), "aws.s3.bucket.inbound") + + override def fileExpirationPeriod: Duration = Duration.ofMillis(getRequired(configuration.getMilliseconds, "aws.s3.upload.link.validity.duration")) - override def accessKeyId = getRequired(configuration.getString(_), "aws.accessKeyId") + override def accessKeyId: String = getRequired(configuration.getString(_), "aws.accessKeyId") - override def secretAccessKey = getRequired(configuration.getString(_), "aws.secretAccessKey") + override def secretAccessKey: String = getRequired(configuration.getString(_), "aws.secretAccessKey") - def getRequired[T](function: String => Option[T], key: String) = + def getRequired[T](function: String => Option[T], key: String): T = function(key).getOrElse(throw new IllegalStateException(s"$key missing")) - override def sessionToken = configuration.getString("aws.sessionToken") + override def sessionToken: Option[String] = configuration.getString("aws.sessionToken") - override def globalFileSizeLimit = getRequired(configuration.getInt, "global.file.size.limit") + override def globalFileSizeLimit: Int = getRequired(configuration.getInt, "global.file.size.limit") override def allowedCallbackProtocols: Seq[String] = configuration @@ -50,10 +53,12 @@ class PlayBasedServiceConfiguration @Inject()(configuration: Configuration) exte .getOrElse(Nil) override def allowedUserAgents: Seq[String] = - configuration.getString("userAgentFilter.allowedUserAgents").map {_ - .split(",") - .toSeq - .filter(isNotBlank) - }.getOrElse(Nil) + configuration + .getString("userAgentFilter.allowedUserAgents") + .map { + _.split(",").toSeq + .filter(isNotBlank) + } + .getOrElse(Nil) } diff --git a/app/connectors/model/UploadFormGenerator.scala b/app/connectors/model/UploadFormGenerator.scala index f4e2b34..2663a4f 100644 --- a/app/connectors/model/UploadFormGenerator.scala +++ b/app/connectors/model/UploadFormGenerator.scala @@ -1,7 +1,5 @@ package connectors.model trait UploadFormGenerator { - def buildEndpoint(bucketName: String): String - def generateFormFields(uploadParameters: UploadParameters): Map[String, String] } diff --git a/app/connectors/s3/S3UploadFormGenerator.scala b/app/connectors/s3/S3UploadFormGenerator.scala index f68bc60..293a370 100644 --- a/app/connectors/s3/S3UploadFormGenerator.scala +++ b/app/connectors/s3/S3UploadFormGenerator.scala @@ -14,8 +14,6 @@ class S3UploadFormGenerator( policySigner: PolicySigner = PolicySigner) extends UploadFormGenerator { - def buildEndpoint(bucketName: String) = s"https://$bucketName.s3.amazonaws.com" - def generateFormFields(uploadParameters: UploadParameters): Map[String, String] = { val timestamp = currentTime() val formattedSigningDate = awsDate(timestamp) diff --git a/app/controllers/PrepareUploadController.scala b/app/controllers/PrepareUploadController.scala index 7078b7b..b48c716 100644 --- a/app/controllers/PrepareUploadController.scala +++ b/app/controllers/PrepareUploadController.scala @@ -30,7 +30,18 @@ class PrepareUploadController @Inject()( implicit val prepareUploadRequestV2Reads: Reads[PrepareUploadRequestV2] = PrepareUploadRequestV2.reads(prepareUploadService.globalFileSizeLimit) - private def prepareUpload[T <: PrepareUpload]()(implicit reads: Reads[T], manifest: Manifest[T]): Action[JsValue] = + def prepareUploadV1(): Action[JsValue] = { + val uploadUrl = s"https://${configuration.inboundBucketName}.s3.amazonaws.com" + prepareUpload[PrepareUploadRequestV1](uploadUrl) + } + + def prepareUploadV2(): Action[JsValue] = { + val uploadUrl = s"${configuration.uploadProxyUrl}/v1/uploads/${configuration.inboundBucketName}" + prepareUpload[PrepareUploadRequestV2](uploadUrl) + } + + private def prepareUpload[T <: PrepareUpload]( + uploadUrl: String)(implicit reads: Reads[T], manifest: Manifest[T]): Action[JsValue] = Action.async(parse.json) { implicit request => val receivedAt = Instant.now(clock) @@ -44,7 +55,7 @@ class PrepareUploadController @Inject()( val result: PreparedUploadResponse = prepareUploadService .prepareUpload( - prepareUploadRequest.toUploadSettings, + prepareUploadRequest.toUploadSettings(uploadUrl), consumingService, requestId, sessionId, @@ -56,12 +67,6 @@ class PrepareUploadController @Inject()( } } - def prepareUploadV1(): Action[JsValue] = - prepareUpload[PrepareUploadRequestV1]() - - def prepareUploadV2(): Action[JsValue] = - prepareUpload[PrepareUploadRequestV2]() - private[controllers] def withAllowedCallbackProtocol[A](callbackUrl: String)( block: => Future[Result]): Future[Result] = { @@ -85,6 +90,5 @@ class PrepareUploadController @Inject()( Future.successful(BadRequest(s"Invalid callback url format: [$callbackUrl]. [${e.getMessage}]")) } } - } } diff --git a/app/controllers/model/PrepareUpload.scala b/app/controllers/model/PrepareUpload.scala index 255704b..03b97ab 100644 --- a/app/controllers/model/PrepareUpload.scala +++ b/app/controllers/model/PrepareUpload.scala @@ -3,5 +3,5 @@ import services.model.UploadSettings trait PrepareUpload { def callbackUrl: String - def toUploadSettings: UploadSettings + def toUploadSettings(uploadUrl: String): UploadSettings } diff --git a/app/controllers/model/PrepareUploadRequestV1.scala b/app/controllers/model/PrepareUploadRequestV1.scala index ab343ac..99ffffc 100644 --- a/app/controllers/model/PrepareUploadRequestV1.scala +++ b/app/controllers/model/PrepareUploadRequestV1.scala @@ -13,7 +13,8 @@ case class PrepareUploadRequestV1( successRedirect: Option[String]) extends PrepareUpload { - def toUploadSettings: UploadSettings = UploadSettings( + def toUploadSettings(uploadUrl: String): UploadSettings = UploadSettings( + uploadUrl = uploadUrl, callbackUrl = callbackUrl, minimumFileSize = minimumFileSize, maximumFileSize = maximumFileSize, diff --git a/app/controllers/model/PrepareUploadRequestV2.scala b/app/controllers/model/PrepareUploadRequestV2.scala index 828a906..25af436 100644 --- a/app/controllers/model/PrepareUploadRequestV2.scala +++ b/app/controllers/model/PrepareUploadRequestV2.scala @@ -14,7 +14,8 @@ case class PrepareUploadRequestV2( expectedContentType: Option[String]) extends PrepareUpload { - def toUploadSettings: UploadSettings = UploadSettings( + def toUploadSettings(uploadUrl: String): UploadSettings = UploadSettings( + uploadUrl = uploadUrl, callbackUrl = callbackUrl, minimumFileSize = minimumFileSize, maximumFileSize = maximumFileSize, diff --git a/app/services/PrepareUploadService.scala b/app/services/PrepareUploadService.scala index f936ed1..a1a0bc3 100644 --- a/app/services/PrepareUploadService.scala +++ b/app/services/PrepareUploadService.scala @@ -83,7 +83,7 @@ class PrepareUploadService @Inject()( ) val form = postSigner.generateFormFields(uploadParameters) - val endpoint = postSigner.buildEndpoint(configuration.inboundBucketName) + val endpoint = settings.uploadUrl UploadFormTemplate(endpoint, form) } diff --git a/app/services/model/UploadSettings.scala b/app/services/model/UploadSettings.scala index 089d7a2..ccc2c96 100644 --- a/app/services/model/UploadSettings.scala +++ b/app/services/model/UploadSettings.scala @@ -1,6 +1,7 @@ package services.model case class UploadSettings( + uploadUrl: String, callbackUrl: String, minimumFileSize: Option[Int], maximumFileSize: Option[Int], diff --git a/conf/application.conf b/conf/application.conf index 3a35eef..3e4c889 100644 --- a/conf/application.conf +++ b/conf/application.conf @@ -94,6 +94,8 @@ metrics { enabled = true } +uploadProxy.url = "ENTER UPLOAD PROXY URL" + aws { s3 { region = "eu-west-2" diff --git a/it/controllers/PrepareUploadControllerISpec.scala b/it/controllers/PrepareUploadControllerISpec.scala index 5d65260..208fc0a 100644 --- a/it/controllers/PrepareUploadControllerISpec.scala +++ b/it/controllers/PrepareUploadControllerISpec.scala @@ -14,7 +14,9 @@ import uk.gov.hmrc.play.test.UnitSpec class PrepareUploadControllerISpec extends UnitSpec with GuiceOneAppPerSuite with GivenWhenThen { override implicit lazy val app: Application = new GuiceApplicationBuilder() .configure( - "userAgentFilter.allowedUserAgents" -> "PrepareUploadControllerISpec" + "userAgentFilter.allowedUserAgents" -> "PrepareUploadControllerISpec", + "uploadProxy.url" -> "https://upload-proxy.tax.service.gov.uk", + "aws.s3.bucket.inbound" -> "inbound-bucket" ) .build() @@ -28,7 +30,11 @@ class PrepareUploadControllerISpec extends UnitSpec with GuiceOneAppPerSuite wit |} """.stripMargin) - behave like prepareUploadTests(postBodyJson, "/upscan/initiate") + behave like prepareUploadTests( + postBodyJson = postBodyJson, + uri = "/upscan/initiate", + href = "https://inbound-bucket.s3.amazonaws.com" + ) } "PrepareUploadController prepareUploadV2" should { @@ -43,12 +49,17 @@ class PrepareUploadControllerISpec extends UnitSpec with GuiceOneAppPerSuite wit |} """.stripMargin) - behave like prepareUploadTests(postBodyJson, "/upscan/v2/initiate") + behave like prepareUploadTests( + postBodyJson = postBodyJson, + uri = "/upscan/v2/initiate", + href = "https://upload-proxy.tax.service.gov.uk/v1/uploads/inbound-bucket" + ) } private def prepareUploadTests( // scalastyle:ignore postBodyJson: JsValue, - uri: String): Unit = { + uri: String, + href: String): Unit = { "include x-amz-meta-consuming-service in the response" in { Given("a request containing a USER_AGENT header contained in the configuration of allowedUserAgents") @@ -69,6 +80,12 @@ class PrepareUploadControllerISpec extends UnitSpec with GuiceOneAppPerSuite wit (responseJson \ "uploadRequest" \ "fields" \ "x-amz-meta-consuming-service") .as[String] shouldBe "PrepareUploadControllerISpec" + + And("the href should be the expected url for the upload") + + (responseJson \ "uploadRequest" \ "href") + .as[String] shouldBe href + } "include x-amz-meta-session-id in the response" in { @@ -90,6 +107,11 @@ class PrepareUploadControllerISpec extends UnitSpec with GuiceOneAppPerSuite wit (responseJson \ "uploadRequest" \ "fields" \ "x-amz-meta-session-id") .as[String] shouldBe "some-session-id" + + And("the href should be the expected url for the upload") + + (responseJson \ "uploadRequest" \ "href") + .as[String] shouldBe href } "set a default x-amz-meta-session-id in the response if no session id passed in" in { @@ -108,6 +130,11 @@ class PrepareUploadControllerISpec extends UnitSpec with GuiceOneAppPerSuite wit (responseJson \ "uploadRequest" \ "fields" \ "x-amz-meta-session-id") .as[String] shouldBe "n/a" + + And("the href should be the expected url for the upload") + + (responseJson \ "uploadRequest" \ "href") + .as[String] shouldBe href } "reject requests which do not include a valid USER_AGENT header" in { diff --git a/test/model/PrepareUploadServiceSpec.scala b/test/model/PrepareUploadServiceSpec.scala index 8bcbc4a..e30809f 100644 --- a/test/model/PrepareUploadServiceSpec.scala +++ b/test/model/PrepareUploadServiceSpec.scala @@ -14,12 +14,14 @@ import uk.gov.hmrc.play.test.UnitSpec class PrepareUploadServiceSpec extends UnitSpec with Matchers with GivenWhenThen { - val serviceConfiguration = new ServiceConfiguration { + private val serviceConfiguration = new ServiceConfiguration { override def accessKeyId: String = ??? override def secretAccessKey: String = ??? + override def uploadProxyUrl: String = ??? + override def inboundBucketName: String = "test-bucket" override def sessionToken: Option[String] = ??? @@ -35,15 +37,15 @@ class PrepareUploadServiceSpec extends UnitSpec with Matchers with GivenWhenThen override def allowedUserAgents: List[String] = ??? } - def metricsStub() = new Metrics { + private def metricsStub() = new Metrics { override val defaultRegistry: MetricRegistry = new MetricRegistry override def toJson: String = ??? } - val s3PostSigner = new UploadFormGenerator { - override def generateFormFields(uploadParameters: UploadParameters) = + private val s3PostSigner = new UploadFormGenerator { + override def generateFormFields(uploadParameters: UploadParameters): Map[String, String] = Map( "bucket" -> uploadParameters.bucketName, "key" -> uploadParameters.objectKey, @@ -56,8 +58,6 @@ class PrepareUploadServiceSpec extends UnitSpec with Matchers with GivenWhenThen } ++ uploadParameters.successRedirect.map { "success_redirect_url" -> _ } ++ uploadParameters.errorRedirect.map { "error_redirect_url" -> _ } - - override def buildEndpoint(bucketName: String): String = s"$bucketName.s3" } val receivedAt: Instant = Instant.now() @@ -72,15 +72,18 @@ class PrepareUploadServiceSpec extends UnitSpec with Matchers with GivenWhenThen Given("there are valid upload settings") + val uploadUrl = s"http://upload-proxy.com" val callbackUrl = "http://www.callback.com" val uploadSettings = UploadSettings( + uploadUrl = uploadUrl, callbackUrl = callbackUrl, minimumFileSize = None, maximumFileSize = None, expectedContentType = Some("application/xml"), successRedirect = None, - errorRedirect = None) + errorRedirect = None + ) When("we setup the upload") @@ -89,7 +92,7 @@ class PrepareUploadServiceSpec extends UnitSpec with Matchers with GivenWhenThen Then("proper upload request form definition should be returned") - result.uploadRequest.href shouldBe s"${serviceConfiguration.inboundBucketName}.s3" + result.uploadRequest.href shouldBe uploadUrl result.uploadRequest.fields shouldBe Map( "bucket" -> serviceConfiguration.inboundBucketName, "key" -> result.reference.value, @@ -114,15 +117,18 @@ class PrepareUploadServiceSpec extends UnitSpec with Matchers with GivenWhenThen Given("there are valid upload settings with size limits") + val uploadUrl = s"http://upload-proxy.com" val callbackUrl = "http://www.callback.com" val uploadSettings = UploadSettings( + uploadUrl = uploadUrl, callbackUrl = callbackUrl, minimumFileSize = Some(100), maximumFileSize = Some(200), expectedContentType = None, successRedirect = None, - errorRedirect = None) + errorRedirect = None + ) When("we setup the upload") @@ -141,15 +147,18 @@ class PrepareUploadServiceSpec extends UnitSpec with Matchers with GivenWhenThen val metrics = metricsStub() + val uploadUrl = s"http://upload-proxy.com" val callbackUrl = "http://www.callback.com" val uploadSettings = UploadSettings( + uploadUrl = uploadUrl, callbackUrl = callbackUrl, minimumFileSize = Some(-1), maximumFileSize = Some(1024), expectedContentType = None, successRedirect = None, - errorRedirect = None) + errorRedirect = None + ) When("we setup the upload") Then("an exception should be thrown") @@ -168,15 +177,18 @@ class PrepareUploadServiceSpec extends UnitSpec with Matchers with GivenWhenThen val metrics = metricsStub() + val uploadUrl = s"http://upload-proxy.com" val callbackUrl = "http://www.callback.com" val uploadSettings = UploadSettings( + uploadUrl = uploadUrl, callbackUrl = callbackUrl, minimumFileSize = Some(0), maximumFileSize = Some(1025), expectedContentType = None, successRedirect = None, - errorRedirect = None) + errorRedirect = None + ) When("we setup the upload") Then("an exception should be thrown") @@ -194,15 +206,18 @@ class PrepareUploadServiceSpec extends UnitSpec with Matchers with GivenWhenThen val metrics = metricsStub() + val uploadUrl = s"http://upload-proxy.com" val callbackUrl = "http://www.callback.com" val uploadSettings = UploadSettings( + uploadUrl = uploadUrl, callbackUrl = callbackUrl, minimumFileSize = Some(1024), maximumFileSize = Some(0), expectedContentType = None, successRedirect = None, - errorRedirect = None) + errorRedirect = None + ) When("we setup the upload") Then("an exception should be thrown") @@ -220,9 +235,11 @@ class PrepareUploadServiceSpec extends UnitSpec with Matchers with GivenWhenThen Given("there are valid upload settings") + val uploadUrl = s"http://upload-proxy.com" val callbackUrl = "http://www.callback.com" val uploadSettings = UploadSettings( + uploadUrl = uploadUrl, callbackUrl = callbackUrl, minimumFileSize = None, maximumFileSize = None, @@ -238,7 +255,7 @@ class PrepareUploadServiceSpec extends UnitSpec with Matchers with GivenWhenThen Then("proper upload request form definition should be returned") - result.uploadRequest.href shouldBe s"${serviceConfiguration.inboundBucketName}.s3" + result.uploadRequest.href shouldBe uploadUrl result.uploadRequest.fields shouldBe Map( "bucket" -> serviceConfiguration.inboundBucketName, "key" -> result.reference.value, @@ -264,9 +281,11 @@ class PrepareUploadServiceSpec extends UnitSpec with Matchers with GivenWhenThen Given("there are valid upload settings") + val uploadUrl = s"http://upload-proxy.com" val callbackUrl = "http://www.callback.com" val uploadSettings = UploadSettings( + uploadUrl = uploadUrl, callbackUrl = callbackUrl, minimumFileSize = None, maximumFileSize = None, @@ -282,7 +301,7 @@ class PrepareUploadServiceSpec extends UnitSpec with Matchers with GivenWhenThen Then("proper upload request form definition should be returned") - result.uploadRequest.href shouldBe s"${serviceConfiguration.inboundBucketName}.s3" + result.uploadRequest.href shouldBe uploadUrl result.uploadRequest.fields shouldBe Map( "bucket" -> serviceConfiguration.inboundBucketName, "key" -> result.reference.value, From 90e064e3dbc93db0d76957841e26299476282941 Mon Sep 17 00:00:00 2001 From: chotai Date: Fri, 21 Jun 2019 15:36:55 +0100 Subject: [PATCH 29/96] BDOG-188 Update readme with important integration pointers --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index e290877..218ec5b 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,11 @@ This service is not for transfer of files from one HMRC service to another. See [![Build Status](https://travis-ci.org/hmrc/upscan-initiate.svg)](https://travis-ci.org/hmrc/upscan-initiate) [ ![Download](https://api.bintray.com/packages/hmrc/releases/upscan-initiate/images/download.svg) ](https://bintray.com/hmrc/releases/upscan-initiate/_latestVersion) +# TLDR + +- *We strongly advise against hardcoding the "fields" in the response of `initiate` and `v2/initiate`. These are subject to change.* +- *The file must be the last field in the actual upload request.* +- *You must use multipart encoding (multipart/form-data) NOT application/x-www-form-urlencoded. The error message returned by AWS is obscure when the wrong content type is used.* # Upscan user manual From 4fbfceec53297afaf42aedc92aeea96f59c80c82 Mon Sep 17 00:00:00 2001 From: Peter Perhac Date: Wed, 10 Jul 2019 09:32:00 +0100 Subject: [PATCH 30/96] cleanup warnings, use named arguments, run scalafmt --- app/config/ServiceConfiguration.scala | 26 +++++++------------ app/connectors/model/UploadParameters.scala | 1 + app/connectors/s3/S3UploadFormGenerator.scala | 2 +- app/controllers/PrepareUploadController.scala | 20 +++++++------- app/controllers/model/PrepareUpload.scala | 1 + .../model/PrepareUploadRequestV1.scala | 5 ++-- .../model/PrepareUploadRequestV2.scala | 1 + app/controllers/model/Reference.scala | 17 ++++++++---- .../model/UploadFormTemplate.scala | 1 + app/services/PrepareUploadService.scala | 24 ++++++++++------- app/utils/UserAgentFilter.scala | 21 +++++++-------- .../s3/S3UploadFormGeneratorSpec.scala | 2 +- test/utils/UserAgentFilterSpec.scala | 4 +-- 13 files changed, 66 insertions(+), 59 deletions(-) diff --git a/app/config/ServiceConfiguration.scala b/app/config/ServiceConfiguration.scala index 05e25c8..1ce47f8 100644 --- a/app/config/ServiceConfiguration.scala +++ b/app/config/ServiceConfiguration.scala @@ -2,6 +2,7 @@ package config import java.time.Duration +import com.typesafe.config.ConfigException import javax.inject.Inject import org.apache.commons.lang3.StringUtils.isNotBlank import play.api.Configuration @@ -36,29 +37,20 @@ class PlayBasedServiceConfiguration @Inject()(configuration: Configuration) exte override def secretAccessKey: String = getRequired(configuration.getString(_), "aws.secretAccessKey") - def getRequired[T](function: String => Option[T], key: String): T = - function(key).getOrElse(throw new IllegalStateException(s"$key missing")) - override def sessionToken: Option[String] = configuration.getString("aws.sessionToken") override def globalFileSizeLimit: Int = getRequired(configuration.getInt, "global.file.size.limit") override def allowedCallbackProtocols: Seq[String] = - configuration - .getString("callbackValidation.allowedProtocols") - .map { - _.split(",").toSeq - .filter(isNotBlank) - } - .getOrElse(Nil) + commaSeparatedList(configuration.getString("callbackValidation.allowedProtocols")) override def allowedUserAgents: Seq[String] = - configuration - .getString("userAgentFilter.allowedUserAgents") - .map { - _.split(",").toSeq - .filter(isNotBlank) - } - .getOrElse(Nil) + commaSeparatedList(configuration.getString("userAgentFilter.allowedUserAgents")) + + private def getRequired[T](read: String => Option[T], path: String): T = + read(path).getOrElse(throw new ConfigException.Missing(path)) + + private def commaSeparatedList(maybeString: Option[String]): List[String] = + maybeString.fold(List.empty[String])(_.split(",").toList.filter(isNotBlank)) } diff --git a/app/connectors/model/UploadParameters.scala b/app/connectors/model/UploadParameters.scala index af261a8..63ca018 100644 --- a/app/connectors/model/UploadParameters.scala +++ b/app/connectors/model/UploadParameters.scala @@ -1,4 +1,5 @@ package connectors.model + import java.time.Instant case class UploadParameters( diff --git a/app/connectors/s3/S3UploadFormGenerator.scala b/app/connectors/s3/S3UploadFormGenerator.scala index 293a370..0c53d7a 100644 --- a/app/connectors/s3/S3UploadFormGenerator.scala +++ b/app/connectors/s3/S3UploadFormGenerator.scala @@ -60,7 +60,7 @@ class S3UploadFormGenerator( "x-amz-signature" -> policySignature, "acl" -> uploadParameters.acl, "key" -> uploadParameters.objectKey, - "x-amz-meta-original-filename" -> "${filename}", + "x-amz-meta-original-filename" -> s"$${filename}", "x-amz-meta-upscan-initiate-response" -> currentTime().toString ) diff --git a/app/controllers/PrepareUploadController.scala b/app/controllers/PrepareUploadController.scala index b48c716..14c13ff 100644 --- a/app/controllers/PrepareUploadController.scala +++ b/app/controllers/PrepareUploadController.scala @@ -7,7 +7,7 @@ import config.ServiceConfiguration import controllers.model.{PrepareUpload, PrepareUploadRequestV1, PrepareUploadRequestV2, PreparedUploadResponse} import javax.inject.{Inject, Singleton} import play.api.Logger -import play.api.libs.json.{Json, _} +import play.api.libs.json._ import play.api.mvc.{Action, Result} import services.PrepareUploadService import uk.gov.hmrc.play.bootstrap.controller.BaseController @@ -55,11 +55,12 @@ class PrepareUploadController @Inject()( val result: PreparedUploadResponse = prepareUploadService .prepareUpload( - prepareUploadRequest.toUploadSettings(uploadUrl), - consumingService, - requestId, - sessionId, - receivedAt) + settings = prepareUploadRequest.toUploadSettings(uploadUrl), + consumingService = consumingService, + requestId = requestId, + sessionId = sessionId, + receivedAt = receivedAt + ) Future.successful(Ok(Json.toJson(result)(PreparedUploadResponse.writes))) } @@ -78,17 +79,16 @@ class PrepareUploadController @Inject()( isAllowedCallbackProtocol match { case Success(true) => block - case Success(false) => { + case Success(false) => Logger.warn(s"Invalid callback url protocol: [$callbackUrl].") Future.successful(BadRequest( s"Invalid callback url protocol: [$callbackUrl]. Protocol must be in: [${allowedCallbackProtocols.mkString(",")}].")) - } - case Failure(e) => { + + case Failure(e) => Logger.warn(s"Invalid callback url format: [$callbackUrl].") Future.successful(BadRequest(s"Invalid callback url format: [$callbackUrl]. [${e.getMessage}]")) - } } } } diff --git a/app/controllers/model/PrepareUpload.scala b/app/controllers/model/PrepareUpload.scala index 03b97ab..9ba792c 100644 --- a/app/controllers/model/PrepareUpload.scala +++ b/app/controllers/model/PrepareUpload.scala @@ -1,4 +1,5 @@ package controllers.model + import services.model.UploadSettings trait PrepareUpload { diff --git a/app/controllers/model/PrepareUploadRequestV1.scala b/app/controllers/model/PrepareUploadRequestV1.scala index 99ffffc..19a33d1 100644 --- a/app/controllers/model/PrepareUploadRequestV1.scala +++ b/app/controllers/model/PrepareUploadRequestV1.scala @@ -1,8 +1,9 @@ package controllers.model + import play.api.data.validation.ValidationError -import play.api.libs.json.{JsPath, Reads} -import play.api.libs.json.Reads.{max, min} import play.api.libs.functional.syntax._ +import play.api.libs.json.Reads.{max, min} +import play.api.libs.json.{JsPath, Reads} import services.model.UploadSettings case class PrepareUploadRequestV1( diff --git a/app/controllers/model/PrepareUploadRequestV2.scala b/app/controllers/model/PrepareUploadRequestV2.scala index 25af436..74c4041 100644 --- a/app/controllers/model/PrepareUploadRequestV2.scala +++ b/app/controllers/model/PrepareUploadRequestV2.scala @@ -1,4 +1,5 @@ package controllers.model + import play.api.data.validation.ValidationError import play.api.libs.json.{JsPath, Reads} import play.api.libs.json.Reads.{max, min} diff --git a/app/controllers/model/Reference.scala b/app/controllers/model/Reference.scala index 8328ad7..4cc8ddd 100644 --- a/app/controllers/model/Reference.scala +++ b/app/controllers/model/Reference.scala @@ -1,11 +1,18 @@ package controllers.model -import play.api.libs.json.{JsString, JsValue, Writes} -case class Reference(value: String) +import java.util.UUID.randomUUID + +import play.api.libs.functional.syntax._ +import play.api.libs.json.Writes + +case class Reference(value: String) extends AnyVal { + override def toString: String = value.toString +} object Reference { - val writes: Writes[Reference] = new Writes[Reference] { - override def writes(o: Reference): JsValue = JsString(o.value) - } + val writes: Writes[Reference] = Writes.of[String].contramap(_.value) + + def generate(): Reference = Reference(randomUUID().toString) + } diff --git a/app/controllers/model/UploadFormTemplate.scala b/app/controllers/model/UploadFormTemplate.scala index 9cbf3e8..47cdf61 100644 --- a/app/controllers/model/UploadFormTemplate.scala +++ b/app/controllers/model/UploadFormTemplate.scala @@ -1,4 +1,5 @@ package controllers.model + import play.api.libs.functional.syntax._ import play.api.libs.json.{OWrites, __} diff --git a/app/services/PrepareUploadService.scala b/app/services/PrepareUploadService.scala index a1a0bc3..0de7261 100644 --- a/app/services/PrepareUploadService.scala +++ b/app/services/PrepareUploadService.scala @@ -1,7 +1,6 @@ package services import java.time.Instant -import java.util.UUID import com.kenshoo.play.metrics.Metrics import config.ServiceConfiguration @@ -24,19 +23,26 @@ class PrepareUploadService @Inject()( requestId: String, sessionId: String, receivedAt: Instant): PreparedUploadResponse = { - val reference = generateReference() + val reference = Reference.generate() val expiration = receivedAt.plus(configuration.fileExpirationPeriod) val result = PreparedUploadResponse( reference = reference, - uploadRequest = - generatePost(reference.value, expiration, settings, consumingService, requestId, sessionId, receivedAt)) + uploadRequest = generatePost( + key = reference, + expiration = expiration, + settings = settings, + consumingService = consumingService, + requestId = requestId, + sessionId = sessionId, + receivedAt = receivedAt) + ) try { - MDC.put("file-reference", reference.value) + MDC.put("file-reference", reference.toString) Logger.info( - s"Generated file-reference: [${reference.value}], for settings: [$settings], with expiration at: [$expiration].") + s"Generated file-reference: [$reference], for settings: [$settings], with expiration at: [$expiration].") metrics.defaultRegistry.counter("uploadInitiated").inc() @@ -46,10 +52,8 @@ class PrepareUploadService @Inject()( } } - private def generateReference() = Reference(UUID.randomUUID().toString) - private def generatePost( - key: String, + key: Reference, expiration: Instant, settings: UploadSettings, consumingService: String, @@ -67,7 +71,7 @@ class PrepareUploadService @Inject()( val uploadParameters = UploadParameters( expirationDateTime = expiration, bucketName = configuration.inboundBucketName, - objectKey = key, + objectKey = key.value, acl = "private", additionalMetadata = Map( "callback-url" -> settings.callbackUrl, diff --git a/app/utils/UserAgentFilter.scala b/app/utils/UserAgentFilter.scala index 2a4a9e4..dcf6487 100644 --- a/app/utils/UserAgentFilter.scala +++ b/app/utils/UserAgentFilter.scala @@ -10,25 +10,24 @@ import scala.concurrent.Future trait UserAgentFilter { - protected val configuration : ServiceConfiguration + protected val configuration: ServiceConfiguration private val userAgents: Seq[String] = configuration.allowedUserAgents - def onlyAllowedServices[A](block: (Request[A], String) => Future[Result]) - (implicit request: Request[A]): Future[Result] = { - + def onlyAllowedServices[A](block: (Request[A], String) => Future[Result])( + implicit request: Request[A]): Future[Result] = request.headers.get(HeaderNames.USER_AGENT) match { case Some(userAgent) if allowedUserAgent(userAgent) => block(request, userAgent) - case _ => { - Logger.warn(s"Invalid User-Agent: [${request.headers.get(HeaderNames.USER_AGENT)}].") + case userAgent => + Logger.warn(s"Invalid User-Agent: [$userAgent].") - Future.successful(Forbidden("This service is not allowed to use upscan-initiate. " + - "If you need to use this service, please contact Platform Services team.")) - } + Future.successful( + Forbidden( + "This service is not allowed to use upscan-initiate. " + + "If you need to use this service, please contact Platform Services team.")) } - } private def allowedUserAgent(userAgent: String): Boolean = userAgents.contains(userAgent) -} \ No newline at end of file +} diff --git a/test/connectors/s3/S3UploadFormGeneratorSpec.scala b/test/connectors/s3/S3UploadFormGeneratorSpec.scala index dc1d9c7..8720de5 100644 --- a/test/connectors/s3/S3UploadFormGeneratorSpec.scala +++ b/test/connectors/s3/S3UploadFormGeneratorSpec.scala @@ -89,7 +89,7 @@ class S3UploadFormGeneratorSpec extends WordSpec with GivenWhenThen with Matcher result("x-amz-date") shouldBe "19970716T192030Z" result("x-amz-meta-key1") shouldBe "value1" result("x-amz-security-token") shouldBe "session-token" - result("x-amz-meta-original-filename") shouldBe "${filename}" + result("x-amz-meta-original-filename") shouldBe s"$${filename}" } "generate a signed link with a success redirect" in { diff --git a/test/utils/UserAgentFilterSpec.scala b/test/utils/UserAgentFilterSpec.scala index de2f93a..dc46748 100644 --- a/test/utils/UserAgentFilterSpec.scala +++ b/test/utils/UserAgentFilterSpec.scala @@ -18,7 +18,7 @@ class UserAgentFilterSpec extends UnitSpec with Matchers with GivenWhenThen with class UserAgentFilterImpl(override val configuration: ServiceConfiguration) extends UserAgentFilter "UserAgentFilter" should { - val block: (Request[_], String) => Future[Result] = (_,_) => Future.successful(Ok("This is a successful result")) + val block: (Request[_], String) => Future[Result] = (_, _) => Future.successful(Ok("This is a successful result")) implicit val timeout = Timeout(3.seconds) @@ -32,7 +32,7 @@ class UserAgentFilterSpec extends UnitSpec with Matchers with GivenWhenThen with val result = filter.onlyAllowedServices(block)(FakeRequest().withHeaders(("User-Agent", "VALID-AGENT"))) Then("the request should be passed through the filter") - status(result) shouldBe 200 + status(result) shouldBe 200 Helpers.contentAsString(result) shouldBe "This is a successful result" } From 8f4ae78c271f925b0024a0699d74fb399098e651 Mon Sep 17 00:00:00 2001 From: Sunny Chotai Date: Tue, 17 Dec 2019 15:15:22 +0000 Subject: [PATCH 31/96] Remove reference to AJAX from documentation --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 218ec5b..b1ba88f 100644 --- a/README.md +++ b/README.md @@ -255,7 +255,7 @@ In order to upload the file, the following form is sent as the body of a POST re ``` -This POST request can be sent programmatically from backend code, by making an async call using AJAX or the submit of a web form. +This POST request can be sent programmatically from backend code or the submit of a web form. Whichever way the form is sent, remember: From c54134b4188503f49004159a1dfbb71887ef5782 Mon Sep 17 00:00:00 2001 From: cjwebb Date: Wed, 29 Jan 2020 14:39:23 +0000 Subject: [PATCH 32/96] BDOG-611: Update docs --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index b1ba88f..3f220f6 100644 --- a/README.md +++ b/README.md @@ -43,8 +43,6 @@ This service is not for transfer of files from one HMRC service to another. See ## Introduction -Please also read the Upscan documentation in Confluence, this is in the "Platform Services" space. - In this "user manual" the collection of microservices that make up Upscan are discussed, not just `upscan-initiate`. This documentation is here as `upscan-initiate` is the microservice which developers will interact with directly. The Upscan service allows consuming services to orchestrate the uploading of files. Upscan provides @@ -74,6 +72,8 @@ To use Upscan, the consuming service must let Platform Services know : * If the consuming service fails to respond to the callback request (e.g. the consuming service is down, the consuming service answered with an HTTP status code other than 2xx), the callback will be retried up to a maximum of 30 retries. The time interval between the retries is 60 seconds Configuration of these values is here (https://github.com/hmrc/upscan-infrastructure/blob/master/modules/sqs/main.tf) +Please view the [Upscan Service & Flow Overview in Confluence](https://confluence.tools.tax.service.gov.uk/pages/viewpage.action?pageId=101663507) for a more visual representation. + [[Back to the top]](#top) ## Service usage @@ -354,7 +354,7 @@ You or the Upscan service team can use the unique file reference to find out mor ## Whitelisting client services -Any service using Upscan must be whitelisted. Please view the "Upscan & Consuming Services" page of the Upscan documentation in Confluence for the onboarding process. The team are also available on Slack [#team-plat-services](https://hmrcdigital.slack.com/messages/C705QD804). +Any service using Upscan must be whitelisted. Please view the "Upscan & Consuming Services" page of the Upscan documentation in Confluence for the onboarding process. The team are also available on Slack [#team-platops](https://hmrcdigital.slack.com/messages/T04RY81HB). Consuming services must identify themselves in requests via the `User-Agent` header. If the supplied value is not in Upscan's list of allowed services then the `/initiate` call will fail with a `403` error. @@ -495,7 +495,7 @@ These commands will give you an access to SBT shell where you can run the servic [[Back to the top]](#top) #### Slack -* [#team-plat-services](https://hmrcdigital.slack.com/messages/C705QD804/) +* [#team-platops](https://hmrcdigital.slack.com/messages/T04RY81HB/) * [#event-upscan](https://hmrcdigital.slack.com/messages/C8XPL559N) [[Back to the top]](#top) From 624dd79697e6409372da63897c1233b8e2fd5050 Mon Sep 17 00:00:00 2001 From: Nigel Perkins Date: Tue, 18 Feb 2020 15:56:24 +0000 Subject: [PATCH 33/96] BDOG-584: document how file reference is passed back on error redirect --- README.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 3f220f6..8d44f42 100644 --- a/README.md +++ b/README.md @@ -92,7 +92,8 @@ Session-ID / Request-ID headers will be used to link the file with user's journe (See: [HttpVerb.scala](https://github.com/hmrc/http-verbs/blob/2807dc65f64009bd7ce1f14b38b356e06dd23512/src/main/scala/uk/gov/hmrc/http/HttpVerb.scala#L53)) The service replies with a pre-filled template for the upload of the file. -The JSON response also contains a globally unique file reference of the upload. This reference can be used by the Upscan service team to view the progress and result of the journey through the different Upscan components. The consuming service can use this reference to correlate the upload request with a successfully uploaded file. +The JSON response also contains a globally unique file reference for the upload. This reference can be used by the Upscan service team to view the progress and result of the journey through the different Upscan components. The consuming service can use this reference to correlate the subsequent upload result with this upscan initiation. + ### POST upscan/v2/initiate @@ -175,9 +176,12 @@ Redirect Response: ``` HTTP Response Code: 303 -Header ("Location" -> "https://myservice.com/errorPage?errorCode=NoSuchKey&errorMessage=The resource you requested does not exist&errorResource=/mybucket/myfoto.jpg$errorRequestId=4442587FB7D0A2F9") +Header ("Location" -> "https://myservice.com/errorPage?key=11370e18-6e24-453e-b45a-76d3e32ea33d&errorCode=NoSuchKey&errorMessage=The+resource+you+requested+does+not+exist&errorResource=/mybucket/myfoto.jpg&errorRequestId=4442587FB7D0A2F9") ``` + +Note that this Location header comprises the errorRedirect URL supplemented with additional information about the error by way of query parameters. The query parameter named "key" contains the globally unique file reference that was allocated by the initiate request to identify the upload. + Json Error Response (can not redirect): ```json @@ -482,6 +486,7 @@ These commands will give you an access to SBT shell where you can run the servic * [upscan-verify](https://github.com/hmrc/upscan-verify) - service responsible for verifying the health of uploaded files * [upscan-notify](https://github.com/hmrc/upscan-notify) - service responsible for notifying consuming services about the status of uploaded files +* [upscan-upload-proxy](https://github.com/hmrc/upscan-upload-proxy) - service that sits in front of S3 to gracefully handle S3 errors (v2 flow only) * [upscan-infrastructue](https://github.com/hmrc/upscan-infrastructure) - AWS infrastructure provisioning scripts [[Back to the top]](#top) From 74986b0161c0f17b29680edb44959d7e6c1ff26f Mon Sep 17 00:00:00 2001 From: Nigel Perkins Date: Thu, 20 Feb 2020 12:10:35 +0000 Subject: [PATCH 34/96] BDOG-586: Remove User-Agent whitelist --- README.md | 49 ++------ app/controllers/PrepareUploadController.scala | 4 +- app/utils/UserAgentFilter.scala | 36 ++---- conf/application.conf | 76 ++++++------ conf/testOnlyDoNotUseInAppConf.routes | 13 -- .../PrepareUploadControllerISpec.scala | 56 ++++----- .../PrepareUploadControllerSpec.scala | 117 ++++-------------- test/utils/UserAgentFilterSpec.scala | 60 ++++----- 8 files changed, 139 insertions(+), 272 deletions(-) delete mode 100644 conf/testOnlyDoNotUseInAppConf.routes diff --git a/README.md b/README.md index 8d44f42..08a0307 100644 --- a/README.md +++ b/README.md @@ -24,22 +24,20 @@ This service is not for transfer of files from one HMRC service to another. See d. [File processing outcome](#service__poutcome) i. [Success](#service__poutcome__success) ii. [Failure](#service__poutcome__failure) -5. [Whitelisting client services](#whitelist) -6. [Error handling](#error) -7. [Design considerations](#design) +5. [Error handling](#error) +6. [Design considerations](#design) a. [Uploading multiple files](#design__multiple) b. [Security](#design__security) c. [File metadata](#design__metadata) -8. [Architecture of the service](#architecture) -9. [Running and maintenance of the service](#run) +7. [Architecture of the service](#architecture) +8. [Running and maintenance of the service](#run) a. [Running locally](#run__local) -10. [Appendix](#appendix) +9. [Appendix](#appendix) a. [Quick reference figures](#appendix__figures) b. [Related projects, useful links](#appendix__links) i. [Testing](#appendix__links__testing) ii. [Slack](#appendix__links__slack) c. [License](#appendix__license) - ## Introduction @@ -52,8 +50,7 @@ Once the upload URL has been requested, upload and verification of a file are pe [[Back to the top]](#top) ## Onboarding requirements -To use Upscan, the consuming service must let Platform Services know : -- the `User-Agent` request header of the service so it can be [whitelisted](#whitelist) +To use Upscan, the consuming service must let Platform Services know: - how long they would like download URLs for their files to be valid for \[max. 7 days] [[Back to the top]](#top) @@ -65,7 +62,7 @@ To use Upscan, the consuming service must let Platform Services know : * Consuming service passes the form details to the end-user, either via a control on a webpage or to an internal/external service which will upload the file * The end user uploads the file * Upscan service performs all checks on the uploaded file -* If file the file passes all checks, a notification is sent as a POST request to a consuming service. This notification contains the URL to GET the file from +* If the file passes all checks, a notification is sent as a POST request to a consuming service. This notification contains the URL to GET the file from * Consuming service downloads the file using provided URL or passes this URL on to another service which will make use of the file location * After specified time, the file is automatically removed from the remote storage. Upscan does NOT keep files indefinitely * If the file fails a check, a notification is sent to the consuming service containing information on the failed check. The file is unavailable for retrieval @@ -80,11 +77,13 @@ Please view the [Upscan Service & Flow Overview in Confluence](https://confluenc ### Requesting a URL to upload to -Assuming the consuming service is whitelisted, it makes a POST request to `/upscan/initiate` or `upscan/v2/initiate`. The service must provide a callbackUrl for asynchronous notification of the outcome of a upload. The callback will be made from inside the MDTP environment. Hence, the callback URL should comprise the MDTP internal callback address and not the public domain address. `upscan/v2/initiate` additionally requires a successRedirect and errorRedirect url. See next section for specifics. +The consuming service makes a POST request to `/upscan/initiate` or `upscan/v2/initiate`. +This request must contain a `User-Agent` header that can be used to identify the service, but a whitelist of authorised services is no longer cross-checked. +The service must also provide a callbackUrl for asynchronous notification of the outcome of an upload. The callback will be made from inside the MDTP environment. Hence, the callback URL should comprise the MDTP internal callback address and not the public domain address. `upscan/v2/initiate` additionally requires a successRedirect and errorRedirect url. See next section for specifics. **Note:** `callbackUrl` must use the `https` protocol. (Although this rule is relaxed when testing locally with [upscan-stub](https://github.com/hmrc/upscan-stub) rather than [upscan-initiate](https://github.com/hmrc/upscan-initiate). -In this stubbed scenario a `callbackUrl` referring to localhost may still specific `http` as the protocol.) +In this stubbed scenario a `callbackUrl` referring to localhost may still specify `http` as the protocol.) Session-ID / Request-ID headers will be used to link the file with user's journey. @@ -282,7 +281,7 @@ If the POST is successful, the service returns a HTTP 204 response with an empty When a file is successfully uploaded it is processed by [upscan-verify](https://github.com/hmrc/upscan-verify) to check for viruses & that it is of an allowed file type. -If these checks pass, the file is made for available for retrieval & the Upscan service will make a POST request to the URL specified as the 'callbackUrl' by the consuming service with the following body: +If these checks pass, the file is made available for retrieval & the Upscan service will make a POST request to the URL specified as the 'callbackUrl' by the consuming service with the following body: ```json { @@ -356,28 +355,6 @@ You or the Upscan service team can use the unique file reference to find out mor [[Back to the top]](#top) -## Whitelisting client services - -Any service using Upscan must be whitelisted. Please view the "Upscan & Consuming Services" page of the Upscan documentation in Confluence for the onboarding process. The team are also available on Slack [#team-platops](https://hmrcdigital.slack.com/messages/T04RY81HB). - -Consuming services must identify themselves in requests via the `User-Agent` header. If the supplied value is not in Upscan's list of allowed services then the `/initiate` call will fail with a `403` error. - -In addition to returning a `403` error, Upscan will log details of the Forbidden request. For example: - -```json -{ - "app":"upscan-initiate", - "message":"Invalid User-Agent: [Some(my-unknown-service-name)].", - "logger":"application", - "level":"WARN" -} -``` - -*Note:* If you are using `[http-verbs](https://github.com/hmrc/http-verbs)` to call Upscan, then the `User-Agent` header will be set automatically. -(See: [HttpVerb.scala](https://github.com/hmrc/http-verbs/blob/2807dc65f64009bd7ce1f14b38b356e06dd23512/src/main/scala/uk/gov/hmrc/http/HttpVerb.scala#L53)) - -[[Back to the top]](#top) - ## Error handling This document indicates the responses from Upscan components, including error/failure cases. @@ -403,7 +380,7 @@ Because of this, the URL must not: [[Back to the top]](#top) ### File metadata -Upscan intentionally doesn't allow consuming services to attach metadata or tags to the uploaded file. It is expected that the consuming service will use the globally unique file reference to correlate any file metadata to an successfully uploaded file when it is notified of the successful upload. +Upscan intentionally doesn't allow consuming services to attach metadata or tags to the uploaded file. It is expected that the consuming service will use the globally unique file reference to correlate any file metadata to a successfully uploaded file when it is notified of the successful upload. Upscan must not be used to route or transfer files between different services on MDTP - it is for files to be uploaded into the HMRC estate. diff --git a/app/controllers/PrepareUploadController.scala b/app/controllers/PrepareUploadController.scala index 14c13ff..5ce53fb 100644 --- a/app/controllers/PrepareUploadController.scala +++ b/app/controllers/PrepareUploadController.scala @@ -19,7 +19,7 @@ import scala.util.{Failure, Success, Try} @Singleton class PrepareUploadController @Inject()( prepareUploadService: PrepareUploadService, - override val configuration: ServiceConfiguration, + configuration: ServiceConfiguration, clock: Clock) extends BaseController with UserAgentFilter { @@ -45,7 +45,7 @@ class PrepareUploadController @Inject()( Action.async(parse.json) { implicit request => val receivedAt = Instant.now(clock) - onlyAllowedServices[JsValue] { (_, consumingService) => + requireUserAgent[JsValue] { (_, consumingService) => withJsonBody[T] { prepareUploadRequest: T => withAllowedCallbackProtocol(prepareUploadRequest.callbackUrl) { Logger.debug(s"Processing request: [$prepareUploadRequest].") diff --git a/app/utils/UserAgentFilter.scala b/app/utils/UserAgentFilter.scala index dcf6487..44364c1 100644 --- a/app/utils/UserAgentFilter.scala +++ b/app/utils/UserAgentFilter.scala @@ -1,33 +1,21 @@ package utils -import config.ServiceConfiguration import play.api.Logger -import play.api.http.HeaderNames +import play.api.http.HeaderNames.USER_AGENT +import play.api.mvc.Results.BadRequest import play.api.mvc.{Request, Result} -import play.api.mvc.Results.Forbidden import scala.concurrent.Future trait UserAgentFilter { - - protected val configuration: ServiceConfiguration - - private val userAgents: Seq[String] = configuration.allowedUserAgents - - def onlyAllowedServices[A](block: (Request[A], String) => Future[Result])( - implicit request: Request[A]): Future[Result] = - request.headers.get(HeaderNames.USER_AGENT) match { - case Some(userAgent) if allowedUserAgent(userAgent) => - block(request, userAgent) - case userAgent => - Logger.warn(s"Invalid User-Agent: [$userAgent].") - - Future.successful( - Forbidden( - "This service is not allowed to use upscan-initiate. " + - "If you need to use this service, please contact Platform Services team.")) - } - - private def allowedUserAgent(userAgent: String): Boolean = - userAgents.contains(userAgent) + /* + * We require the user agent to be set with the name of the client service. + */ + def requireUserAgent[A](block: (Request[A], String) => Future[Result])(implicit request: Request[A]): Future[Result] = + request.headers.get(USER_AGENT).fold(onMissingUserAgent())(block(request, _)) + + private def onMissingUserAgent(): Future[Result] = { + Logger.warn(s"No $USER_AGENT Request Header found - unable to identify client service") + Future.successful(BadRequest(s"Missing $USER_AGENT Header")) + } } diff --git a/conf/application.conf b/conf/application.conf index 3e4c889..bdf711e 100644 --- a/conf/application.conf +++ b/conf/application.conf @@ -115,57 +115,55 @@ callbackValidation.allowedProtocols = "https" global.file.size.limit = 104857600 -userAgentFilter.allowedUserAgents = "" - - # Microservice specific config - - Test { - auditing { - enabled=true - traceRequests=true - consumer { - baseUri { - host = localhost - port = 11111 - } +# Microservice specific config + +Test { + auditing { + enabled = true + traceRequests = true + consumer { + baseUri { + host = localhost + port = 11111 } } + } - microservice { - metrics { - graphite { - enabled = false - } - gauges { - interval = 1 second - } + microservice { + metrics { + graphite { + enabled = false + } + gauges { + interval = 1 second } } } +} - Dev { - auditing { - enabled=true - traceRequests=true - consumer { - baseUri { - host = localhost - port = 8100 - } +Dev { + auditing { + enabled = true + traceRequests = true + consumer { + baseUri { + host = localhost + port = 8100 } } + } - microservice { - metrics { - graphite { - enabled = false - } - gauges { - interval = 1 second - } + microservice { + metrics { + graphite { + enabled = false + } + gauges { + interval = 1 second } - } } +} + diff --git a/conf/testOnlyDoNotUseInAppConf.routes b/conf/testOnlyDoNotUseInAppConf.routes deleted file mode 100644 index b98c783..0000000 --- a/conf/testOnlyDoNotUseInAppConf.routes +++ /dev/null @@ -1,13 +0,0 @@ -# IF THE MICRO-SERVICE DOES NOT NEED ANY TEST-ONLY END-POINTS (ALWAYS PREFERRED) DELETE THIS FILE. - -# !!!WARNING!!! This file MUST NOT be referenced in the "application.conf" file to avoid risk of rolling test routes in the production environment. -# If you need test routes when running tests in CI make sure that the profile for this micro-service (used by service-manager) defines this router as parameter. -# To do so add the following line to the micro-service profile: "-Dapplication.router=testOnlyDoNotUseInAppConf.Routes" -# To start the micro-service locally using the test routes run the following command: "sbt run -Dapplication.router=testOnlyDoNotUseInAppConf.Routes" - -# Any test-only end-point should be defined here. -# !!!WARNING!!! Every route defined in this file MUST be prefixed with "/test-only/". This is because NGINX is blocking every uri containing the string "test-only" in production. -# Failing to follow this rule may result in test routes deployed in production. - -# Add all the application routes to the prod.routes file --> / prod.Routes diff --git a/it/controllers/PrepareUploadControllerISpec.scala b/it/controllers/PrepareUploadControllerISpec.scala index 208fc0a..5b570a3 100644 --- a/it/controllers/PrepareUploadControllerISpec.scala +++ b/it/controllers/PrepareUploadControllerISpec.scala @@ -4,6 +4,7 @@ import org.scalatest.GivenWhenThen import org.scalatestplus.play.guice.GuiceOneAppPerSuite import play.api.Application import play.api.http.HeaderNames.USER_AGENT +import play.api.http.Status.BAD_REQUEST import play.api.inject.guice.GuiceApplicationBuilder import play.api.libs.json.{JsValue, Json} import play.api.test.Helpers._ @@ -12,11 +13,13 @@ import uk.gov.hmrc.http.HeaderNames.xSessionId import uk.gov.hmrc.play.test.UnitSpec class PrepareUploadControllerISpec extends UnitSpec with GuiceOneAppPerSuite with GivenWhenThen { + + import PrepareUploadControllerISpec._ + override implicit lazy val app: Application = new GuiceApplicationBuilder() .configure( - "userAgentFilter.allowedUserAgents" -> "PrepareUploadControllerISpec", - "uploadProxy.url" -> "https://upload-proxy.tax.service.gov.uk", - "aws.s3.bucket.inbound" -> "inbound-bucket" + "uploadProxy.url" -> "https://upload-proxy.tax.service.gov.uk", + "aws.s3.bucket.inbound" -> "inbound-bucket" ) .build() @@ -62,11 +65,11 @@ class PrepareUploadControllerISpec extends UnitSpec with GuiceOneAppPerSuite wit href: String): Unit = { "include x-amz-meta-consuming-service in the response" in { - Given("a request containing a USER_AGENT header contained in the configuration of allowedUserAgents") + Given("a request containing a User-Agent header") val initiateRequest = FakeRequest( POST, uri, - FakeHeaders(Seq((USER_AGENT, "PrepareUploadControllerISpec"), (xSessionId, "some-session-id"))), + FakeHeaders(Seq((USER_AGENT, SomeConsumingService), (xSessionId, "some-session-id"))), postBodyJson) When("a request is posted to the /initiate endpoint") @@ -77,23 +80,20 @@ class PrepareUploadControllerISpec extends UnitSpec with GuiceOneAppPerSuite wit And("the response should include the expected value for x-amz-meta-consuming-service") val responseJson = contentAsJson(initiateResponse) - (responseJson \ "uploadRequest" \ "fields" \ "x-amz-meta-consuming-service") - .as[String] shouldBe "PrepareUploadControllerISpec" + .as[String] shouldBe SomeConsumingService And("the href should be the expected url for the upload") - (responseJson \ "uploadRequest" \ "href") .as[String] shouldBe href - } "include x-amz-meta-session-id in the response" in { - Given("a request containing a x-session-id header") + Given("a valid request containing a x-session-id header") val initiateRequest = FakeRequest( POST, uri, - FakeHeaders(Seq((USER_AGENT, "PrepareUploadControllerISpec"), (xSessionId, "some-session-id"))), + FakeHeaders(Seq((USER_AGENT, SomeConsumingService), (xSessionId, "some-session-id"))), postBodyJson) When("a request is posted to the /initiate endpoint") @@ -104,20 +104,17 @@ class PrepareUploadControllerISpec extends UnitSpec with GuiceOneAppPerSuite wit And("the response should include the expected value for x-amz-meta-consuming-service") val responseJson = contentAsJson(initiateResponse) - (responseJson \ "uploadRequest" \ "fields" \ "x-amz-meta-session-id") .as[String] shouldBe "some-session-id" And("the href should be the expected url for the upload") - (responseJson \ "uploadRequest" \ "href") .as[String] shouldBe href } "set a default x-amz-meta-session-id in the response if no session id passed in" in { Given("a request containing a x-session-id header") - val initiateRequest = - FakeRequest(POST, uri, FakeHeaders(Seq((USER_AGENT, "PrepareUploadControllerISpec"))), postBodyJson) + val initiateRequest = FakeRequest(POST, uri, FakeHeaders(Seq((USER_AGENT, SomeConsumingService))), postBodyJson) When("a request is posted to the /initiate endpoint") val initiateResponse = route(app, initiateRequest).get @@ -127,38 +124,27 @@ class PrepareUploadControllerISpec extends UnitSpec with GuiceOneAppPerSuite wit And("the response should include the expected value for x-amz-meta-consuming-service") val responseJson = contentAsJson(initiateResponse) - (responseJson \ "uploadRequest" \ "fields" \ "x-amz-meta-session-id") .as[String] shouldBe "n/a" And("the href should be the expected url for the upload") - (responseJson \ "uploadRequest" \ "href") .as[String] shouldBe href } - "reject requests which do not include a valid USER_AGENT header" in { - Given("a request containing a USER_AGENT header not contained in the configuration of allowedUserAgents") - val initiateRequest = - FakeRequest(POST, uri, FakeHeaders(Seq((USER_AGENT, "SomeInvalidUserAgent"))), postBodyJson) - - When("a request is posted to the /initiate endpoint") - val initiateResponse = route(app, initiateRequest).get - - Then("the response should indicate the request is Forbidden") - status(initiateResponse) shouldBe 403 - } - - "reject requests which do not include a USER_AGENT header" in { - Given("a request not containing a USER_AGENT header") - val initiateRequest = - FakeRequest(POST, uri, FakeHeaders(Seq((xSessionId, "some-session-id"))), postBodyJson) + "reject requests which do not include a User-Agent header" in { + Given("a request not containing a User-Agent header") + val initiateRequest = FakeRequest(POST, uri, FakeHeaders(Seq((xSessionId, "some-session-id"))), postBodyJson) When("a request is posted to the /initiate endpoint") val initiateResponse = route(app, initiateRequest).get - Then("the response should indicate the request is Forbidden") - status(initiateResponse) shouldBe 403 + Then("the response should indicate the request is invalid") + status(initiateResponse) shouldBe BAD_REQUEST } } } + +private object PrepareUploadControllerISpec { + val SomeConsumingService = "PrepareUploadControllerISpec" +} \ No newline at end of file diff --git a/test/controllers/PrepareUploadControllerSpec.scala b/test/controllers/PrepareUploadControllerSpec.scala index b122009..e500fa6 100644 --- a/test/controllers/PrepareUploadControllerSpec.scala +++ b/test/controllers/PrepareUploadControllerSpec.scala @@ -12,6 +12,8 @@ import org.mockito.invocation.InvocationOnMock import org.mockito.stubbing.Answer import org.scalatest.mockito.MockitoSugar import org.scalatest.{GivenWhenThen, Matchers} +import play.api.http.HeaderNames.USER_AGENT +import play.api.http.Status.{BAD_REQUEST, OK} import play.api.libs.json.{JsObject, JsValue, Json} import play.api.mvc.Action import play.api.mvc.Results.Ok @@ -37,7 +39,7 @@ class PrepareUploadControllerSpec extends UnitSpec with Matchers with GivenWhenT "PaymentController prepareUploadV1" should { - behave like prapareUploadTests(_.prepareUploadV1()) + behave like prepareUploadTests(_.prepareUploadV1()) } "PaymentController prepareUploadV2" should { @@ -49,16 +51,15 @@ class PrepareUploadControllerSpec extends UnitSpec with Matchers with GivenWhenT "success_action_redirect" -> "https://www.example.com/nextpage", "error_action_redirect" -> "https://www.example.com/error") - behave like prapareUploadTests(_.prepareUploadV2(), extraRequestFields, extraResponseFields) + behave like prepareUploadTests(_.prepareUploadV2(), extraRequestFields, extraResponseFields) } - private def prapareUploadTests( // scalastyle:ignore + private def prepareUploadTests( // scalastyle:ignore prepareUploadAction: PrepareUploadController => Action[JsValue], extraRequestFields: JsObject = JsObject(Seq()), extraResponseFields: JsObject = JsObject(Seq())) { val config = mock[ServiceConfiguration] - Mockito.when(config.allowedUserAgents).thenReturn(List("VALID-AGENT")) Mockito.when(config.allowedCallbackProtocols).thenReturn(List("https")) "build and return upload URL if valid request with all data" in { @@ -68,12 +69,12 @@ class PrepareUploadControllerSpec extends UnitSpec with Matchers with GivenWhenT val request: FakeRequest[JsValue] = FakeRequest() .withHeaders( - ("User-Agent", "VALID-AGENT"), + (USER_AGENT, "SOME-USER-AGENT"), ("x-session-id", "some-session-id"), ("x-request-id", "some-request-id")) .withBody( Json.obj( - "id" -> "1", + "id" -> "1", "callbackUrl" -> "https://www.example.com", "minimumFileSize" -> 0, "maximumFileSize" -> 1024) ++ extraRequestFields) @@ -84,7 +85,7 @@ class PrepareUploadControllerSpec extends UnitSpec with Matchers with GivenWhenT Then("service returns valid response with reference and template of upload form") - withClue(Helpers.contentAsString(result)) { status(result) shouldBe 200 } + withClue(Helpers.contentAsString(result)) { status(result) shouldBe OK } val json = Helpers.contentAsJson(result) json shouldBe Json.obj( "reference" -> "TEST", @@ -106,12 +107,12 @@ class PrepareUploadControllerSpec extends UnitSpec with Matchers with GivenWhenT val request: FakeRequest[JsValue] = FakeRequest() .withHeaders( - ("User-Agent", "VALID-AGENT"), + (USER_AGENT, "SOME-USER-AGENT"), ("x-session-id", "some-session-id"), ("x-request-id", "some-request-id")) .withBody( Json.obj( - "id" -> "1", + "id" -> "1", "callbackUrl" -> "https://www.example.com", "successRedirect" -> "https://www.example.com/nextpage", "minimumFileSize" -> 0, @@ -123,14 +124,14 @@ class PrepareUploadControllerSpec extends UnitSpec with Matchers with GivenWhenT Then("service returns valid response with reference and template of upload form") - withClue(Helpers.contentAsString(result)) { status(result) shouldBe 200 } + withClue(Helpers.contentAsString(result)) { status(result) shouldBe OK } val json = Helpers.contentAsJson(result) json shouldBe Json.obj( "reference" -> "TEST", "uploadRequest" -> Json.obj( "href" -> "https://www.example.com", "fields" -> (Json.obj( - "minFileSize" -> "0", + "minFileSize" -> "0", "maxFileSize" -> "1024", "sessionId" -> "some-session-id", "requestId" -> "some-request-id", @@ -147,7 +148,7 @@ class PrepareUploadControllerSpec extends UnitSpec with Matchers with GivenWhenT val request: FakeRequest[JsValue] = FakeRequest() .withHeaders( - ("User-Agent", "VALID-AGENT"), + (USER_AGENT, "SOME-USER-AGENT"), ("x-session-id", "some-session-id"), ("x-request-id", "some-request-id")) .withBody(Json.obj("callbackUrl" -> "https://www.example.com") ++ extraRequestFields) @@ -158,7 +159,7 @@ class PrepareUploadControllerSpec extends UnitSpec with Matchers with GivenWhenT Then("service returns valid response with reference and template of upload form") - withClue(Helpers.contentAsString(result)) { status(result) shouldBe 200 } + withClue(Helpers.contentAsString(result)) { status(result) shouldBe OK } val json = Helpers.contentAsJson(result) json shouldBe Json.obj( "reference" -> "TEST", @@ -176,7 +177,7 @@ class PrepareUploadControllerSpec extends UnitSpec with Matchers with GivenWhenT Given("there is a valid upload request with minimal data") val request: FakeRequest[JsValue] = FakeRequest() - .withHeaders(("User-Agent", "VALID-AGENT")) + .withHeaders((USER_AGENT, "SOME-USER-AGENT")) .withBody(Json.obj("callbackUrl" -> "https://www.example.com") ++ extraRequestFields) When("upload initiation has been requested") @@ -185,12 +186,12 @@ class PrepareUploadControllerSpec extends UnitSpec with Matchers with GivenWhenT Then("service returns valid response with reference and template of upload form") - withClue(Helpers.contentAsString(result)) { status(result) shouldBe 200 } + withClue(Helpers.contentAsString(result)) { status(result) shouldBe OK } val json = Helpers.contentAsJson(result) json shouldBe Json.obj( "reference" -> "TEST", "uploadRequest" -> Json.obj( - "href" -> "https://www.example.com", + "href" -> "https://www.example.com", "fields" -> (Json.obj("sessionId" -> "n/a", "requestId" -> "n/a") ++ extraResponseFields) ) ) @@ -203,7 +204,7 @@ class PrepareUploadControllerSpec extends UnitSpec with Matchers with GivenWhenT val request: FakeRequest[JsValue] = FakeRequest() .withHeaders( - ("User-Agent", "VALID-AGENT"), + (USER_AGENT, "SOME-USER-AGENT"), ("x-session-id", "some-session-id"), ("x-request-id", "some-request-id")) .withBody(Json.obj("invalid" -> "body")) @@ -214,7 +215,7 @@ class PrepareUploadControllerSpec extends UnitSpec with Matchers with GivenWhenT Then("service returns error response") - withClue(Helpers.contentAsString(result)) { status(result) shouldBe 400 } + withClue(Helpers.contentAsString(result)) { status(result) shouldBe BAD_REQUEST } } "return a bad request error if invalid request - incorrect maximum file size " in { @@ -223,7 +224,7 @@ class PrepareUploadControllerSpec extends UnitSpec with Matchers with GivenWhenT Given("there is an invalid upload request") val request: FakeRequest[JsValue] = FakeRequest() - .withHeaders(("User-Agent", "VALID-AGENT"), ("x-session-id", "some-session-id")) + .withHeaders((USER_AGENT, "SOME-USER-AGENT"), ("x-session-id", "some-session-id")) .withBody(Json.obj("callbackUrl" -> "https://www.example.com", "maximumFileSize" -> 2048)) When("upload initiation has been requested") @@ -233,77 +234,7 @@ class PrepareUploadControllerSpec extends UnitSpec with Matchers with GivenWhenT Then("service returns error response") withClue(Helpers.contentAsString(result)) { - status(result) shouldBe 400 - } - } - - "return okay if service is allowed on whitelist " in { - val controller = new PrepareUploadController(prepareUploadService, config, clock) - - Given("there is a valid upload request from a whitelisted service") - - val request: FakeRequest[JsValue] = FakeRequest() - .withHeaders( - ("User-Agent", "VALID-AGENT"), - ("x-session-id", "some-session-id"), - ("x-request-id", "some-request-id")) - .withBody(Json.obj( - "id" -> "1", - "callbackUrl" -> "https://www.example.com", - "minimumFileSize" -> 0, - "maximumFileSize" -> 1024, - "session-id" -> "some-session-id", - "request-id" -> "some-request-id" - ) ++ extraRequestFields) - - When("upload initiation has been requested") - - val result = prepareUploadAction(controller)(request) - - Then("service returns error response") - - withClue(Helpers.contentAsString(result)) { status(result) shouldBe 200 } - val json = Helpers.contentAsJson(result) - json shouldBe Json.obj( - "reference" -> "TEST", - "uploadRequest" -> Json.obj( - "href" -> "https://www.example.com", - "fields" -> (Json.obj( - "minFileSize" -> "0", - "maxFileSize" -> "1024", - "sessionId" -> "some-session-id", - "requestId" -> "some-request-id") ++ extraResponseFields) - ) - ) - } - - "return a forbidden error if service is not whitelisted " in { - val controller = new PrepareUploadController(prepareUploadService, config, clock) - - Given("there is a valid upload request from a non-whitelisted service") - - val request: FakeRequest[JsValue] = FakeRequest() - .withHeaders( - ("User-Agent", "INVALID-AGENT"), - ("x-session-id", "some-session-id"), - ("x-request-id", "some-request-id")) - .withBody(Json.obj( - "id" -> "1", - "callbackUrl" -> "http://www.example.com", - "minimumFileSize" -> 0, - "maximumFileSize" -> 1024, - "session-id" -> "some-session-id", - "request-id" -> "some-request-id" - ) ++ extraRequestFields) - - When("upload initiation has been requested") - - val result = prepareUploadAction(controller)(request) - - Then("service returns error response") - - withClue(Helpers.contentAsString(result)) { - status(result) shouldBe 403 + status(result) shouldBe BAD_REQUEST } } @@ -314,7 +245,7 @@ class PrepareUploadControllerSpec extends UnitSpec with Matchers with GivenWhenT Future.successful(Ok) } - status(result) shouldBe 200 + status(result) shouldBe OK } "disallow http callback urls" in { @@ -324,7 +255,7 @@ class PrepareUploadControllerSpec extends UnitSpec with Matchers with GivenWhenT Future.failed(new RuntimeException("This block should not have been invoked.")) } - status(result) shouldBe 400 + status(result) shouldBe BAD_REQUEST contentAsString(result) should include("Invalid callback url protocol") } @@ -334,7 +265,7 @@ class PrepareUploadControllerSpec extends UnitSpec with Matchers with GivenWhenT val result = controller.withAllowedCallbackProtocol("123") { Future.failed(new RuntimeException("This block should not have been invoked.")) } - status(result) shouldBe 400 + status(result) shouldBe BAD_REQUEST contentAsString(result) should include("Invalid callback url format") } } diff --git a/test/utils/UserAgentFilterSpec.scala b/test/utils/UserAgentFilterSpec.scala index dc46748..c383fa8 100644 --- a/test/utils/UserAgentFilterSpec.scala +++ b/test/utils/UserAgentFilterSpec.scala @@ -1,54 +1,54 @@ package utils import akka.util.Timeout -import config.ServiceConfiguration -import org.mockito.Mockito -import org.scalatest.mockito.MockitoSugar import org.scalatest.{GivenWhenThen, Matchers} -import play.api.mvc.{Request, Result} +import play.api.http.Status.{BAD_REQUEST, OK} import play.api.mvc.Results._ +import play.api.mvc.{Request, Result} +import play.api.test.Helpers.contentAsString import play.api.test.{FakeRequest, Helpers} +import play.mvc.Http.HeaderNames.USER_AGENT import uk.gov.hmrc.play.test.UnitSpec import scala.concurrent.Future import scala.concurrent.duration._ -class UserAgentFilterSpec extends UnitSpec with Matchers with GivenWhenThen with MockitoSugar { +class UserAgentFilterSpec extends UnitSpec with Matchers with GivenWhenThen { - class UserAgentFilterImpl(override val configuration: ServiceConfiguration) extends UserAgentFilter + import UserAgentFilterSpec._ "UserAgentFilter" should { - val block: (Request[_], String) => Future[Result] = (_, _) => Future.successful(Ok("This is a successful result")) - - implicit val timeout = Timeout(3.seconds) + "accept a request when the User-Agent header is specified" in { + Given("a request that specifies a User-Agent header") + val request = FakeRequest().withHeaders((USER_AGENT, SomeUserAgent)) - "accept request if user agent in whitelist" in { - Given("a service configuration with no whitelist set") - val config = mock[ServiceConfiguration] - Mockito.when(config.allowedUserAgents).thenReturn(List("VALID-AGENT")) - val filter = new UserAgentFilterImpl(config) + When("the request is received") + val result = UserAgentFilter.requireUserAgent(block)(request) - When("a request is received") - val result = filter.onlyAllowedServices(block)(FakeRequest().withHeaders(("User-Agent", "VALID-AGENT"))) + Then("the request should be accepted") + status(result) shouldBe OK - Then("the request should be passed through the filter") - status(result) shouldBe 200 - Helpers.contentAsString(result) shouldBe "This is a successful result" + And("the User-Agent header value should be passed to the block") + contentAsString(result) shouldBe SomeUserAgent } - "reject request if user agent not in whitelist" in { - Given("a service configuration with no whitelist set") - val config = mock[ServiceConfiguration] - Mockito.when(config.allowedUserAgents).thenReturn(List("VALID-AGENT")) - val filter = new UserAgentFilterImpl(config) + "reject a request when the User-Agent header is not specified" in { + Given("a request that does not specify a User-Agent header") + val request = FakeRequest() - When("a request is received") - val result = filter.onlyAllowedServices(block)(FakeRequest()) + When("the request is received") + val result = UserAgentFilter.requireUserAgent(block)(request) - Then("the filter should reject as forbidden") - status(result) shouldBe 403 - Helpers.contentAsString(result) shouldBe "This service is not allowed to use upscan-initiate. " + - "If you need to use this service, please contact Platform Services team." + Then("the request should be rejected") + status(result) shouldBe BAD_REQUEST } } } + +private object UserAgentFilterSpec { + implicit val timeout: Timeout = Timeout(3.seconds) + val SomeUserAgent = "SOME_USER-AGENT" + val block: (Request[_], String) => Future[Result] = (_, userAgent) => Future.successful(Ok(userAgent)) + + object UserAgentFilter extends UserAgentFilter +} \ No newline at end of file From d6f018712c5d5b4739d3f2f2ba32f5e544e44ba9 Mon Sep 17 00:00:00 2001 From: Nigel Perkins Date: Mon, 24 Feb 2020 09:11:10 +0000 Subject: [PATCH 35/96] BDOG-586: add LICENSE file and copyright headers --- LICENSE | 201 ++++++++++++++++++ app/UpscanInitiateModule.scala | 16 ++ app/config/ServiceConfiguration.scala | 16 ++ app/connectors/model/AwsCredentials.scala | 16 ++ app/connectors/model/ContentLengthRange.scala | 16 ++ .../model/UploadFormGenerator.scala | 16 ++ app/connectors/model/UploadParameters.scala | 16 ++ app/connectors/s3/PolicySigner.scala | 16 ++ app/connectors/s3/S3Module.scala | 16 ++ app/connectors/s3/S3UploadFormGenerator.scala | 16 ++ .../s3/S3UploadFormGeneratorProvider.scala | 16 ++ app/controllers/PrepareUploadController.scala | 16 ++ app/controllers/model/PrepareUpload.scala | 16 ++ .../model/PrepareUploadRequestV1.scala | 16 ++ .../model/PrepareUploadRequestV2.scala | 16 ++ .../model/PreparedUploadResponse.scala | 16 ++ app/controllers/model/Reference.scala | 16 ++ .../model/UploadFormTemplate.scala | 16 ++ app/services/PrepareUploadService.scala | 16 ++ app/services/model/UploadSettings.scala | 16 ++ app/utils/UserAgentFilter.scala | 16 ++ conf/application.conf | 13 ++ test/connectors/s3/PolicySignerSpec.scala | 16 ++ .../s3/S3UploadFormGeneratorSpec.scala | 16 ++ .../PrepareUploadControllerSpec.scala | 16 ++ test/model/PrepareUploadServiceSpec.scala | 16 ++ test/utils/UserAgentFilterSpec.scala | 16 ++ 27 files changed, 614 insertions(+) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..8dada3e --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/app/UpscanInitiateModule.scala b/app/UpscanInitiateModule.scala index 242499a..5bafce1 100644 --- a/app/UpscanInitiateModule.scala +++ b/app/UpscanInitiateModule.scala @@ -1,3 +1,19 @@ +/* + * Copyright 2020 HM Revenue & Customs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + import java.time.Clock import config.{PlayBasedServiceConfiguration, ServiceConfiguration} diff --git a/app/config/ServiceConfiguration.scala b/app/config/ServiceConfiguration.scala index 1ce47f8..d6bccee 100644 --- a/app/config/ServiceConfiguration.scala +++ b/app/config/ServiceConfiguration.scala @@ -1,3 +1,19 @@ +/* + * Copyright 2020 HM Revenue & Customs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package config import java.time.Duration diff --git a/app/connectors/model/AwsCredentials.scala b/app/connectors/model/AwsCredentials.scala index 7652a7c..e55897a 100644 --- a/app/connectors/model/AwsCredentials.scala +++ b/app/connectors/model/AwsCredentials.scala @@ -1,3 +1,19 @@ +/* + * Copyright 2020 HM Revenue & Customs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package connectors.model final case class AwsCredentials(accessKeyId: String, secretKey: String, sessionToken: Option[String]) diff --git a/app/connectors/model/ContentLengthRange.scala b/app/connectors/model/ContentLengthRange.scala index 81a5523..44c7f4a 100644 --- a/app/connectors/model/ContentLengthRange.scala +++ b/app/connectors/model/ContentLengthRange.scala @@ -1,3 +1,19 @@ +/* + * Copyright 2020 HM Revenue & Customs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package connectors.model case class ContentLengthRange(min: Int, max: Int) diff --git a/app/connectors/model/UploadFormGenerator.scala b/app/connectors/model/UploadFormGenerator.scala index 2663a4f..42a5165 100644 --- a/app/connectors/model/UploadFormGenerator.scala +++ b/app/connectors/model/UploadFormGenerator.scala @@ -1,3 +1,19 @@ +/* + * Copyright 2020 HM Revenue & Customs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package connectors.model trait UploadFormGenerator { diff --git a/app/connectors/model/UploadParameters.scala b/app/connectors/model/UploadParameters.scala index 63ca018..2e5f400 100644 --- a/app/connectors/model/UploadParameters.scala +++ b/app/connectors/model/UploadParameters.scala @@ -1,3 +1,19 @@ +/* + * Copyright 2020 HM Revenue & Customs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package connectors.model import java.time.Instant diff --git a/app/connectors/s3/PolicySigner.scala b/app/connectors/s3/PolicySigner.scala index a1e7a84..f9073cc 100644 --- a/app/connectors/s3/PolicySigner.scala +++ b/app/connectors/s3/PolicySigner.scala @@ -1,3 +1,19 @@ +/* + * Copyright 2020 HM Revenue & Customs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package connectors.s3 import java.nio.charset.Charset diff --git a/app/connectors/s3/S3Module.scala b/app/connectors/s3/S3Module.scala index 6edf981..391c8c2 100644 --- a/app/connectors/s3/S3Module.scala +++ b/app/connectors/s3/S3Module.scala @@ -1,3 +1,19 @@ +/* + * Copyright 2020 HM Revenue & Customs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package connectors.s3 import connectors.model.UploadFormGenerator diff --git a/app/connectors/s3/S3UploadFormGenerator.scala b/app/connectors/s3/S3UploadFormGenerator.scala index 0c53d7a..9055df5 100644 --- a/app/connectors/s3/S3UploadFormGenerator.scala +++ b/app/connectors/s3/S3UploadFormGenerator.scala @@ -1,3 +1,19 @@ +/* + * Copyright 2020 HM Revenue & Customs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package connectors.s3 import java.time.format.DateTimeFormatter diff --git a/app/connectors/s3/S3UploadFormGeneratorProvider.scala b/app/connectors/s3/S3UploadFormGeneratorProvider.scala index b9d378a..40bddd4 100644 --- a/app/connectors/s3/S3UploadFormGeneratorProvider.scala +++ b/app/connectors/s3/S3UploadFormGeneratorProvider.scala @@ -1,3 +1,19 @@ +/* + * Copyright 2020 HM Revenue & Customs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package connectors.s3 import config.ServiceConfiguration diff --git a/app/controllers/PrepareUploadController.scala b/app/controllers/PrepareUploadController.scala index 5ce53fb..5985980 100644 --- a/app/controllers/PrepareUploadController.scala +++ b/app/controllers/PrepareUploadController.scala @@ -1,3 +1,19 @@ +/* + * Copyright 2020 HM Revenue & Customs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package controllers import java.net.URL diff --git a/app/controllers/model/PrepareUpload.scala b/app/controllers/model/PrepareUpload.scala index 9ba792c..4cea64c 100644 --- a/app/controllers/model/PrepareUpload.scala +++ b/app/controllers/model/PrepareUpload.scala @@ -1,3 +1,19 @@ +/* + * Copyright 2020 HM Revenue & Customs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package controllers.model import services.model.UploadSettings diff --git a/app/controllers/model/PrepareUploadRequestV1.scala b/app/controllers/model/PrepareUploadRequestV1.scala index 19a33d1..a1b51d8 100644 --- a/app/controllers/model/PrepareUploadRequestV1.scala +++ b/app/controllers/model/PrepareUploadRequestV1.scala @@ -1,3 +1,19 @@ +/* + * Copyright 2020 HM Revenue & Customs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package controllers.model import play.api.data.validation.ValidationError diff --git a/app/controllers/model/PrepareUploadRequestV2.scala b/app/controllers/model/PrepareUploadRequestV2.scala index 74c4041..3e6dde4 100644 --- a/app/controllers/model/PrepareUploadRequestV2.scala +++ b/app/controllers/model/PrepareUploadRequestV2.scala @@ -1,3 +1,19 @@ +/* + * Copyright 2020 HM Revenue & Customs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package controllers.model import play.api.data.validation.ValidationError diff --git a/app/controllers/model/PreparedUploadResponse.scala b/app/controllers/model/PreparedUploadResponse.scala index 4261fc7..bde5e80 100644 --- a/app/controllers/model/PreparedUploadResponse.scala +++ b/app/controllers/model/PreparedUploadResponse.scala @@ -1,3 +1,19 @@ +/* + * Copyright 2020 HM Revenue & Customs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package controllers.model import play.api.libs.functional.syntax._ diff --git a/app/controllers/model/Reference.scala b/app/controllers/model/Reference.scala index 4cc8ddd..27e440f 100644 --- a/app/controllers/model/Reference.scala +++ b/app/controllers/model/Reference.scala @@ -1,3 +1,19 @@ +/* + * Copyright 2020 HM Revenue & Customs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package controllers.model import java.util.UUID.randomUUID diff --git a/app/controllers/model/UploadFormTemplate.scala b/app/controllers/model/UploadFormTemplate.scala index 47cdf61..8db2618 100644 --- a/app/controllers/model/UploadFormTemplate.scala +++ b/app/controllers/model/UploadFormTemplate.scala @@ -1,3 +1,19 @@ +/* + * Copyright 2020 HM Revenue & Customs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package controllers.model import play.api.libs.functional.syntax._ diff --git a/app/services/PrepareUploadService.scala b/app/services/PrepareUploadService.scala index 0de7261..545b952 100644 --- a/app/services/PrepareUploadService.scala +++ b/app/services/PrepareUploadService.scala @@ -1,3 +1,19 @@ +/* + * Copyright 2020 HM Revenue & Customs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package services import java.time.Instant diff --git a/app/services/model/UploadSettings.scala b/app/services/model/UploadSettings.scala index ccc2c96..db6b97f 100644 --- a/app/services/model/UploadSettings.scala +++ b/app/services/model/UploadSettings.scala @@ -1,3 +1,19 @@ +/* + * Copyright 2020 HM Revenue & Customs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package services.model case class UploadSettings( diff --git a/app/utils/UserAgentFilter.scala b/app/utils/UserAgentFilter.scala index 44364c1..b30a24b 100644 --- a/app/utils/UserAgentFilter.scala +++ b/app/utils/UserAgentFilter.scala @@ -1,3 +1,19 @@ +/* + * Copyright 2020 HM Revenue & Customs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package utils import play.api.Logger diff --git a/conf/application.conf b/conf/application.conf index bdf711e..8431c8d 100644 --- a/conf/application.conf +++ b/conf/application.conf @@ -1,3 +1,16 @@ +# Copyright 2020 HM Revenue & Customs +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # This is the main configuration file for the application. # ~~~~~ diff --git a/test/connectors/s3/PolicySignerSpec.scala b/test/connectors/s3/PolicySignerSpec.scala index f0073f4..2fb7994 100644 --- a/test/connectors/s3/PolicySignerSpec.scala +++ b/test/connectors/s3/PolicySignerSpec.scala @@ -1,3 +1,19 @@ +/* + * Copyright 2020 HM Revenue & Customs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package connectors.s3 import connectors.model.AwsCredentials diff --git a/test/connectors/s3/S3UploadFormGeneratorSpec.scala b/test/connectors/s3/S3UploadFormGeneratorSpec.scala index 8720de5..eade6c0 100644 --- a/test/connectors/s3/S3UploadFormGeneratorSpec.scala +++ b/test/connectors/s3/S3UploadFormGeneratorSpec.scala @@ -1,3 +1,19 @@ +/* + * Copyright 2020 HM Revenue & Customs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package connectors.s3 import java.time.Instant diff --git a/test/controllers/PrepareUploadControllerSpec.scala b/test/controllers/PrepareUploadControllerSpec.scala index e500fa6..de52859 100644 --- a/test/controllers/PrepareUploadControllerSpec.scala +++ b/test/controllers/PrepareUploadControllerSpec.scala @@ -1,3 +1,19 @@ +/* + * Copyright 2020 HM Revenue & Customs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package controllers import java.time.Clock diff --git a/test/model/PrepareUploadServiceSpec.scala b/test/model/PrepareUploadServiceSpec.scala index e30809f..a728e5d 100644 --- a/test/model/PrepareUploadServiceSpec.scala +++ b/test/model/PrepareUploadServiceSpec.scala @@ -1,3 +1,19 @@ +/* + * Copyright 2020 HM Revenue & Customs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package model import java.time diff --git a/test/utils/UserAgentFilterSpec.scala b/test/utils/UserAgentFilterSpec.scala index c383fa8..d0919a3 100644 --- a/test/utils/UserAgentFilterSpec.scala +++ b/test/utils/UserAgentFilterSpec.scala @@ -1,3 +1,19 @@ +/* + * Copyright 2020 HM Revenue & Customs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package utils import akka.util.Timeout From 3525a5a047da438139f8671d37ddbed0be6daff6 Mon Sep 17 00:00:00 2001 From: Nigel Perkins Date: Tue, 25 Feb 2020 10:15:29 +0000 Subject: [PATCH 36/96] BDOG-586: remove unused config accessor --- app/config/ServiceConfiguration.scala | 5 ----- test/model/PrepareUploadServiceSpec.scala | 2 -- 2 files changed, 7 deletions(-) diff --git a/app/config/ServiceConfiguration.scala b/app/config/ServiceConfiguration.scala index d6bccee..70aecb5 100644 --- a/app/config/ServiceConfiguration.scala +++ b/app/config/ServiceConfiguration.scala @@ -34,8 +34,6 @@ trait ServiceConfiguration { def fileExpirationPeriod: java.time.Duration def globalFileSizeLimit: Int def allowedCallbackProtocols: Seq[String] - def allowedUserAgents: Seq[String] - } class PlayBasedServiceConfiguration @Inject()(configuration: Configuration) extends ServiceConfiguration { @@ -60,9 +58,6 @@ class PlayBasedServiceConfiguration @Inject()(configuration: Configuration) exte override def allowedCallbackProtocols: Seq[String] = commaSeparatedList(configuration.getString("callbackValidation.allowedProtocols")) - override def allowedUserAgents: Seq[String] = - commaSeparatedList(configuration.getString("userAgentFilter.allowedUserAgents")) - private def getRequired[T](read: String => Option[T], path: String): T = read(path).getOrElse(throw new ConfigException.Missing(path)) diff --git a/test/model/PrepareUploadServiceSpec.scala b/test/model/PrepareUploadServiceSpec.scala index a728e5d..0b3da1d 100644 --- a/test/model/PrepareUploadServiceSpec.scala +++ b/test/model/PrepareUploadServiceSpec.scala @@ -49,8 +49,6 @@ class PrepareUploadServiceSpec extends UnitSpec with Matchers with GivenWhenThen override def globalFileSizeLimit = 1024 override def allowedCallbackProtocols: List[String] = List("https") - - override def allowedUserAgents: List[String] = ??? } private def metricsStub() = new Metrics { From 42b0b490a365aa8921f445a3f83f0cc7fc9870d2 Mon Sep 17 00:00:00 2001 From: Nigel Perkins Date: Tue, 25 Feb 2020 15:00:07 +0000 Subject: [PATCH 37/96] BDOG-585: document default S3 Url Expiry and how to configure per service --- README.md | 58 ++++++++++++++++++++++++++++++++----------------------- 1 file changed, 34 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 08a0307..79d7016 100644 --- a/README.md +++ b/README.md @@ -15,24 +15,23 @@ This service is not for transfer of files from one HMRC service to another. See ## Contents 1. [Introduction](#introduction) -2. [Onboarding requirements](#onboard) -3. [File upload workflow](#workflow) -4. [Service usage](#service) +2. [File upload workflow](#workflow) +3. [Service usage](#service) a. [Requesting a URL to upload to](#service__request) b. [The file upload](#service__upload) c. [File upload outcome](#service__uoutcome) d. [File processing outcome](#service__poutcome) i. [Success](#service__poutcome__success) ii. [Failure](#service__poutcome__failure) -5. [Error handling](#error) -6. [Design considerations](#design) +4. [Error handling](#error) +5. [Design considerations](#design) a. [Uploading multiple files](#design__multiple) b. [Security](#design__security) c. [File metadata](#design__metadata) -7. [Architecture of the service](#architecture) -8. [Running and maintenance of the service](#run) +6. [Architecture of the service](#architecture) +7. [Running and maintenance of the service](#run) a. [Running locally](#run__local) -9. [Appendix](#appendix) +8. [Appendix](#appendix) a. [Quick reference figures](#appendix__figures) b. [Related projects, useful links](#appendix__links) i. [Testing](#appendix__links__testing) @@ -49,22 +48,16 @@ Once the upload URL has been requested, upload and verification of a file are pe [[Back to the top]](#top) -## Onboarding requirements -To use Upscan, the consuming service must let Platform Services know: -- how long they would like download URLs for their files to be valid for \[max. 7 days] - -[[Back to the top]](#top) - ## File upload workflow * Consuming service requests upload of a single file. It makes a HTTP POST call to the `/upscan/initiate` endpoint with details of the requested upload (including a callback URL and optional file size constraints) * Upscan service replies with a template of the HTTP POST form that will be used to upload the file. A unique file reference is also provided to allow the consuming service to correlate the upload request with notifications to the callback URL provided * Consuming service passes the form details to the end-user, either via a control on a webpage or to an internal/external service which will upload the file -* The end user uploads the file +* The end user uploads the file. This must take place within 7 days of the upload being initiated. * Upscan service performs all checks on the uploaded file * If the file passes all checks, a notification is sent as a POST request to a consuming service. This notification contains the URL to GET the file from -* Consuming service downloads the file using provided URL or passes this URL on to another service which will make use of the file location -* After specified time, the file is automatically removed from the remote storage. Upscan does NOT keep files indefinitely +* Consuming service downloads the file using the provided URL or passes this URL on to another service which will make use of the file location. This download URL only remains valid for a limited duration (see [Success](#service__poutcome__success)) +* After some time the file is automatically removed from the remote storage. Upscan does NOT keep files indefinitely * If the file fails a check, a notification is sent to the consuming service containing information on the failed check. The file is unavailable for retrieval * If the consuming service fails to respond to the callback request (e.g. the consuming service is down, the consuming service answered with an HTTP status code other than 2xx), the callback will be retried up to a maximum of 30 retries. The time interval between the retries is 60 seconds Configuration of these values is here (https://github.com/hmrc/upscan-infrastructure/blob/master/modules/sqs/main.tf) @@ -85,7 +78,7 @@ The service must also provide a callbackUrl for asynchronous notification of the (Although this rule is relaxed when testing locally with [upscan-stub](https://github.com/hmrc/upscan-stub) rather than [upscan-initiate](https://github.com/hmrc/upscan-initiate). In this stubbed scenario a `callbackUrl` referring to localhost may still specify `http` as the protocol.) -Session-ID / Request-ID headers will be used to link the file with user's journey. +Session-ID / Request-ID headers will be used to link the file with the user's journey. *Note:* If you are using `[http-verbs](https://github.com/hmrc/http-verbs)` to call Upscan, all the headers will be set automatically (See: [HttpVerb.scala](https://github.com/hmrc/http-verbs/blob/2807dc65f64009bd7ce1f14b38b356e06dd23512/src/main/scala/uk/gov/hmrc/http/HttpVerb.scala#L53)) @@ -305,6 +298,23 @@ Note the block entitled 'uploadDetails', see the Confluence page 'Upscan & Non-R - `fileMimeType` - Detected MIME type of the file. Please note that this refers to actual contents of the file, not to the name (if user uploads PDF document named `data.png`, it will be detected as a `application/pdf`) +The downloadUrl will expire after 1 day by default. This can be configured on a per-consuming service basis. +For example, to limit to 1 hour add the following configuration (substituting the appropriate `User-Agent` service identifier) to [upscan-notify.conf](https://github.com/hmrc/app-config-base/blob/master/upscan-notify.conf): + +``` +Prod { + upscan { + { + aws { + s3 { + urlExpirationPeriod = 1.hour + } + } + } + } +} +``` + [[Back to the top]](#top) #### Failure @@ -450,12 +460,12 @@ These commands will give you an access to SBT shell where you can run the servic ## Appendix ### Quick reference figures -| Metric | Value | Comments | -| ------------------------------------------------------- | -------------- |----------| -| Expiration of S3 upload pre-signed URL | Up to 7 days | A relatively long period, since we can't control exactly when users will initiate the upload process | -| Expiration of S3 download pre-signed URL (scanned docs) | Up to 1 day | Upscan is not intended as a storage solution for services | -| Callback request retry time | 60 seconds | | -| Maximum callback notification retries | 30 | | +| Metric | Value | Comments | +| ------------------------------------------------------- | ----------------|----------| +| Expiration of S3 upload pre-signed URL | 7 days | A relatively long period, since we can't control exactly when users will initiate the upload process | +| Expiration of S3 download pre-signed URL (scanned docs) | 1 day (default) | Configurable per-service up to 7 days. Upscan is not intended as a storage solution for services | +| Callback request retry time | 60 seconds | | +| Maximum callback notification retries | 30 | | [[Back to the top]](#top) From 987f573157614071acdf38e37464d388dbc80067 Mon Sep 17 00:00:00 2001 From: Nigel Perkins Date: Thu, 27 Feb 2020 08:59:16 +0000 Subject: [PATCH 38/96] BDOG-643: update to bootstrap-play-26 --- app/controllers/PrepareUploadController.scala | 9 ++++--- .../model/PrepareUploadRequestV1.scala | 5 ++-- .../model/PrepareUploadRequestV2.scala | 7 +++--- conf/app.routes | 4 ++-- conf/prod.routes | 2 +- project/MicroService.scala | 5 +--- project/MicroServiceBuild.scala | 11 +++++---- project/plugins.sbt | 2 +- .../s3/S3UploadFormGeneratorSpec.scala | 2 +- .../PrepareUploadControllerSpec.scala | 24 +++++++++---------- test/utils/UserAgentFilterSpec.scala | 2 +- 11 files changed, 34 insertions(+), 39 deletions(-) diff --git a/app/controllers/PrepareUploadController.scala b/app/controllers/PrepareUploadController.scala index 5985980..e63173b 100644 --- a/app/controllers/PrepareUploadController.scala +++ b/app/controllers/PrepareUploadController.scala @@ -24,9 +24,9 @@ import controllers.model.{PrepareUpload, PrepareUploadRequestV1, PrepareUploadRe import javax.inject.{Inject, Singleton} import play.api.Logger import play.api.libs.json._ -import play.api.mvc.{Action, Result} +import play.api.mvc.{Action, ControllerComponents, Result} import services.PrepareUploadService -import uk.gov.hmrc.play.bootstrap.controller.BaseController +import uk.gov.hmrc.play.bootstrap.controller.BackendController import utils.UserAgentFilter import scala.concurrent.Future @@ -36,9 +36,8 @@ import scala.util.{Failure, Success, Try} class PrepareUploadController @Inject()( prepareUploadService: PrepareUploadService, configuration: ServiceConfiguration, - clock: Clock) - extends BaseController - with UserAgentFilter { + clock: Clock, + controllerComponents: ControllerComponents) extends BackendController(controllerComponents) with UserAgentFilter { implicit val prepareUploadRequestReads: Reads[PrepareUploadRequestV1] = PrepareUploadRequestV1.reads(prepareUploadService.globalFileSizeLimit) diff --git a/app/controllers/model/PrepareUploadRequestV1.scala b/app/controllers/model/PrepareUploadRequestV1.scala index a1b51d8..f299d34 100644 --- a/app/controllers/model/PrepareUploadRequestV1.scala +++ b/app/controllers/model/PrepareUploadRequestV1.scala @@ -16,10 +16,9 @@ package controllers.model -import play.api.data.validation.ValidationError import play.api.libs.functional.syntax._ import play.api.libs.json.Reads.{max, min} -import play.api.libs.json.{JsPath, Reads} +import play.api.libs.json.{JsPath, JsonValidationError, Reads} import services.model.UploadSettings case class PrepareUploadRequestV1( @@ -50,7 +49,7 @@ object PrepareUploadRequestV1 { (JsPath \ "maximumFileSize").readNullable[Int](min(0) keepAnd max(maxFileSize + 1)) and (JsPath \ "expectedContentType").readNullable[String] and (JsPath \ "successRedirect").readNullable[String])(PrepareUploadRequestV1.apply _) - .filter(ValidationError("Maximum file size must be equal or greater than minimum file size"))(request => + .filter(JsonValidationError("Maximum file size must be equal or greater than minimum file size"))(request => request.minimumFileSize.getOrElse(0) <= request.maximumFileSize.getOrElse(maxFileSize)) } diff --git a/app/controllers/model/PrepareUploadRequestV2.scala b/app/controllers/model/PrepareUploadRequestV2.scala index 3e6dde4..48661d5 100644 --- a/app/controllers/model/PrepareUploadRequestV2.scala +++ b/app/controllers/model/PrepareUploadRequestV2.scala @@ -16,11 +16,10 @@ package controllers.model -import play.api.data.validation.ValidationError -import play.api.libs.json.{JsPath, Reads} +import play.api.libs.functional.syntax._ import play.api.libs.json.Reads.{max, min} +import play.api.libs.json.{JsPath, JsonValidationError, Reads} import services.model.UploadSettings -import play.api.libs.functional.syntax._ case class PrepareUploadRequestV2( callbackUrl: String, @@ -51,7 +50,7 @@ object PrepareUploadRequestV2 { (JsPath \ "minimumFileSize").readNullable[Int](min(0)) and (JsPath \ "maximumFileSize").readNullable[Int](min(0) keepAnd max(maxFileSize + 1)) and (JsPath \ "expectedContentType").readNullable[String])(PrepareUploadRequestV2.apply _) - .filter(ValidationError("Maximum file size must be equal or greater than minimum file size"))(request => + .filter(JsonValidationError("Maximum file size must be equal or greater than minimum file size"))(request => request.minimumFileSize.getOrElse(0) <= request.maximumFileSize.getOrElse(maxFileSize)) } diff --git a/conf/app.routes b/conf/app.routes index 6e7f283..deea2da 100644 --- a/conf/app.routes +++ b/conf/app.routes @@ -1,2 +1,2 @@ -POST /initiate @controllers.PrepareUploadController.prepareUploadV1 -POST /v2/initiate @controllers.PrepareUploadController.prepareUploadV2 +POST /initiate controllers.PrepareUploadController.prepareUploadV1 +POST /v2/initiate controllers.PrepareUploadController.prepareUploadV2 diff --git a/conf/prod.routes b/conf/prod.routes index 4f7d3e3..3337678 100644 --- a/conf/prod.routes +++ b/conf/prod.routes @@ -1,4 +1,4 @@ # Add all the application routes to the app.routes file -> /upscan app.Routes -> / health.Routes -GET /admin/metrics @com.kenshoo.play.metrics.MetricsController.metrics \ No newline at end of file +GET /admin/metrics com.kenshoo.play.metrics.MetricsController.metrics \ No newline at end of file diff --git a/project/MicroService.scala b/project/MicroService.scala index 0a6238a..a8b2fef 100644 --- a/project/MicroService.scala +++ b/project/MicroService.scala @@ -27,8 +27,6 @@ trait MicroService { ) lazy val playSettings: Seq[Setting[_]] = Seq.empty - routesGenerator := InjectedRoutesGenerator - lazy val scoverageSettings = { Seq( // Semicolon-separated list of regexs matching classes to exclude @@ -59,8 +57,7 @@ trait MicroService { libraryDependencies ++= appDependencies, parallelExecution in Test := false, fork in Test := false, - retrieveManaged := true, - routesGenerator := StaticRoutesGenerator + retrieveManaged := true ) .settings(inConfig(TemplateTest)(Defaults.testSettings): _*) .configs(IntegrationTest) diff --git a/project/MicroServiceBuild.scala b/project/MicroServiceBuild.scala index b55e34d..3c46ee1 100644 --- a/project/MicroServiceBuild.scala +++ b/project/MicroServiceBuild.scala @@ -12,7 +12,8 @@ private object AppDependencies { import play.core.PlayVersion val compile = Seq( - "uk.gov.hmrc" %% "bootstrap-play-25" % "4.9.0", + "uk.gov.hmrc" %% "bootstrap-play-26" % "1.4.0", + "com.typesafe.play" %% "play-json" % "2.6.14", "com.amazonaws" % "aws-java-sdk-s3" % "1.11.500", "com.typesafe.akka" %% "akka-stream" % "2.5.6" ) @@ -25,15 +26,15 @@ private object AppDependencies { private def commonTestDependencies(scope: String) = Seq( "uk.gov.hmrc" %% "hmrctest" % "3.3.0" % scope, "uk.gov.hmrc" %% "http-verbs-test" % "1.3.0" % scope, - "org.scalatest" %% "scalatest" % "2.2.6" % scope, - "org.pegdown" % "pegdown" % "1.6.0" % scope, "com.typesafe.play" %% "play-test" % PlayVersion.current % scope, + "com.typesafe.play" %% "play-ws" % PlayVersion.current % scope, + "org.scalatest" %% "scalatest" % "3.0.8" % scope, + "org.scalatestplus.play" %% "scalatestplus-play" % "3.1.2" % scope, + "org.pegdown" % "pegdown" % "1.6.0" % scope, "org.mockito" % "mockito-core" % "2.6.2" % scope, "com.github.tomakehurst" % "wiremock" % "2.2.2" % scope, "org.scalamock" %% "scalamock-scalatest-support" % "3.5.0" % scope, - "org.scalatestplus.play" %% "scalatestplus-play" % "2.0.0" % scope, "io.findify" %% "s3mock" % "0.2.4" % scope, - "com.typesafe.play" %% "play-ws" % "2.5.6" % scope, "commons-io" % "commons-io" % "2.6" % scope, "org.scalacheck" %% "scalacheck" % "1.13.4" % scope ) diff --git a/project/plugins.sbt b/project/plugins.sbt index cd7f59f..36ea930 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -5,7 +5,7 @@ resolvers += Resolver.bintrayRepo("hmrc", "releases") addSbtPlugin("com.github.gseitz" % "sbt-release" % "0.8.5") -addSbtPlugin("com.typesafe.play" % "sbt-plugin" % "2.5.19") +addSbtPlugin("com.typesafe.play" % "sbt-plugin" % "2.6.25") addSbtPlugin("uk.gov.hmrc" % "sbt-auto-build" % "1.13.0") diff --git a/test/connectors/s3/S3UploadFormGeneratorSpec.scala b/test/connectors/s3/S3UploadFormGeneratorSpec.scala index eade6c0..6249320 100644 --- a/test/connectors/s3/S3UploadFormGeneratorSpec.scala +++ b/test/connectors/s3/S3UploadFormGeneratorSpec.scala @@ -22,8 +22,8 @@ import java.util.Base64 import connectors.model.{AwsCredentials, ContentLengthRange, UploadParameters} import org.mockito.ArgumentMatchers.any import org.mockito.Mockito.{verify, when} -import org.scalatest.mockito.MockitoSugar import org.scalatest.{GivenWhenThen, Matchers, WordSpec} +import org.scalatestplus.mockito.MockitoSugar import play.api.libs.json.{JsArray, JsValue, Json} class S3UploadFormGeneratorSpec extends WordSpec with GivenWhenThen with Matchers with MockitoSugar { diff --git a/test/controllers/PrepareUploadControllerSpec.scala b/test/controllers/PrepareUploadControllerSpec.scala index de52859..96da278 100644 --- a/test/controllers/PrepareUploadControllerSpec.scala +++ b/test/controllers/PrepareUploadControllerSpec.scala @@ -26,15 +26,15 @@ import org.mockito.ArgumentMatchers.any import org.mockito.Mockito import org.mockito.invocation.InvocationOnMock import org.mockito.stubbing.Answer -import org.scalatest.mockito.MockitoSugar import org.scalatest.{GivenWhenThen, Matchers} +import org.scalatestplus.mockito.MockitoSugar import play.api.http.HeaderNames.USER_AGENT import play.api.http.Status.{BAD_REQUEST, OK} import play.api.libs.json.{JsObject, JsValue, Json} import play.api.mvc.Action import play.api.mvc.Results.Ok import play.api.test.Helpers.contentAsString -import play.api.test.{FakeRequest, Helpers} +import play.api.test.{FakeRequest, Helpers, StubControllerComponentsFactory} import services.PrepareUploadService import services.model.UploadSettings import uk.gov.hmrc.play.test.UnitSpec @@ -43,7 +43,7 @@ import scala.concurrent.Future import scala.concurrent.duration._ import scala.language.postfixOps -class PrepareUploadControllerSpec extends UnitSpec with Matchers with GivenWhenThen with MockitoSugar { +class PrepareUploadControllerSpec extends UnitSpec with StubControllerComponentsFactory with Matchers with GivenWhenThen with MockitoSugar { implicit val actorSystem: ActorSystem = ActorSystem() @@ -79,7 +79,7 @@ class PrepareUploadControllerSpec extends UnitSpec with Matchers with GivenWhenT Mockito.when(config.allowedCallbackProtocols).thenReturn(List("https")) "build and return upload URL if valid request with all data" in { - val controller = new PrepareUploadController(prepareUploadService, config, clock) + val controller = new PrepareUploadController(prepareUploadService, config, clock, stubControllerComponents()) Given("there is a valid upload request with all data") @@ -117,7 +117,7 @@ class PrepareUploadControllerSpec extends UnitSpec with Matchers with GivenWhenT } "build and return upload URL if valid request with redirect on success url" in { - val controller = new PrepareUploadController(prepareUploadService, config, clock) + val controller = new PrepareUploadController(prepareUploadService, config, clock, stubControllerComponents()) Given("there is a valid upload request with all data") @@ -158,7 +158,7 @@ class PrepareUploadControllerSpec extends UnitSpec with Matchers with GivenWhenT } "build and return upload URL if valid request with minimal data including session id and request id" in { - val controller = new PrepareUploadController(prepareUploadService, config, clock) + val controller = new PrepareUploadController(prepareUploadService, config, clock, stubControllerComponents()) Given("there is a valid upload request with minimal data") @@ -188,7 +188,7 @@ class PrepareUploadControllerSpec extends UnitSpec with Matchers with GivenWhenT } "build and return upload URL if valid request with minimal data excluding session id and request id" in { - val controller = new PrepareUploadController(prepareUploadService, config, clock) + val controller = new PrepareUploadController(prepareUploadService, config, clock, stubControllerComponents()) Given("there is a valid upload request with minimal data") @@ -214,7 +214,7 @@ class PrepareUploadControllerSpec extends UnitSpec with Matchers with GivenWhenT } "return a bad request error if invalid request - wrong structure" in { - val controller = new PrepareUploadController(prepareUploadService, config, clock) + val controller = new PrepareUploadController(prepareUploadService, config, clock, stubControllerComponents()) Given("there is an invalid upload request") @@ -235,7 +235,7 @@ class PrepareUploadControllerSpec extends UnitSpec with Matchers with GivenWhenT } "return a bad request error if invalid request - incorrect maximum file size " in { - val controller = new PrepareUploadController(prepareUploadService, config, clock) + val controller = new PrepareUploadController(prepareUploadService, config, clock, stubControllerComponents()) Given("there is an invalid upload request") @@ -255,7 +255,7 @@ class PrepareUploadControllerSpec extends UnitSpec with Matchers with GivenWhenT } "allow https callback urls" in { - val controller = new PrepareUploadController(prepareUploadService, config, clock) + val controller = new PrepareUploadController(prepareUploadService, config, clock, stubControllerComponents()) val result = controller.withAllowedCallbackProtocol("https://my.callback.url") { Future.successful(Ok) @@ -265,7 +265,7 @@ class PrepareUploadControllerSpec extends UnitSpec with Matchers with GivenWhenT } "disallow http callback urls" in { - val controller = new PrepareUploadController(prepareUploadService, config, clock) + val controller = new PrepareUploadController(prepareUploadService, config, clock, stubControllerComponents()) val result = controller.withAllowedCallbackProtocol("http://my.callback.url") { Future.failed(new RuntimeException("This block should not have been invoked.")) @@ -276,7 +276,7 @@ class PrepareUploadControllerSpec extends UnitSpec with Matchers with GivenWhenT } "disallow invalidly formatted callback urls" in { - val controller = new PrepareUploadController(prepareUploadService, config, clock) + val controller = new PrepareUploadController(prepareUploadService, config, clock, stubControllerComponents()) val result = controller.withAllowedCallbackProtocol("123") { Future.failed(new RuntimeException("This block should not have been invoked.")) diff --git a/test/utils/UserAgentFilterSpec.scala b/test/utils/UserAgentFilterSpec.scala index d0919a3..0141af4 100644 --- a/test/utils/UserAgentFilterSpec.scala +++ b/test/utils/UserAgentFilterSpec.scala @@ -21,8 +21,8 @@ import org.scalatest.{GivenWhenThen, Matchers} import play.api.http.Status.{BAD_REQUEST, OK} import play.api.mvc.Results._ import play.api.mvc.{Request, Result} +import play.api.test.FakeRequest import play.api.test.Helpers.contentAsString -import play.api.test.{FakeRequest, Helpers} import play.mvc.Http.HeaderNames.USER_AGENT import uk.gov.hmrc.play.test.UnitSpec From 9a9caead6e7f01758561868f63f81c81c02fdbfa Mon Sep 17 00:00:00 2001 From: Nigel Perkins Date: Thu, 27 Feb 2020 09:51:04 +0000 Subject: [PATCH 39/96] BDOG-643: update to SBT 1.3 --- build.sbt | 35 ++++++++ ...rviceBuild.scala => AppDependencies.scala} | 12 +-- project/MicroService.scala | 90 ------------------- project/build.properties | 2 +- project/plugins.sbt | 18 ++-- 5 files changed, 47 insertions(+), 110 deletions(-) create mode 100644 build.sbt rename project/{MicroServiceBuild.scala => AppDependencies.scala} (91%) delete mode 100644 project/MicroService.scala diff --git a/build.sbt b/build.sbt new file mode 100644 index 0000000..2e25176 --- /dev/null +++ b/build.sbt @@ -0,0 +1,35 @@ +import play.sbt.PlayImport.PlayKeys.playDefaultPort +import sbt.Keys._ +import sbt._ +import scoverage.ScoverageKeys +import uk.gov.hmrc.DefaultBuildSettings.integrationTestSettings +import uk.gov.hmrc.sbtdistributables.SbtDistributablesPlugin + + +val appName = "upscan-initiate" + +lazy val scoverageSettings = Seq( + // Semicolon-separated list of regexs matching classes to exclude + ScoverageKeys.coverageExcludedPackages := ";Reverse.*;.*AuthService.*;models/.data/..*;view.*", + ScoverageKeys.coverageExcludedFiles := + ".*/frontendGlobal.*;.*/frontendAppConfig.*;.*/frontendWiring.*;.*/views/.*_template.*;.*/govuk_wrapper.*;.*/routes_routing.*;.*/BuildInfo.*", + // Minimum is deliberately low to avoid failures initially - please increase as we add more coverage + ScoverageKeys.coverageMinimum := 25, + ScoverageKeys.coverageFailOnMinimum := false, + ScoverageKeys.coverageHighlighting := true +) + +lazy val microservice = Project(appName, file(".")) + .enablePlugins(Seq(play.sbt.PlayScala, SbtAutoBuildPlugin, SbtGitVersioning, SbtDistributablesPlugin, SbtArtifactory): _*) + .disablePlugins(JUnitXmlReportPlugin) //Required to prevent https://github.com/scalatest/scalatest/issues/1427 + .settings(scoverageSettings: _*) + .settings(majorVersion := 0) + .settings(SbtDistributablesPlugin.publishingSettings: _*) + .settings(playDefaultPort := 9571) + .settings(libraryDependencies ++= AppDependencies()) + .settings(resolvers += Resolver.jcenterRepo) + .settings(scalacOptions := Seq("-target:jvm-1.8")) + .settings(scalaVersion := "2.11.12") + .settings(Test / parallelExecution := false) + .configs(IntegrationTest) + .settings(integrationTestSettings(): _*) \ No newline at end of file diff --git a/project/MicroServiceBuild.scala b/project/AppDependencies.scala similarity index 91% rename from project/MicroServiceBuild.scala rename to project/AppDependencies.scala index 3c46ee1..26ae650 100644 --- a/project/MicroServiceBuild.scala +++ b/project/AppDependencies.scala @@ -1,14 +1,6 @@ import sbt._ -object MicroServiceBuild extends Build with MicroService { - - val appName = "upscan-initiate" - - override lazy val appDependencies: Seq[ModuleID] = AppDependencies() - -} - -private object AppDependencies { +object AppDependencies { import play.core.PlayVersion val compile = Seq( @@ -57,4 +49,4 @@ private object AppDependencies { } def apply() = compile ++ Test() ++ IntegrationTest() -} +} \ No newline at end of file diff --git a/project/MicroService.scala b/project/MicroService.scala deleted file mode 100644 index a8b2fef..0000000 --- a/project/MicroService.scala +++ /dev/null @@ -1,90 +0,0 @@ -import play.routes.compiler.{InjectedRoutesGenerator, StaticRoutesGenerator} -import play.sbt.PlayImport.PlayKeys -import play.sbt.routes.RoutesKeys.routesGenerator -import sbt.Keys._ -import sbt.Tests.{Group, SubProcess} -import sbt._ -import uk.gov.hmrc.sbtdistributables.SbtDistributablesPlugin -import uk.gov.hmrc.versioning.SbtGitVersioning -import uk.gov.hmrc.versioning.SbtGitVersioning.autoImport.majorVersion - -trait MicroService { - - import TestPhases._ - import scoverage.ScoverageKeys - import uk.gov.hmrc.DefaultBuildSettings.{addTestReportOption, defaultSettings, scalaSettings, targetJvm} - import uk.gov.hmrc._ - - val appName: String - - lazy val appDependencies: Seq[ModuleID] = ??? - lazy val plugins: Seq[Plugins] = Seq( - play.sbt.PlayScala, - SbtAutoBuildPlugin, - SbtGitVersioning, - SbtDistributablesPlugin, - SbtArtifactory - ) - lazy val playSettings: Seq[Setting[_]] = Seq.empty - - lazy val scoverageSettings = { - Seq( - // Semicolon-separated list of regexs matching classes to exclude - ScoverageKeys.coverageExcludedPackages := ";Reverse.*;.*AuthService.*;models/.data/..*;view.*", - ScoverageKeys.coverageExcludedFiles := - ".*/frontendGlobal.*;.*/frontendAppConfig.*;.*/frontendWiring.*;.*/views/.*_template.*;.*/govuk_wrapper.*;.*/routes_routing.*;.*/BuildInfo.*", - // Minimum is deliberately low to avoid failures initially - please increase as we add more coverage - ScoverageKeys.coverageMinimum := 25, - ScoverageKeys.coverageFailOnMinimum := false, - ScoverageKeys.coverageHighlighting := true, - parallelExecution in Test := false - ) - } - - lazy val microservice = Project(appName, file(".")) - .enablePlugins(plugins: _*) - .settings(playSettings: _*) - .settings(scoverageSettings: _*) - .settings(playSettings: _*) - .settings(scalaSettings: _*) - .settings(defaultSettings(): _*) - .settings(SbtDistributablesPlugin.publishingSettings: _*) - .settings(majorVersion := 0) - .settings( - PlayKeys.playDefaultPort := 9571, - targetJvm := "jvm-1.8", - scalaVersion := "2.11.12", - libraryDependencies ++= appDependencies, - parallelExecution in Test := false, - fork in Test := false, - retrieveManaged := true - ) - .settings(inConfig(TemplateTest)(Defaults.testSettings): _*) - .configs(IntegrationTest) - .settings(inConfig(TemplateItTest)(Defaults.itSettings): _*) - .settings( - Keys.fork in IntegrationTest := false, - unmanagedSourceDirectories in IntegrationTest <<= (baseDirectory in IntegrationTest)(base => Seq(base / "it")), - addTestReportOption(IntegrationTest, "int-test-reports"), - testGrouping in IntegrationTest := oneForkedJvmPerTest((definedTests in IntegrationTest).value), - parallelExecution in IntegrationTest := false - ) - .settings( - resolvers += Resolver.bintrayRepo("hmrc", "releases"), - resolvers += Resolver.jcenterRepo - ) -} - -private object TestPhases { - - val allPhases = "tt->test;test->test;test->compile;compile->compile" - val allItPhases = "tit->it;it->it;it->compile;compile->compile" - - lazy val TemplateTest = config("tt") extend Test - lazy val TemplateItTest = config("tit") extend IntegrationTest - - def oneForkedJvmPerTest(tests: Seq[TestDefinition]) = - tests map { test => - Group(test.name, Seq(test), SubProcess(ForkOptions(runJVMOptions = Seq("-Dtest.name=" + test.name)))) - } -} diff --git a/project/build.properties b/project/build.properties index 133a8f1..35b49ae 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1 +1 @@ -sbt.version=0.13.17 +sbt.version=1.3.8 \ No newline at end of file diff --git a/project/plugins.sbt b/project/plugins.sbt index 36ea930..f0dd90a 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -3,22 +3,22 @@ import sbt._ resolvers += Resolver.bintrayIvyRepo("hmrc", "sbt-plugin-releases") resolvers += Resolver.bintrayRepo("hmrc", "releases") -addSbtPlugin("com.github.gseitz" % "sbt-release" % "0.8.5") +addSbtPlugin("com.github.gseitz" % "sbt-release" % "1.0.13") addSbtPlugin("com.typesafe.play" % "sbt-plugin" % "2.6.25") -addSbtPlugin("uk.gov.hmrc" % "sbt-auto-build" % "1.13.0") +addSbtPlugin("uk.gov.hmrc" % "sbt-auto-build" % "2.6.0") -addSbtPlugin("uk.gov.hmrc" % "sbt-settings" % "3.9.0") +addSbtPlugin("uk.gov.hmrc" % "sbt-settings" % "4.0.0") -addSbtPlugin("uk.gov.hmrc" % "sbt-artifactory" % "0.14.0") +addSbtPlugin("uk.gov.hmrc" % "sbt-artifactory" % "1.0.0") -addSbtPlugin("uk.gov.hmrc" % "sbt-git-versioning" % "1.15.0") +addSbtPlugin("uk.gov.hmrc" % "sbt-git-versioning" % "2.1.0") -addSbtPlugin("uk.gov.hmrc" % "sbt-distributables" % "1.2.0") +addSbtPlugin("uk.gov.hmrc" % "sbt-distributables" % "2.0.0") -addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.3.5") +addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.6.1") -addSbtPlugin("org.scalastyle" %% "scalastyle-sbt-plugin" % "0.8.0") +addSbtPlugin("org.scalastyle" %% "scalastyle-sbt-plugin" % "1.0.0") -addSbtPlugin("net.virtual-void" % "sbt-dependency-graph" % "0.8.2") +addSbtPlugin("net.virtual-void" % "sbt-dependency-graph" % "0.9.2") From 526d4855c108486a716f0312504932d9ed246153 Mon Sep 17 00:00:00 2001 From: Nigel Perkins Date: Thu, 27 Feb 2020 15:27:15 +0000 Subject: [PATCH 40/96] BDOG-643: uplift dependencies --- project/AppDependencies.scala | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/project/AppDependencies.scala b/project/AppDependencies.scala index 26ae650..8e7f883 100644 --- a/project/AppDependencies.scala +++ b/project/AppDependencies.scala @@ -6,8 +6,7 @@ object AppDependencies { val compile = Seq( "uk.gov.hmrc" %% "bootstrap-play-26" % "1.4.0", "com.typesafe.play" %% "play-json" % "2.6.14", - "com.amazonaws" % "aws-java-sdk-s3" % "1.11.500", - "com.typesafe.akka" %% "akka-stream" % "2.5.6" + "com.amazonaws" % "aws-java-sdk-s3" % "1.11.699" ) trait TestDependencies { @@ -16,19 +15,15 @@ object AppDependencies { } private def commonTestDependencies(scope: String) = Seq( - "uk.gov.hmrc" %% "hmrctest" % "3.3.0" % scope, - "uk.gov.hmrc" %% "http-verbs-test" % "1.3.0" % scope, + "uk.gov.hmrc" %% "hmrctest" % "3.9.0-play-26" % scope, "com.typesafe.play" %% "play-test" % PlayVersion.current % scope, "com.typesafe.play" %% "play-ws" % PlayVersion.current % scope, "org.scalatest" %% "scalatest" % "3.0.8" % scope, "org.scalatestplus.play" %% "scalatestplus-play" % "3.1.2" % scope, - "org.pegdown" % "pegdown" % "1.6.0" % scope, - "org.mockito" % "mockito-core" % "2.6.2" % scope, - "com.github.tomakehurst" % "wiremock" % "2.2.2" % scope, - "org.scalamock" %% "scalamock-scalatest-support" % "3.5.0" % scope, - "io.findify" %% "s3mock" % "0.2.4" % scope, - "commons-io" % "commons-io" % "2.6" % scope, - "org.scalacheck" %% "scalacheck" % "1.13.4" % scope + "org.pegdown" % "pegdown" % "1.6.0" % scope, + "org.mockito" % "mockito-core" % "3.3.0" % scope, + "com.github.tomakehurst" % "wiremock" % "2.26.0" % scope, + "commons-io" % "commons-io" % "2.6" % scope ) object Test { From 13153e2d88b0c32e9fb7cd18baf70291765e9819 Mon Sep 17 00:00:00 2001 From: Nigel Perkins Date: Thu, 27 Feb 2020 15:41:02 +0000 Subject: [PATCH 41/96] BDOG-643: update to Scala 2.12 --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index 2e25176..1fefb3a 100644 --- a/build.sbt +++ b/build.sbt @@ -29,7 +29,7 @@ lazy val microservice = Project(appName, file(".")) .settings(libraryDependencies ++= AppDependencies()) .settings(resolvers += Resolver.jcenterRepo) .settings(scalacOptions := Seq("-target:jvm-1.8")) - .settings(scalaVersion := "2.11.12") + .settings(scalaVersion := "2.12.10") .settings(Test / parallelExecution := false) .configs(IntegrationTest) .settings(integrationTestSettings(): _*) \ No newline at end of file From dff3e03f9f842945732733392bbedd92b98e3d8e Mon Sep 17 00:00:00 2001 From: Nigel Perkins Date: Thu, 27 Feb 2020 16:49:36 +0000 Subject: [PATCH 42/96] BDOG-643: eliminate deprecation warnings --- app/config/ServiceConfiguration.scala | 22 ++++++++++++---------- build.sbt | 2 +- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/app/config/ServiceConfiguration.scala b/app/config/ServiceConfiguration.scala index 70aecb5..74756b5 100644 --- a/app/config/ServiceConfiguration.scala +++ b/app/config/ServiceConfiguration.scala @@ -38,25 +38,27 @@ trait ServiceConfiguration { class PlayBasedServiceConfiguration @Inject()(configuration: Configuration) extends ServiceConfiguration { - override def region: String = getRequired(configuration.getString(_), "aws.s3.region") + override def region: String = getRequired(configuration.getOptional[String](_), "aws.s3.region") - override def uploadProxyUrl: String = getRequired(configuration.getString(_), "uploadProxy.url") + override def uploadProxyUrl: String = getRequired(configuration.getOptional[String](_), "uploadProxy.url") - override def inboundBucketName: String = getRequired(configuration.getString(_), "aws.s3.bucket.inbound") + override def inboundBucketName: String = getRequired(configuration.getOptional[String](_), "aws.s3.bucket.inbound") - override def fileExpirationPeriod: Duration = - Duration.ofMillis(getRequired(configuration.getMilliseconds, "aws.s3.upload.link.validity.duration")) + override def fileExpirationPeriod: Duration = { + val readAsMillis: String => Option[Long] = configuration.getOptional[scala.concurrent.duration.Duration](_).map(_.toMillis) + Duration.ofMillis(getRequired(readAsMillis, "aws.s3.upload.link.validity.duration")) + } - override def accessKeyId: String = getRequired(configuration.getString(_), "aws.accessKeyId") + override def accessKeyId: String = getRequired(configuration.getOptional[String](_), "aws.accessKeyId") - override def secretAccessKey: String = getRequired(configuration.getString(_), "aws.secretAccessKey") + override def secretAccessKey: String = getRequired(configuration.getOptional[String](_), "aws.secretAccessKey") - override def sessionToken: Option[String] = configuration.getString("aws.sessionToken") + override def sessionToken: Option[String] = configuration.getOptional[String]("aws.sessionToken") - override def globalFileSizeLimit: Int = getRequired(configuration.getInt, "global.file.size.limit") + override def globalFileSizeLimit: Int = getRequired(configuration.getOptional[Int](_), "global.file.size.limit") override def allowedCallbackProtocols: Seq[String] = - commaSeparatedList(configuration.getString("callbackValidation.allowedProtocols")) + commaSeparatedList(configuration.getOptional[String]("callbackValidation.allowedProtocols")) private def getRequired[T](read: String => Option[T], path: String): T = read(path).getOrElse(throw new ConfigException.Missing(path)) diff --git a/build.sbt b/build.sbt index 1fefb3a..0a80553 100644 --- a/build.sbt +++ b/build.sbt @@ -28,7 +28,7 @@ lazy val microservice = Project(appName, file(".")) .settings(playDefaultPort := 9571) .settings(libraryDependencies ++= AppDependencies()) .settings(resolvers += Resolver.jcenterRepo) - .settings(scalacOptions := Seq("-target:jvm-1.8")) + .settings(scalacOptions += "-target:jvm-1.8") .settings(scalaVersion := "2.12.10") .settings(Test / parallelExecution := false) .configs(IntegrationTest) From 4330243f6950006f35667f1e442dde3377f3f318 Mon Sep 17 00:00:00 2001 From: Nigel Perkins Date: Mon, 2 Mar 2020 11:30:44 +0000 Subject: [PATCH 43/96] BDOG-643: changes in response to review comments --- project/AppDependencies.scala | 46 +++++++++-------------------------- 1 file changed, 11 insertions(+), 35 deletions(-) diff --git a/project/AppDependencies.scala b/project/AppDependencies.scala index 8e7f883..161d0bd 100644 --- a/project/AppDependencies.scala +++ b/project/AppDependencies.scala @@ -3,45 +3,21 @@ import sbt._ object AppDependencies { import play.core.PlayVersion - val compile = Seq( + private val compile = Seq( "uk.gov.hmrc" %% "bootstrap-play-26" % "1.4.0", "com.typesafe.play" %% "play-json" % "2.6.14", - "com.amazonaws" % "aws-java-sdk-s3" % "1.11.699" + "com.amazonaws" % "aws-java-sdk-s3" % "1.11.699", + "org.apache.commons" % "commons-lang3" % "3.9" ) - trait TestDependencies { - lazy val scope: String = "test" - lazy val test: Seq[ModuleID] = ??? - } - - private def commonTestDependencies(scope: String) = Seq( - "uk.gov.hmrc" %% "hmrctest" % "3.9.0-play-26" % scope, - "com.typesafe.play" %% "play-test" % PlayVersion.current % scope, - "com.typesafe.play" %% "play-ws" % PlayVersion.current % scope, - "org.scalatest" %% "scalatest" % "3.0.8" % scope, - "org.scalatestplus.play" %% "scalatestplus-play" % "3.1.2" % scope, - "org.pegdown" % "pegdown" % "1.6.0" % scope, - "org.mockito" % "mockito-core" % "3.3.0" % scope, - "com.github.tomakehurst" % "wiremock" % "2.26.0" % scope, - "commons-io" % "commons-io" % "2.6" % scope + private val test = Seq( + "uk.gov.hmrc" %% "hmrctest" % "3.9.0-play-26" % s"$Test,$IntegrationTest", + "com.typesafe.play" %% "play-test" % PlayVersion.current % s"$Test,$IntegrationTest", + "org.scalatest" %% "scalatest" % "3.0.8" % s"$Test,$IntegrationTest", + "org.scalatestplus.play" %% "scalatestplus-play" % "3.1.2" % s"$Test,$IntegrationTest", + "org.pegdown" % "pegdown" % "1.6.0" % s"$Test,$IntegrationTest", + "org.mockito" % "mockito-core" % "3.3.0" % s"$Test,$IntegrationTest" ) - object Test { - def apply() = - new TestDependencies { - override lazy val test = commonTestDependencies(scope) - }.test - } - - object IntegrationTest { - def apply() = - new TestDependencies { - - override lazy val scope: String = "it" - - override lazy val test = commonTestDependencies(scope) - }.test - } - - def apply() = compile ++ Test() ++ IntegrationTest() + def apply(): Seq[ModuleID] = compile ++ test } \ No newline at end of file From bea67af50165ff48bbdf096a686ee89ccd549f75 Mon Sep 17 00:00:00 2001 From: Nigel Perkins Date: Tue, 3 Mar 2020 14:52:01 +0000 Subject: [PATCH 44/96] BDOG-644: update documentation following removal of RunMode prefixed config keys --- README.md | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 79d7016..a4b0809 100644 --- a/README.md +++ b/README.md @@ -302,15 +302,13 @@ The downloadUrl will expire after 1 day by default. This can be configured on a For example, to limit to 1 hour add the following configuration (substituting the appropriate `User-Agent` service identifier) to [upscan-notify.conf](https://github.com/hmrc/app-config-base/blob/master/upscan-notify.conf): ``` -Prod { - upscan { - { - aws { - s3 { - urlExpirationPeriod = 1.hour - } - } +consuming-services { + { + aws { + s3 { + urlExpirationPeriod = 1.hour } + } } } ``` From ba5ab7d9384441183c5952dbf1c6b295b4f9d7d3 Mon Sep 17 00:00:00 2001 From: Nigel Perkins Date: Wed, 25 Mar 2020 15:41:45 +0000 Subject: [PATCH 45/96] BDOG-643: cleanup development config --- conf/application.conf | 95 ++++++------------------------------------- 1 file changed, 13 insertions(+), 82 deletions(-) diff --git a/conf/application.conf b/conf/application.conf index 8431c8d..eb8e6ac 100644 --- a/conf/application.conf +++ b/conf/application.conf @@ -14,14 +14,12 @@ # This is the main configuration file for the application. # ~~~~~ +include "backend.conf" appName=upscan-initiate play.modules.enabled += "com.kenshoo.play.metrics.PlayModule" -#Required for object based play2.5 applications -play.http.requestHandler = "play.api.http.GlobalSettingsHttpRequestHandler" - # An ApplicationLoader that uses Guice to bootstrap the application. play.application.loader = "uk.gov.hmrc.play.bootstrap.ApplicationLoader" @@ -47,23 +45,16 @@ play.http.filters = "uk.gov.hmrc.play.bootstrap.filters.MicroserviceFilters" # Json error handler play.http.errorHandler = "uk.gov.hmrc.play.bootstrap.http.JsonErrorHandler" -# Session Timeout -# ~~~~ -# The default session timeout for the app is 15 minutes (900seconds). -# Updating this is the responsibility of the app - it must issue a new cookie with each request or the session will -# timeout 15 minutes after login (regardless of user activity). -# session.maxAge=900 - # Secret key # ~~~~~ # The secret key is used to secure cryptographics functions. # If you deploy your application to several instances be sure to use the same key! -application.secret="mKW4mMngaPz1UVOrEkgkJrZT9kloW9Neva5wUFDUq84n9xlz42lQMGrtlrQ2bMF4" +play.crypto.secret="mKW4mMngaPz1UVOrEkgkJrZT9kloW9Neva5wUFDUq84n9xlz42lQMGrtlrQ2bMF4" +play.http.secret.key="mKW4mMngaPz1UVOrEkgkJrZT9kloW9Neva5wUFDUq84n9xlz42lQMGrtlrQ2bMF4" # Session configuration # ~~~~~ application.session.httpOnly=false - application.session.secure=false # The application languages @@ -83,28 +74,10 @@ application.langs="en" application.router=prod.Routes -# Controller -# ~~~~~ -# By default all controllers will have authorisation, logging and -# auditing (transaction monitoring) enabled. -# The below controllers are the default exceptions to this rule. - -controllers { - com.kenshoo.play.metrics.MetricsController = { - needsAuth = false - needsLogging = false - needsAuditing = false - } -} - # Metrics plugin settings - graphite reporting is configured on a per env basis metrics { - name = ${appName} - rateUnit = SECONDS - durationUnit = SECONDS - showSamples = true - jvm = true - enabled = true + name = ${appName} + enabled = true } uploadProxy.url = "ENTER UPLOAD PROXY URL" @@ -128,55 +101,13 @@ callbackValidation.allowedProtocols = "https" global.file.size.limit = 104857600 -# Microservice specific config - -Test { - auditing { - enabled = true - traceRequests = true - consumer { - baseUri { - host = localhost - port = 11111 - } +auditing { + enabled = true + traceRequests = true + consumer { + baseUri { + host = localhost + port = 8100 } } - - microservice { - metrics { - graphite { - enabled = false - } - gauges { - interval = 1 second - } - } - } -} - -Dev { - auditing { - enabled = true - traceRequests = true - consumer { - baseUri { - host = localhost - port = 8100 - } - } - } - - microservice { - metrics { - graphite { - enabled = false - } - gauges { - interval = 1 second - } - } - - } - -} - +} \ No newline at end of file From 99dbda55f0049048ce3d472b0857d007ae575381 Mon Sep 17 00:00:00 2001 From: chotai Date: Wed, 1 Apr 2020 15:17:51 +0100 Subject: [PATCH 46/96] BDOG-768 Update out of date docs --- README.md | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/README.md b/README.md index a4b0809..dfbad36 100644 --- a/README.md +++ b/README.md @@ -411,23 +411,6 @@ Prerequisites: - AWS user with correct roles & MFA enabled - AWS credential configuration set up according to this document [aws-credential-configuration](https://github.com/hmrc/aws-users), with the credentials below: -``` -[webops-users] -mfa_serial = arn:aws:iam::638924580364:mfa/YOUR_AWS_USER_NAME -aws_access_key_id = YOUR_ACCESS_KEY_HERE -aws_secret_access_key = YOUR_SECRET_KEY_HERE -output = json -region = eu-west-2 - -[upscan-labs-engineer] -source_profile = webops-users -role_arn = arn:aws:iam::063874132475:role/RoleServiceUpscanEngineer -mfa_serial = arn:aws:iam::638924580364:mfa/YOUR_AWS_USER_NAME -aws_access_key_id = YOUR_ACCESS_KEY_HERE -aws_secret_access_key = YOUR_SECRET_KEY_HERE -output = json -region = eu-west-2 -``` - Python 2.7 installed - Botocore and awscli python modules installed locally: - Linux: From 0c89d5ac65b0eaf3f15c9943148ce41d41fa1330 Mon Sep 17 00:00:00 2001 From: Sunny Chotai Date: Sun, 5 Apr 2020 10:56:55 +0100 Subject: [PATCH 47/96] Update readme --- README.md | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/README.md b/README.md index dfbad36..5e0642e 100644 --- a/README.md +++ b/README.md @@ -251,10 +251,7 @@ In order to upload the file, the following form is sent as the body of a POST re ``` -This POST request can be sent programmatically from backend code or the submit of a web form. - -Whichever way the form is sent, remember: - +- You must make this request client side. Making this request server side defeats the objective of Upscan, which is to virus scan files before they are allowed on MDTP. - You must use multipart encoding (`multipart/form-data`) NOT `application/x-www-form-urlencoded`. If you use` application/x-www-form-urlencoded`, AWS will return a response from which this error is not clear. - The 'file' field must be the last field in the submitted form. From 88e985ed88d26e5be425687e37bf838aef974ae5 Mon Sep 17 00:00:00 2001 From: Nigel Perkins Date: Tue, 28 Apr 2020 15:17:19 +0100 Subject: [PATCH 48/96] BDOG-760: update to Play 2.7 --- app/controllers/PrepareUploadController.scala | 10 +++++----- app/controllers/model/Reference.scala | 1 - app/services/PrepareUploadService.scala | 6 +++--- app/utils/UserAgentFilter.scala | 6 +++--- project/AppDependencies.scala | 12 ++++++------ project/plugins.sbt | 8 ++++---- test/utils/UserAgentFilterSpec.scala | 3 ++- 7 files changed, 23 insertions(+), 23 deletions(-) diff --git a/app/controllers/PrepareUploadController.scala b/app/controllers/PrepareUploadController.scala index e63173b..39d24a2 100644 --- a/app/controllers/PrepareUploadController.scala +++ b/app/controllers/PrepareUploadController.scala @@ -22,7 +22,7 @@ import java.time.{Clock, Instant} import config.ServiceConfiguration import controllers.model.{PrepareUpload, PrepareUploadRequestV1, PrepareUploadRequestV2, PreparedUploadResponse} import javax.inject.{Inject, Singleton} -import play.api.Logger +import play.api.Logging import play.api.libs.json._ import play.api.mvc.{Action, ControllerComponents, Result} import services.PrepareUploadService @@ -37,7 +37,7 @@ class PrepareUploadController @Inject()( prepareUploadService: PrepareUploadService, configuration: ServiceConfiguration, clock: Clock, - controllerComponents: ControllerComponents) extends BackendController(controllerComponents) with UserAgentFilter { + controllerComponents: ControllerComponents) extends BackendController(controllerComponents) with UserAgentFilter with Logging { implicit val prepareUploadRequestReads: Reads[PrepareUploadRequestV1] = PrepareUploadRequestV1.reads(prepareUploadService.globalFileSizeLimit) @@ -63,7 +63,7 @@ class PrepareUploadController @Inject()( requireUserAgent[JsValue] { (_, consumingService) => withJsonBody[T] { prepareUploadRequest: T => withAllowedCallbackProtocol(prepareUploadRequest.callbackUrl) { - Logger.debug(s"Processing request: [$prepareUploadRequest].") + logger.debug(s"Processing request: [$prepareUploadRequest].") val sessionId = hc(request).sessionId.map(_.value).getOrElse("n/a") val requestId = hc(request).requestId.map(_.value).getOrElse("n/a") @@ -95,13 +95,13 @@ class PrepareUploadController @Inject()( isAllowedCallbackProtocol match { case Success(true) => block case Success(false) => - Logger.warn(s"Invalid callback url protocol: [$callbackUrl].") + logger.warn(s"Invalid callback url protocol: [$callbackUrl].") Future.successful(BadRequest( s"Invalid callback url protocol: [$callbackUrl]. Protocol must be in: [${allowedCallbackProtocols.mkString(",")}].")) case Failure(e) => - Logger.warn(s"Invalid callback url format: [$callbackUrl].") + logger.warn(s"Invalid callback url format: [$callbackUrl].") Future.successful(BadRequest(s"Invalid callback url format: [$callbackUrl]. [${e.getMessage}]")) } diff --git a/app/controllers/model/Reference.scala b/app/controllers/model/Reference.scala index 27e440f..ed14653 100644 --- a/app/controllers/model/Reference.scala +++ b/app/controllers/model/Reference.scala @@ -18,7 +18,6 @@ package controllers.model import java.util.UUID.randomUUID -import play.api.libs.functional.syntax._ import play.api.libs.json.Writes case class Reference(value: String) extends AnyVal { diff --git a/app/services/PrepareUploadService.scala b/app/services/PrepareUploadService.scala index 545b952..18a7b79 100644 --- a/app/services/PrepareUploadService.scala +++ b/app/services/PrepareUploadService.scala @@ -24,14 +24,14 @@ import connectors.model.{ContentLengthRange, UploadFormGenerator, UploadParamete import controllers.model.{PreparedUploadResponse, Reference, UploadFormTemplate} import javax.inject.{Inject, Singleton} import org.slf4j.MDC -import play.api.Logger +import play.api.Logging import services.model.UploadSettings @Singleton class PrepareUploadService @Inject()( postSigner: UploadFormGenerator, configuration: ServiceConfiguration, - metrics: Metrics) { + metrics: Metrics) extends Logging { def prepareUpload( settings: UploadSettings, @@ -57,7 +57,7 @@ class PrepareUploadService @Inject()( try { MDC.put("file-reference", reference.toString) - Logger.info( + logger.info( s"Generated file-reference: [$reference], for settings: [$settings], with expiration at: [$expiration].") metrics.defaultRegistry.counter("uploadInitiated").inc() diff --git a/app/utils/UserAgentFilter.scala b/app/utils/UserAgentFilter.scala index b30a24b..02d75f0 100644 --- a/app/utils/UserAgentFilter.scala +++ b/app/utils/UserAgentFilter.scala @@ -16,14 +16,14 @@ package utils -import play.api.Logger +import play.api.Logging import play.api.http.HeaderNames.USER_AGENT import play.api.mvc.Results.BadRequest import play.api.mvc.{Request, Result} import scala.concurrent.Future -trait UserAgentFilter { +trait UserAgentFilter { this: Logging => /* * We require the user agent to be set with the name of the client service. */ @@ -31,7 +31,7 @@ trait UserAgentFilter { request.headers.get(USER_AGENT).fold(onMissingUserAgent())(block(request, _)) private def onMissingUserAgent(): Future[Result] = { - Logger.warn(s"No $USER_AGENT Request Header found - unable to identify client service") + logger.warn(s"No $USER_AGENT Request Header found - unable to identify client service") Future.successful(BadRequest(s"Missing $USER_AGENT Header")) } } diff --git a/project/AppDependencies.scala b/project/AppDependencies.scala index 161d0bd..8615ee6 100644 --- a/project/AppDependencies.scala +++ b/project/AppDependencies.scala @@ -4,19 +4,19 @@ object AppDependencies { import play.core.PlayVersion private val compile = Seq( - "uk.gov.hmrc" %% "bootstrap-play-26" % "1.4.0", - "com.typesafe.play" %% "play-json" % "2.6.14", - "com.amazonaws" % "aws-java-sdk-s3" % "1.11.699", - "org.apache.commons" % "commons-lang3" % "3.9" + "uk.gov.hmrc" %% "bootstrap-backend-play-27" % "2.4.0", + "com.typesafe.play" %% "play-json" % PlayVersion.current, + "com.amazonaws" % "aws-java-sdk-s3" % "1.11.769", + "org.apache.commons" % "commons-lang3" % "3.10" ) private val test = Seq( "uk.gov.hmrc" %% "hmrctest" % "3.9.0-play-26" % s"$Test,$IntegrationTest", "com.typesafe.play" %% "play-test" % PlayVersion.current % s"$Test,$IntegrationTest", "org.scalatest" %% "scalatest" % "3.0.8" % s"$Test,$IntegrationTest", - "org.scalatestplus.play" %% "scalatestplus-play" % "3.1.2" % s"$Test,$IntegrationTest", + "org.scalatestplus.play" %% "scalatestplus-play" % "4.0.3" % s"$Test,$IntegrationTest", "org.pegdown" % "pegdown" % "1.6.0" % s"$Test,$IntegrationTest", - "org.mockito" % "mockito-core" % "3.3.0" % s"$Test,$IntegrationTest" + "org.mockito" % "mockito-core" % "3.3.3" % s"$Test,$IntegrationTest" ) def apply(): Seq[ModuleID] = compile ++ test diff --git a/project/plugins.sbt b/project/plugins.sbt index f0dd90a..82547b8 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -5,13 +5,13 @@ resolvers += Resolver.bintrayRepo("hmrc", "releases") addSbtPlugin("com.github.gseitz" % "sbt-release" % "1.0.13") -addSbtPlugin("com.typesafe.play" % "sbt-plugin" % "2.6.25") +addSbtPlugin("com.typesafe.play" % "sbt-plugin" % "2.7.4") addSbtPlugin("uk.gov.hmrc" % "sbt-auto-build" % "2.6.0") -addSbtPlugin("uk.gov.hmrc" % "sbt-settings" % "4.0.0") +addSbtPlugin("uk.gov.hmrc" % "sbt-settings" % "4.1.0") -addSbtPlugin("uk.gov.hmrc" % "sbt-artifactory" % "1.0.0") +addSbtPlugin("uk.gov.hmrc" % "sbt-artifactory" % "1.2.0") addSbtPlugin("uk.gov.hmrc" % "sbt-git-versioning" % "2.1.0") @@ -21,4 +21,4 @@ addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.6.1") addSbtPlugin("org.scalastyle" %% "scalastyle-sbt-plugin" % "1.0.0") -addSbtPlugin("net.virtual-void" % "sbt-dependency-graph" % "0.9.2") +addSbtPlugin("net.virtual-void" % "sbt-dependency-graph" % "0.10.0-RC1") diff --git a/test/utils/UserAgentFilterSpec.scala b/test/utils/UserAgentFilterSpec.scala index 0141af4..6b35fd4 100644 --- a/test/utils/UserAgentFilterSpec.scala +++ b/test/utils/UserAgentFilterSpec.scala @@ -18,6 +18,7 @@ package utils import akka.util.Timeout import org.scalatest.{GivenWhenThen, Matchers} +import play.api.Logging import play.api.http.Status.{BAD_REQUEST, OK} import play.api.mvc.Results._ import play.api.mvc.{Request, Result} @@ -66,5 +67,5 @@ private object UserAgentFilterSpec { val SomeUserAgent = "SOME_USER-AGENT" val block: (Request[_], String) => Future[Result] = (_, userAgent) => Future.successful(Ok(userAgent)) - object UserAgentFilter extends UserAgentFilter + object UserAgentFilter extends Logging with UserAgentFilter } \ No newline at end of file From 3448d0026de75d68c7961c32e3756d84b6764f8e Mon Sep 17 00:00:00 2001 From: Nigel Perkins Date: Tue, 28 Apr 2020 16:19:03 +0100 Subject: [PATCH 49/96] BDOG-760: update deprecated config keys --- conf/application.conf | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/conf/application.conf b/conf/application.conf index eb8e6ac..5744f1c 100644 --- a/conf/application.conf +++ b/conf/application.conf @@ -49,17 +49,11 @@ play.http.errorHandler = "uk.gov.hmrc.play.bootstrap.http.JsonErrorHandler" # ~~~~~ # The secret key is used to secure cryptographics functions. # If you deploy your application to several instances be sure to use the same key! -play.crypto.secret="mKW4mMngaPz1UVOrEkgkJrZT9kloW9Neva5wUFDUq84n9xlz42lQMGrtlrQ2bMF4" play.http.secret.key="mKW4mMngaPz1UVOrEkgkJrZT9kloW9Neva5wUFDUq84n9xlz42lQMGrtlrQ2bMF4" -# Session configuration -# ~~~~~ -application.session.httpOnly=false -application.session.secure=false - # The application languages # ~~~~~ -application.langs="en" +play.i18n.langs = ["en"] # Router # ~~~~~ @@ -71,7 +65,7 @@ application.langs="en" # you may need to define a router file `conf/my.application.routes`. # Default to Routes in the root package (and conf/routes) # !!!WARNING!!! DO NOT CHANGE THIS ROUTER -application.router=prod.Routes +play.http.router=prod.Routes # Metrics plugin settings - graphite reporting is configured on a per env basis From 414a03c67b140e950077c0c05f9ed48af739d4cf Mon Sep 17 00:00:00 2001 From: Nigel Perkins Date: Tue, 28 Apr 2020 17:01:40 +0100 Subject: [PATCH 50/96] BDOG-760: remove dependency on deprecated library: hmrctest --- .../PrepareUploadControllerISpec.scala | 5 ++--- project/AppDependencies.scala | 1 - test/connectors/s3/PolicySignerSpec.scala | 4 ++-- .../s3/S3UploadFormGeneratorSpec.scala | 6 ++--- .../PrepareUploadControllerSpec.scala | 7 +++--- test/model/PrepareUploadServiceSpec.scala | 4 ++-- test/uk/gov/hmrc/play/test/UnitSpec.scala | 22 +++++++++++++++++++ test/utils/UserAgentFilterSpec.scala | 6 ++--- 8 files changed, 37 insertions(+), 18 deletions(-) create mode 100644 test/uk/gov/hmrc/play/test/UnitSpec.scala diff --git a/it/controllers/PrepareUploadControllerISpec.scala b/it/controllers/PrepareUploadControllerISpec.scala index 5b570a3..f436c74 100644 --- a/it/controllers/PrepareUploadControllerISpec.scala +++ b/it/controllers/PrepareUploadControllerISpec.scala @@ -1,6 +1,6 @@ package controllers -import org.scalatest.GivenWhenThen +import org.scalatest.{GivenWhenThen, Matchers, WordSpecLike} import org.scalatestplus.play.guice.GuiceOneAppPerSuite import play.api.Application import play.api.http.HeaderNames.USER_AGENT @@ -10,9 +10,8 @@ import play.api.libs.json.{JsValue, Json} import play.api.test.Helpers._ import play.api.test.{FakeHeaders, FakeRequest} import uk.gov.hmrc.http.HeaderNames.xSessionId -import uk.gov.hmrc.play.test.UnitSpec -class PrepareUploadControllerISpec extends UnitSpec with GuiceOneAppPerSuite with GivenWhenThen { +class PrepareUploadControllerISpec extends WordSpecLike with Matchers with GuiceOneAppPerSuite with GivenWhenThen { import PrepareUploadControllerISpec._ diff --git a/project/AppDependencies.scala b/project/AppDependencies.scala index 8615ee6..f5711c3 100644 --- a/project/AppDependencies.scala +++ b/project/AppDependencies.scala @@ -11,7 +11,6 @@ object AppDependencies { ) private val test = Seq( - "uk.gov.hmrc" %% "hmrctest" % "3.9.0-play-26" % s"$Test,$IntegrationTest", "com.typesafe.play" %% "play-test" % PlayVersion.current % s"$Test,$IntegrationTest", "org.scalatest" %% "scalatest" % "3.0.8" % s"$Test,$IntegrationTest", "org.scalatestplus.play" %% "scalatestplus-play" % "4.0.3" % s"$Test,$IntegrationTest", diff --git a/test/connectors/s3/PolicySignerSpec.scala b/test/connectors/s3/PolicySignerSpec.scala index 2fb7994..a844d12 100644 --- a/test/connectors/s3/PolicySignerSpec.scala +++ b/test/connectors/s3/PolicySignerSpec.scala @@ -17,9 +17,9 @@ package connectors.s3 import connectors.model.AwsCredentials -import org.scalatest.{Matchers, WordSpec} +import uk.gov.hmrc.play.test.UnitSpec -class PolicySignerSpec extends WordSpec with Matchers { +class PolicySignerSpec extends UnitSpec { "Policy signer" should { "sign policy document" in { diff --git a/test/connectors/s3/S3UploadFormGeneratorSpec.scala b/test/connectors/s3/S3UploadFormGeneratorSpec.scala index 6249320..7c95d7b 100644 --- a/test/connectors/s3/S3UploadFormGeneratorSpec.scala +++ b/test/connectors/s3/S3UploadFormGeneratorSpec.scala @@ -22,11 +22,11 @@ import java.util.Base64 import connectors.model.{AwsCredentials, ContentLengthRange, UploadParameters} import org.mockito.ArgumentMatchers.any import org.mockito.Mockito.{verify, when} -import org.scalatest.{GivenWhenThen, Matchers, WordSpec} -import org.scalatestplus.mockito.MockitoSugar +import org.scalatest.GivenWhenThen import play.api.libs.json.{JsArray, JsValue, Json} +import uk.gov.hmrc.play.test.UnitSpec -class S3UploadFormGeneratorSpec extends WordSpec with GivenWhenThen with Matchers with MockitoSugar { +class S3UploadFormGeneratorSpec extends UnitSpec with GivenWhenThen { "S3UploadFormGenerator" should { "generate required fields for a presigned POST request" in { diff --git a/test/controllers/PrepareUploadControllerSpec.scala b/test/controllers/PrepareUploadControllerSpec.scala index 96da278..51eb3c6 100644 --- a/test/controllers/PrepareUploadControllerSpec.scala +++ b/test/controllers/PrepareUploadControllerSpec.scala @@ -26,14 +26,13 @@ import org.mockito.ArgumentMatchers.any import org.mockito.Mockito import org.mockito.invocation.InvocationOnMock import org.mockito.stubbing.Answer -import org.scalatest.{GivenWhenThen, Matchers} -import org.scalatestplus.mockito.MockitoSugar +import org.scalatest.GivenWhenThen import play.api.http.HeaderNames.USER_AGENT import play.api.http.Status.{BAD_REQUEST, OK} import play.api.libs.json.{JsObject, JsValue, Json} import play.api.mvc.Action import play.api.mvc.Results.Ok -import play.api.test.Helpers.contentAsString +import play.api.test.Helpers.{contentAsString, status} import play.api.test.{FakeRequest, Helpers, StubControllerComponentsFactory} import services.PrepareUploadService import services.model.UploadSettings @@ -43,7 +42,7 @@ import scala.concurrent.Future import scala.concurrent.duration._ import scala.language.postfixOps -class PrepareUploadControllerSpec extends UnitSpec with StubControllerComponentsFactory with Matchers with GivenWhenThen with MockitoSugar { +class PrepareUploadControllerSpec extends UnitSpec with StubControllerComponentsFactory with GivenWhenThen { implicit val actorSystem: ActorSystem = ActorSystem() diff --git a/test/model/PrepareUploadServiceSpec.scala b/test/model/PrepareUploadServiceSpec.scala index 0b3da1d..7b5a49d 100644 --- a/test/model/PrepareUploadServiceSpec.scala +++ b/test/model/PrepareUploadServiceSpec.scala @@ -23,12 +23,12 @@ import com.codahale.metrics.MetricRegistry import com.kenshoo.play.metrics.Metrics import config.ServiceConfiguration import connectors.model.{UploadFormGenerator, UploadParameters} -import org.scalatest.{GivenWhenThen, Matchers} +import org.scalatest.GivenWhenThen import services.PrepareUploadService import services.model.UploadSettings import uk.gov.hmrc.play.test.UnitSpec -class PrepareUploadServiceSpec extends UnitSpec with Matchers with GivenWhenThen { +class PrepareUploadServiceSpec extends UnitSpec with GivenWhenThen { private val serviceConfiguration = new ServiceConfiguration { diff --git a/test/uk/gov/hmrc/play/test/UnitSpec.scala b/test/uk/gov/hmrc/play/test/UnitSpec.scala new file mode 100644 index 0000000..74fb58f --- /dev/null +++ b/test/uk/gov/hmrc/play/test/UnitSpec.scala @@ -0,0 +1,22 @@ +/* + * Copyright 2020 HM Revenue & Customs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package uk.gov.hmrc.play.test + +import org.scalatest.{Matchers, WordSpecLike} +import org.scalatestplus.mockito.MockitoSugar + +trait UnitSpec extends WordSpecLike with Matchers with MockitoSugar diff --git a/test/utils/UserAgentFilterSpec.scala b/test/utils/UserAgentFilterSpec.scala index 6b35fd4..489822f 100644 --- a/test/utils/UserAgentFilterSpec.scala +++ b/test/utils/UserAgentFilterSpec.scala @@ -17,20 +17,20 @@ package utils import akka.util.Timeout -import org.scalatest.{GivenWhenThen, Matchers} +import org.scalatest.GivenWhenThen import play.api.Logging import play.api.http.Status.{BAD_REQUEST, OK} import play.api.mvc.Results._ import play.api.mvc.{Request, Result} import play.api.test.FakeRequest -import play.api.test.Helpers.contentAsString +import play.api.test.Helpers.{contentAsString, status} import play.mvc.Http.HeaderNames.USER_AGENT import uk.gov.hmrc.play.test.UnitSpec import scala.concurrent.Future import scala.concurrent.duration._ -class UserAgentFilterSpec extends UnitSpec with Matchers with GivenWhenThen { +class UserAgentFilterSpec extends UnitSpec with GivenWhenThen { import UserAgentFilterSpec._ From f79e761af03db0a374e3feaf70801b165d49681b Mon Sep 17 00:00:00 2001 From: Nigel Perkins Date: Thu, 30 Apr 2020 08:27:35 +0100 Subject: [PATCH 51/96] BDOG-760: changes based on review feedback --- it/controllers/PrepareUploadControllerISpec.scala | 6 ++++-- project/AppDependencies.scala | 6 +++--- project/plugins.sbt | 2 -- test/connectors/s3/PolicySignerSpec.scala | 2 +- test/connectors/s3/S3UploadFormGeneratorSpec.scala | 2 +- test/controllers/PrepareUploadControllerSpec.scala | 2 +- test/model/PrepareUploadServiceSpec.scala | 2 +- test/{uk/gov/hmrc/play => }/test/UnitSpec.scala | 7 ++++--- test/utils/UserAgentFilterSpec.scala | 2 +- 9 files changed, 16 insertions(+), 15 deletions(-) rename test/{uk/gov/hmrc/play => }/test/UnitSpec.scala (78%) diff --git a/it/controllers/PrepareUploadControllerISpec.scala b/it/controllers/PrepareUploadControllerISpec.scala index f436c74..21af441 100644 --- a/it/controllers/PrepareUploadControllerISpec.scala +++ b/it/controllers/PrepareUploadControllerISpec.scala @@ -1,6 +1,8 @@ package controllers -import org.scalatest.{GivenWhenThen, Matchers, WordSpecLike} +import org.scalatest.GivenWhenThen +import org.scalatest.matchers.should +import org.scalatest.wordspec.AnyWordSpecLike import org.scalatestplus.play.guice.GuiceOneAppPerSuite import play.api.Application import play.api.http.HeaderNames.USER_AGENT @@ -11,7 +13,7 @@ import play.api.test.Helpers._ import play.api.test.{FakeHeaders, FakeRequest} import uk.gov.hmrc.http.HeaderNames.xSessionId -class PrepareUploadControllerISpec extends WordSpecLike with Matchers with GuiceOneAppPerSuite with GivenWhenThen { +class PrepareUploadControllerISpec extends AnyWordSpecLike with should.Matchers with GuiceOneAppPerSuite with GivenWhenThen { import PrepareUploadControllerISpec._ diff --git a/project/AppDependencies.scala b/project/AppDependencies.scala index f5711c3..12991d2 100644 --- a/project/AppDependencies.scala +++ b/project/AppDependencies.scala @@ -12,10 +12,10 @@ object AppDependencies { private val test = Seq( "com.typesafe.play" %% "play-test" % PlayVersion.current % s"$Test,$IntegrationTest", - "org.scalatest" %% "scalatest" % "3.0.8" % s"$Test,$IntegrationTest", + "org.scalatest" %% "scalatest" % "3.1.1" % s"$Test,$IntegrationTest", + "org.scalatestplus" %% "mockito-3-2" % "3.1.1.0" % Test, "org.scalatestplus.play" %% "scalatestplus-play" % "4.0.3" % s"$Test,$IntegrationTest", - "org.pegdown" % "pegdown" % "1.6.0" % s"$Test,$IntegrationTest", - "org.mockito" % "mockito-core" % "3.3.3" % s"$Test,$IntegrationTest" + "com.vladsch.flexmark" % "flexmark-all" % "0.35.10" % s"$Test,$IntegrationTest" ) def apply(): Seq[ModuleID] = compile ++ test diff --git a/project/plugins.sbt b/project/plugins.sbt index 82547b8..a5866e3 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -20,5 +20,3 @@ addSbtPlugin("uk.gov.hmrc" % "sbt-distributables" % "2.0.0") addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.6.1") addSbtPlugin("org.scalastyle" %% "scalastyle-sbt-plugin" % "1.0.0") - -addSbtPlugin("net.virtual-void" % "sbt-dependency-graph" % "0.10.0-RC1") diff --git a/test/connectors/s3/PolicySignerSpec.scala b/test/connectors/s3/PolicySignerSpec.scala index a844d12..695a633 100644 --- a/test/connectors/s3/PolicySignerSpec.scala +++ b/test/connectors/s3/PolicySignerSpec.scala @@ -17,7 +17,7 @@ package connectors.s3 import connectors.model.AwsCredentials -import uk.gov.hmrc.play.test.UnitSpec +import test.UnitSpec class PolicySignerSpec extends UnitSpec { diff --git a/test/connectors/s3/S3UploadFormGeneratorSpec.scala b/test/connectors/s3/S3UploadFormGeneratorSpec.scala index 7c95d7b..6f65fc6 100644 --- a/test/connectors/s3/S3UploadFormGeneratorSpec.scala +++ b/test/connectors/s3/S3UploadFormGeneratorSpec.scala @@ -24,7 +24,7 @@ import org.mockito.ArgumentMatchers.any import org.mockito.Mockito.{verify, when} import org.scalatest.GivenWhenThen import play.api.libs.json.{JsArray, JsValue, Json} -import uk.gov.hmrc.play.test.UnitSpec +import test.UnitSpec class S3UploadFormGeneratorSpec extends UnitSpec with GivenWhenThen { diff --git a/test/controllers/PrepareUploadControllerSpec.scala b/test/controllers/PrepareUploadControllerSpec.scala index 51eb3c6..7fe3156 100644 --- a/test/controllers/PrepareUploadControllerSpec.scala +++ b/test/controllers/PrepareUploadControllerSpec.scala @@ -36,7 +36,7 @@ import play.api.test.Helpers.{contentAsString, status} import play.api.test.{FakeRequest, Helpers, StubControllerComponentsFactory} import services.PrepareUploadService import services.model.UploadSettings -import uk.gov.hmrc.play.test.UnitSpec +import test.UnitSpec import scala.concurrent.Future import scala.concurrent.duration._ diff --git a/test/model/PrepareUploadServiceSpec.scala b/test/model/PrepareUploadServiceSpec.scala index 7b5a49d..50a35e4 100644 --- a/test/model/PrepareUploadServiceSpec.scala +++ b/test/model/PrepareUploadServiceSpec.scala @@ -26,7 +26,7 @@ import connectors.model.{UploadFormGenerator, UploadParameters} import org.scalatest.GivenWhenThen import services.PrepareUploadService import services.model.UploadSettings -import uk.gov.hmrc.play.test.UnitSpec +import test.UnitSpec class PrepareUploadServiceSpec extends UnitSpec with GivenWhenThen { diff --git a/test/uk/gov/hmrc/play/test/UnitSpec.scala b/test/test/UnitSpec.scala similarity index 78% rename from test/uk/gov/hmrc/play/test/UnitSpec.scala rename to test/test/UnitSpec.scala index 74fb58f..3842015 100644 --- a/test/uk/gov/hmrc/play/test/UnitSpec.scala +++ b/test/test/UnitSpec.scala @@ -14,9 +14,10 @@ * limitations under the License. */ -package uk.gov.hmrc.play.test +package test -import org.scalatest.{Matchers, WordSpecLike} +import org.scalatest.matchers.should +import org.scalatest.wordspec.AnyWordSpecLike import org.scalatestplus.mockito.MockitoSugar -trait UnitSpec extends WordSpecLike with Matchers with MockitoSugar +trait UnitSpec extends AnyWordSpecLike with should.Matchers with MockitoSugar diff --git a/test/utils/UserAgentFilterSpec.scala b/test/utils/UserAgentFilterSpec.scala index 489822f..8c36637 100644 --- a/test/utils/UserAgentFilterSpec.scala +++ b/test/utils/UserAgentFilterSpec.scala @@ -25,7 +25,7 @@ import play.api.mvc.{Request, Result} import play.api.test.FakeRequest import play.api.test.Helpers.{contentAsString, status} import play.mvc.Http.HeaderNames.USER_AGENT -import uk.gov.hmrc.play.test.UnitSpec +import test.UnitSpec import scala.concurrent.Future import scala.concurrent.duration._ From 4fb1b0a9b9f22d244f2993a84c71118b0f678022 Mon Sep 17 00:00:00 2001 From: Nigel Perkins Date: Thu, 30 Apr 2020 09:40:26 +0100 Subject: [PATCH 52/96] BDOG-760: switch from mockito to mockito-scala --- project/AppDependencies.scala | 4 +- .../s3/S3UploadFormGeneratorSpec.scala | 8 +- .../PrepareUploadControllerSpec.scala | 81 +++++++++---------- test/test/UnitSpec.scala | 2 +- test/utils/UserAgentFilterSpec.scala | 3 +- 5 files changed, 46 insertions(+), 52 deletions(-) diff --git a/project/AppDependencies.scala b/project/AppDependencies.scala index 12991d2..f359a24 100644 --- a/project/AppDependencies.scala +++ b/project/AppDependencies.scala @@ -13,9 +13,9 @@ object AppDependencies { private val test = Seq( "com.typesafe.play" %% "play-test" % PlayVersion.current % s"$Test,$IntegrationTest", "org.scalatest" %% "scalatest" % "3.1.1" % s"$Test,$IntegrationTest", - "org.scalatestplus" %% "mockito-3-2" % "3.1.1.0" % Test, "org.scalatestplus.play" %% "scalatestplus-play" % "4.0.3" % s"$Test,$IntegrationTest", - "com.vladsch.flexmark" % "flexmark-all" % "0.35.10" % s"$Test,$IntegrationTest" + "com.vladsch.flexmark" % "flexmark-all" % "0.35.10" % s"$Test,$IntegrationTest", + "org.mockito" %% "mockito-scala-scalatest" % "1.13.10" % Test ) def apply(): Seq[ModuleID] = compile ++ test diff --git a/test/connectors/s3/S3UploadFormGeneratorSpec.scala b/test/connectors/s3/S3UploadFormGeneratorSpec.scala index 6f65fc6..3018890 100644 --- a/test/connectors/s3/S3UploadFormGeneratorSpec.scala +++ b/test/connectors/s3/S3UploadFormGeneratorSpec.scala @@ -20,8 +20,6 @@ import java.time.Instant import java.util.Base64 import connectors.model.{AwsCredentials, ContentLengthRange, UploadParameters} -import org.mockito.ArgumentMatchers.any -import org.mockito.Mockito.{verify, when} import org.scalatest.GivenWhenThen import play.api.libs.json.{JsArray, JsValue, Json} import test.UnitSpec @@ -40,7 +38,7 @@ class S3UploadFormGeneratorSpec extends UnitSpec with GivenWhenThen { val policySigner = mock[PolicySigner] val testSignature = "test-signature" - when(policySigner.signPolicy(any(), any(), any(), any())).thenReturn(testSignature) + when(policySigner.signPolicy(any[AwsCredentials], any[String], any[String], any[String])).thenReturn(testSignature) val generator = new S3UploadFormGenerator(credentials, regionName, currentTime, policySigner) @@ -117,7 +115,7 @@ class S3UploadFormGeneratorSpec extends UnitSpec with GivenWhenThen { val policySigner = mock[PolicySigner] val testSignature = "test-signature" - when(policySigner.signPolicy(any(), any(), any(), any())).thenReturn(testSignature) + when(policySigner.signPolicy(any[AwsCredentials], any[String], any[String], any[String])).thenReturn(testSignature) val generator = new S3UploadFormGenerator(credentials, regionName, currentTime, policySigner) @@ -150,7 +148,7 @@ class S3UploadFormGeneratorSpec extends UnitSpec with GivenWhenThen { val policySigner = mock[PolicySigner] val testSignature = "test-signature" - when(policySigner.signPolicy(any(), any(), any(), any())).thenReturn(testSignature) + when(policySigner.signPolicy(any[AwsCredentials], any[String], any[String], any[String])).thenReturn(testSignature) val generator = new S3UploadFormGenerator(credentials, regionName, currentTime, policySigner) diff --git a/test/controllers/PrepareUploadControllerSpec.scala b/test/controllers/PrepareUploadControllerSpec.scala index 7fe3156..5428c96 100644 --- a/test/controllers/PrepareUploadControllerSpec.scala +++ b/test/controllers/PrepareUploadControllerSpec.scala @@ -16,16 +16,12 @@ package controllers -import java.time.Clock +import java.time.{Clock, Instant} import akka.actor.ActorSystem import akka.stream.ActorMaterializer import config.ServiceConfiguration import controllers.model.{PreparedUploadResponse, Reference, UploadFormTemplate} -import org.mockito.ArgumentMatchers.any -import org.mockito.Mockito -import org.mockito.invocation.InvocationOnMock -import org.mockito.stubbing.Answer import org.scalatest.GivenWhenThen import play.api.http.HeaderNames.USER_AGENT import play.api.http.Status.{BAD_REQUEST, OK} @@ -44,20 +40,20 @@ import scala.language.postfixOps class PrepareUploadControllerSpec extends UnitSpec with StubControllerComponentsFactory with GivenWhenThen { - implicit val actorSystem: ActorSystem = ActorSystem() + private implicit val actorSystem: ActorSystem = ActorSystem() - implicit val materializer: ActorMaterializer = ActorMaterializer() + private implicit val materializer: ActorMaterializer = ActorMaterializer() - implicit val timeout: akka.util.Timeout = 10 seconds + private implicit val timeout: akka.util.Timeout = 10 seconds private val clock: Clock = Clock.systemDefaultZone() - "PaymentController prepareUploadV1" should { + "PrepareUploadController prepareUploadV1" should { behave like prepareUploadTests(_.prepareUploadV1()) } - "PaymentController prepareUploadV2" should { + "PrepareUploadController prepareUploadV2" should { val extraRequestFields = Json .obj("successRedirect" -> "https://www.example.com/nextpage", "errorRedirect" -> "https://www.example.com/error") @@ -75,10 +71,10 @@ class PrepareUploadControllerSpec extends UnitSpec with StubControllerComponents extraResponseFields: JsObject = JsObject(Seq())) { val config = mock[ServiceConfiguration] - Mockito.when(config.allowedCallbackProtocols).thenReturn(List("https")) + when(config.allowedCallbackProtocols).thenReturn(List("https")) "build and return upload URL if valid request with all data" in { - val controller = new PrepareUploadController(prepareUploadService, config, clock, stubControllerComponents()) + val controller = new PrepareUploadController(prepareUploadServiceWithUpload, config, clock, stubControllerComponents()) Given("there is a valid upload request with all data") @@ -116,7 +112,7 @@ class PrepareUploadControllerSpec extends UnitSpec with StubControllerComponents } "build and return upload URL if valid request with redirect on success url" in { - val controller = new PrepareUploadController(prepareUploadService, config, clock, stubControllerComponents()) + val controller = new PrepareUploadController(prepareUploadServiceWithUpload, config, clock, stubControllerComponents()) Given("there is a valid upload request with all data") @@ -157,7 +153,7 @@ class PrepareUploadControllerSpec extends UnitSpec with StubControllerComponents } "build and return upload URL if valid request with minimal data including session id and request id" in { - val controller = new PrepareUploadController(prepareUploadService, config, clock, stubControllerComponents()) + val controller = new PrepareUploadController(prepareUploadServiceWithUpload, config, clock, stubControllerComponents()) Given("there is a valid upload request with minimal data") @@ -187,7 +183,7 @@ class PrepareUploadControllerSpec extends UnitSpec with StubControllerComponents } "build and return upload URL if valid request with minimal data excluding session id and request id" in { - val controller = new PrepareUploadController(prepareUploadService, config, clock, stubControllerComponents()) + val controller = new PrepareUploadController(prepareUploadServiceWithUpload, config, clock, stubControllerComponents()) Given("there is a valid upload request with minimal data") @@ -213,7 +209,7 @@ class PrepareUploadControllerSpec extends UnitSpec with StubControllerComponents } "return a bad request error if invalid request - wrong structure" in { - val controller = new PrepareUploadController(prepareUploadService, config, clock, stubControllerComponents()) + val controller = new PrepareUploadController(prepareUploadServiceNoUpload, config, clock, stubControllerComponents()) Given("there is an invalid upload request") @@ -234,7 +230,7 @@ class PrepareUploadControllerSpec extends UnitSpec with StubControllerComponents } "return a bad request error if invalid request - incorrect maximum file size " in { - val controller = new PrepareUploadController(prepareUploadService, config, clock, stubControllerComponents()) + val controller = new PrepareUploadController(prepareUploadServiceNoUpload, config, clock, stubControllerComponents()) Given("there is an invalid upload request") @@ -254,7 +250,7 @@ class PrepareUploadControllerSpec extends UnitSpec with StubControllerComponents } "allow https callback urls" in { - val controller = new PrepareUploadController(prepareUploadService, config, clock, stubControllerComponents()) + val controller = new PrepareUploadController(prepareUploadServiceNoUpload, config, clock, stubControllerComponents()) val result = controller.withAllowedCallbackProtocol("https://my.callback.url") { Future.successful(Ok) @@ -264,7 +260,7 @@ class PrepareUploadControllerSpec extends UnitSpec with StubControllerComponents } "disallow http callback urls" in { - val controller = new PrepareUploadController(prepareUploadService, config, clock, stubControllerComponents()) + val controller = new PrepareUploadController(prepareUploadServiceNoUpload, config, clock, stubControllerComponents()) val result = controller.withAllowedCallbackProtocol("http://my.callback.url") { Future.failed(new RuntimeException("This block should not have been invoked.")) @@ -275,7 +271,7 @@ class PrepareUploadControllerSpec extends UnitSpec with StubControllerComponents } "disallow invalidly formatted callback urls" in { - val controller = new PrepareUploadController(prepareUploadService, config, clock, stubControllerComponents()) + val controller = new PrepareUploadController(prepareUploadServiceNoUpload, config, clock, stubControllerComponents()) val result = controller.withAllowedCallbackProtocol("123") { Future.failed(new RuntimeException("This block should not have been invoked.")) @@ -285,32 +281,31 @@ class PrepareUploadControllerSpec extends UnitSpec with StubControllerComponents } } - private def prepareUploadService: PrepareUploadService = { + private def prepareUploadServiceNoUpload: PrepareUploadService = { val service = mock[PrepareUploadService] - Mockito.when(service.globalFileSizeLimit).thenReturn(1024) - Mockito - .when(service.prepareUpload(any(), any(), any(), any(), any())) - .thenAnswer(new Answer[PreparedUploadResponse]() { - override def answer(invocationOnMock: InvocationOnMock): PreparedUploadResponse = { - val settings = invocationOnMock.getArgument[UploadSettings](0) - val requestId = invocationOnMock.getArgument[String](2) - val sessionId = invocationOnMock.getArgument[String](3) - PreparedUploadResponse( - Reference("TEST"), - UploadFormTemplate( - settings.callbackUrl, - Map.empty ++ - settings.minimumFileSize.map(s => Map("minFileSize" -> s.toString).head) ++ - settings.maximumFileSize.map(s => Map("maxFileSize" -> s.toString).head) ++ - Map("sessionId" -> sessionId) ++ - Map("requestId" -> requestId) ++ - settings.successRedirect.map(url => Map("success_action_redirect" -> url)).getOrElse(Map.empty) ++ - settings.errorRedirect.map(url => Map("error_action_redirect" -> url)).getOrElse(Map.empty) - ) + when(service.globalFileSizeLimit).thenReturn(1024) + } + + private def prepareUploadServiceWithUpload: PrepareUploadService = { + val service = mock[PrepareUploadService] + when(service.globalFileSizeLimit).thenReturn(1024) + when(service.prepareUpload(any[UploadSettings], any[String], any[String], any[String], any[Instant])) + .thenAnswer{ (settings: UploadSettings, _: String, requestId: String, sessionId: String, _: Instant) => + PreparedUploadResponse( + Reference("TEST"), + UploadFormTemplate( + settings.callbackUrl, + Map.empty ++ + settings.minimumFileSize.map(s => Map("minFileSize" -> s.toString).head) ++ + settings.maximumFileSize.map(s => Map("maxFileSize" -> s.toString).head) ++ + Map("sessionId" -> sessionId) ++ + Map("requestId" -> requestId) ++ + settings.successRedirect.map(url => Map("success_action_redirect" -> url)).getOrElse(Map.empty) ++ + settings.errorRedirect.map(url => Map("error_action_redirect" -> url)).getOrElse(Map.empty) ) - } + ) + } - }) service } diff --git a/test/test/UnitSpec.scala b/test/test/UnitSpec.scala index 3842015..82d0e82 100644 --- a/test/test/UnitSpec.scala +++ b/test/test/UnitSpec.scala @@ -16,8 +16,8 @@ package test +import org.mockito.scalatest.MockitoSugar import org.scalatest.matchers.should import org.scalatest.wordspec.AnyWordSpecLike -import org.scalatestplus.mockito.MockitoSugar trait UnitSpec extends AnyWordSpecLike with should.Matchers with MockitoSugar diff --git a/test/utils/UserAgentFilterSpec.scala b/test/utils/UserAgentFilterSpec.scala index 8c36637..7a29aff 100644 --- a/test/utils/UserAgentFilterSpec.scala +++ b/test/utils/UserAgentFilterSpec.scala @@ -34,6 +34,8 @@ class UserAgentFilterSpec extends UnitSpec with GivenWhenThen { import UserAgentFilterSpec._ + private implicit val timeout: Timeout = Timeout(3.seconds) + "UserAgentFilter" should { "accept a request when the User-Agent header is specified" in { Given("a request that specifies a User-Agent header") @@ -63,7 +65,6 @@ class UserAgentFilterSpec extends UnitSpec with GivenWhenThen { } private object UserAgentFilterSpec { - implicit val timeout: Timeout = Timeout(3.seconds) val SomeUserAgent = "SOME_USER-AGENT" val block: (Request[_], String) => Future[Result] = (_, userAgent) => Future.successful(Ok(userAgent)) From 974e0391953c8fa5c181235e0b1b45ca83dda3ea Mon Sep 17 00:00:00 2001 From: Nigel Perkins Date: Thu, 30 Apr 2020 11:17:24 +0100 Subject: [PATCH 53/96] BDOG-760: do not use bobby-ruled bootstrap version --- project/AppDependencies.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/AppDependencies.scala b/project/AppDependencies.scala index f359a24..f598386 100644 --- a/project/AppDependencies.scala +++ b/project/AppDependencies.scala @@ -4,7 +4,7 @@ object AppDependencies { import play.core.PlayVersion private val compile = Seq( - "uk.gov.hmrc" %% "bootstrap-backend-play-27" % "2.4.0", + "uk.gov.hmrc" %% "bootstrap-backend-play-27" % "2.5.0", "com.typesafe.play" %% "play-json" % PlayVersion.current, "com.amazonaws" % "aws-java-sdk-s3" % "1.11.769", "org.apache.commons" % "commons-lang3" % "3.10" From e7807b4d68b45eb92723ebe7c659a488c2d8c325 Mon Sep 17 00:00:00 2001 From: Nigel Perkins Date: Wed, 27 May 2020 15:09:04 +0100 Subject: [PATCH 54/96] BDOG-885: uplift Bootstrap to prevent Malformed Datastream requests --- project/AppDependencies.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/AppDependencies.scala b/project/AppDependencies.scala index f598386..7ba2c0f 100644 --- a/project/AppDependencies.scala +++ b/project/AppDependencies.scala @@ -4,7 +4,7 @@ object AppDependencies { import play.core.PlayVersion private val compile = Seq( - "uk.gov.hmrc" %% "bootstrap-backend-play-27" % "2.5.0", + "uk.gov.hmrc" %% "bootstrap-backend-play-27" % "2.14.0", "com.typesafe.play" %% "play-json" % PlayVersion.current, "com.amazonaws" % "aws-java-sdk-s3" % "1.11.769", "org.apache.commons" % "commons-lang3" % "3.10" From f28b90780b31ee5783169452508f32386a4cf36c Mon Sep 17 00:00:00 2001 From: Nigel Perkins Date: Wed, 19 Aug 2020 12:44:34 +0100 Subject: [PATCH 55/96] BDOG-1006: successRedirect url is now optional for both v1 & v2 --- README.md | 8 +- app/controllers/PrepareUploadController.scala | 6 +- .../model/PrepareUploadRequestV2.scala | 7 +- .../PrepareUploadControllerISpec.scala | 225 +++++++++++++----- .../model/PrepareUploadRequestV2Spec.scala | 81 +++++++ 5 files changed, 252 insertions(+), 75 deletions(-) create mode 100644 test/controllers/model/PrepareUploadRequestV2Spec.scala diff --git a/README.md b/README.md index 5e0642e..4174567 100644 --- a/README.md +++ b/README.md @@ -71,8 +71,8 @@ Please view the [Upscan Service & Flow Overview in Confluence](https://confluenc ### Requesting a URL to upload to The consuming service makes a POST request to `/upscan/initiate` or `upscan/v2/initiate`. -This request must contain a `User-Agent` header that can be used to identify the service, but a whitelist of authorised services is no longer cross-checked. -The service must also provide a callbackUrl for asynchronous notification of the outcome of an upload. The callback will be made from inside the MDTP environment. Hence, the callback URL should comprise the MDTP internal callback address and not the public domain address. `upscan/v2/initiate` additionally requires a successRedirect and errorRedirect url. See next section for specifics. +This request must contain a `User-Agent` header that can be used to identify the service, but an allow list of authorised services is no longer cross-checked. +The service must also provide a callbackUrl for asynchronous notification of the outcome of an upload. The callback will be made from inside the MDTP environment. Hence, the callback URL should comprise the MDTP internal callback address and not the public domain address. `upscan/v2/initiate` additionally requires an errorRedirect url. See next section for specifics. **Note:** `callbackUrl` must use the `https` protocol. (Although this rule is relaxed when testing locally with [upscan-stub](https://github.com/hmrc/upscan-stub) rather than [upscan-initiate](https://github.com/hmrc/upscan-initiate). @@ -117,10 +117,11 @@ Example `upscan/v2/initiate` request: | Parameter name|Description|Required| |--------------|-----------|--------| |callbackUrl |Url that will be called to report the outcome of file checking and upload, including retrieval details if successful. Notification format is detailed further down in this file. Must be https.| yes| -|successRedirect|Url to redirect to after file has been successfully uploaded.|yes| +|successRedirect|Url to redirect to after file has been successfully uploaded.|no| |errorRedirect|Url to redirect to if error encountered during upload.|yes| |minimumFileSize|Minimum file size (in Bytes). Default is 0.|no| |maximumFileSize|Maximum file size (in Bytes). Cannot be greater than 100MB. Default is 100MB.|no| +|expectedContentType|MIME type describing the upload contents.|no| Example response @@ -211,6 +212,7 @@ Example request: |minimumFileSize|Minimum file size (in Bytes). Default is 0.|no| |maximumFileSize|Maximum file size (in Bytes). Cannot be greater than 100MB. Default is 100MB.|no| |successRedirect|Url to redirect to after file has been successfully uploaded.|no| +|expectedContentType|MIME type describing the upload contents.|no| diff --git a/app/controllers/PrepareUploadController.scala b/app/controllers/PrepareUploadController.scala index 39d24a2..1d6ec30 100644 --- a/app/controllers/PrepareUploadController.scala +++ b/app/controllers/PrepareUploadController.scala @@ -26,7 +26,7 @@ import play.api.Logging import play.api.libs.json._ import play.api.mvc.{Action, ControllerComponents, Result} import services.PrepareUploadService -import uk.gov.hmrc.play.bootstrap.controller.BackendController +import uk.gov.hmrc.play.bootstrap.backend.controller.BackendController import utils.UserAgentFilter import scala.concurrent.Future @@ -55,8 +55,8 @@ class PrepareUploadController @Inject()( prepareUpload[PrepareUploadRequestV2](uploadUrl) } - private def prepareUpload[T <: PrepareUpload]( - uploadUrl: String)(implicit reads: Reads[T], manifest: Manifest[T]): Action[JsValue] = + private def prepareUpload[T <: PrepareUpload](uploadUrl: String) + (implicit reads: Reads[T], manifest: Manifest[T]): Action[JsValue] = Action.async(parse.json) { implicit request => val receivedAt = Instant.now(clock) diff --git a/app/controllers/model/PrepareUploadRequestV2.scala b/app/controllers/model/PrepareUploadRequestV2.scala index 48661d5..0a88c5c 100644 --- a/app/controllers/model/PrepareUploadRequestV2.scala +++ b/app/controllers/model/PrepareUploadRequestV2.scala @@ -23,7 +23,7 @@ import services.model.UploadSettings case class PrepareUploadRequestV2( callbackUrl: String, - successRedirect: String, + successRedirect: Option[String], errorRedirect: String, minimumFileSize: Option[Int], maximumFileSize: Option[Int], @@ -36,7 +36,7 @@ case class PrepareUploadRequestV2( minimumFileSize = minimumFileSize, maximumFileSize = maximumFileSize, expectedContentType = expectedContentType, - successRedirect = Some(successRedirect), + successRedirect = successRedirect, errorRedirect = Some(errorRedirect) ) } @@ -45,12 +45,11 @@ object PrepareUploadRequestV2 { def reads(maxFileSize: Int): Reads[PrepareUploadRequestV2] = ((JsPath \ "callbackUrl").read[String] and - (JsPath \ "successRedirect").read[String] and + (JsPath \ "successRedirect").readNullable[String] and (JsPath \ "errorRedirect").read[String] and (JsPath \ "minimumFileSize").readNullable[Int](min(0)) and (JsPath \ "maximumFileSize").readNullable[Int](min(0) keepAnd max(maxFileSize + 1)) and (JsPath \ "expectedContentType").readNullable[String])(PrepareUploadRequestV2.apply _) .filter(JsonValidationError("Maximum file size must be equal or greater than minimum file size"))(request => request.minimumFileSize.getOrElse(0) <= request.maximumFileSize.getOrElse(maxFileSize)) - } diff --git a/it/controllers/PrepareUploadControllerISpec.scala b/it/controllers/PrepareUploadControllerISpec.scala index 21af441..b46cc3e 100644 --- a/it/controllers/PrepareUploadControllerISpec.scala +++ b/it/controllers/PrepareUploadControllerISpec.scala @@ -24,24 +24,63 @@ class PrepareUploadControllerISpec extends AnyWordSpecLike with should.Matchers ) .build() - "PrepareUploadController prepareUploadV1" should { - val postBodyJson = Json.parse(""" - |{ - | "callbackUrl": "https://some-url/callback", - | "minimumFileSize" : 0, - | "maximumFileSize" : 1024, - | "expectedMimeType": "application/xml" - |} - """.stripMargin) + "PrepareUploadController prepareUploadV1 with all request values" in { + val postBodyJson = Json.parse("""|{ + | "callbackUrl": "https://some-url/callback", + | "minimumFileSize" : 0, + | "maximumFileSize" : 1024, + | "expectedContentType": "application/xml", + | "successRedirect": "https://some-url/success" + |}""".stripMargin) + + Given("a request containing a User-Agent header") + val headers = FakeHeaders(Seq((USER_AGENT, SomeConsumingService))) + val initiateRequest = FakeRequest(POST, uri = "/upscan/initiate", headers, postBodyJson) + + When("a request is posted to the /initiate endpoint") + val initiateResponse = route(app, initiateRequest).get + + Then("the response should indicate success") + status(initiateResponse) shouldBe OK + + And("the response should include an AWS S3 upload URL") + val responseJson = contentAsJson(initiateResponse) + (responseJson \ "uploadRequest" \ "href").as[String] shouldBe "https://inbound-bucket.s3.amazonaws.com" + + And("the response should contain the requested upload fields") + val fields = (responseJson \ "uploadRequest" \ "fields").as[Map[String, String]] + fields.get("x-amz-meta-callback-url") should contain ("https://some-url/callback") + fields.get("Content-Type") should contain ("application/xml") + fields.get("success_action_redirect") should contain ("https://some-url/success") + } - behave like prepareUploadTests( - postBodyJson = postBodyJson, - uri = "/upscan/initiate", - href = "https://inbound-bucket.s3.amazonaws.com" - ) + "PrepareUploadController prepareUploadV1 with only mandatory request values" in { + val postBodyJson = Json.parse("""|{ + | "callbackUrl": "https://some-url/callback" + |}""".stripMargin) + + Given("a request containing a User-Agent header") + val headers = FakeHeaders(Seq((USER_AGENT, SomeConsumingService))) + val initiateRequest = FakeRequest(POST, uri = "/upscan/initiate", headers, postBodyJson) + + When("a request is posted to the /initiate endpoint") + val initiateResponse = route(app, initiateRequest).get + + Then("the response should indicate success") + status(initiateResponse) shouldBe OK + + And("the response should include an AWS S3 upload URL") + val responseJson = contentAsJson(initiateResponse) + (responseJson \ "uploadRequest" \ "href").as[String] shouldBe "https://inbound-bucket.s3.amazonaws.com" + + And("the response should contain the requested upload fields") + val fields = (responseJson \ "uploadRequest" \ "fields").as[Map[String, String]] + fields.get("x-amz-meta-callback-url") should contain ("https://some-url/callback") + fields.get("Content-Type") shouldBe empty + fields.get("success_action_redirect") shouldBe empty } - "PrepareUploadController prepareUploadV2" should { + "PrepareUploadController prepareUploadV2 with all request values" in { val postBodyJson = Json.parse(""" |{ | "callbackUrl": "https://some-url/callback", @@ -49,53 +88,107 @@ class PrepareUploadControllerISpec extends AnyWordSpecLike with should.Matchers | "errorRedirect": "https://some-url/error", | "minimumFileSize" : 0, | "maximumFileSize" : 1024, - | "expectedMimeType": "application/xml" + | "expectedContentType": "application/xml" |} """.stripMargin) - behave like prepareUploadTests( - postBodyJson = postBodyJson, - uri = "/upscan/v2/initiate", - href = "https://upload-proxy.tax.service.gov.uk/v1/uploads/inbound-bucket" - ) + Given("a request containing a User-Agent header") + val headers = FakeHeaders(Seq((USER_AGENT, SomeConsumingService))) + val initiateRequest = FakeRequest(POST, uri = "/upscan/v2/initiate", headers, postBodyJson) + + When("a request is posted to the /initiate endpoint") + val initiateResponse = route(app, initiateRequest).get + + Then("the response should indicate success") + status(initiateResponse) shouldBe OK + + And("the response should include an upscan-proxy upload URL") + val responseJson = contentAsJson(initiateResponse) + (responseJson \ "uploadRequest" \ "href").as[String] shouldBe "https://upload-proxy.tax.service.gov.uk/v1/uploads/inbound-bucket" + + And("the response should contain the requested upload fields") + val fields = (responseJson \ "uploadRequest" \ "fields").as[Map[String, String]] + fields.get("x-amz-meta-callback-url") should contain ("https://some-url/callback") + fields.get("success_action_redirect") should contain ("https://some-url/success") + fields.get("error_action_redirect") should contain ("https://some-url/error") + fields.get("Content-Type") should contain ("application/xml") + } + + "PrepareUploadController prepareUploadV2 with only mandatory request values" in { + val postBodyJson = Json.parse("""|{ + | "callbackUrl": "https://some-url/callback", + | "errorRedirect": "https://some-url/error" + |}""".stripMargin) + + Given("a request containing a User-Agent header") + val headers = FakeHeaders(Seq((USER_AGENT, SomeConsumingService))) + val initiateRequest = FakeRequest(POST, uri = "/upscan/v2/initiate", headers, postBodyJson) + + When("a request is posted to the /initiate endpoint") + val initiateResponse = route(app, initiateRequest).get + + Then("the response should indicate success") + status(initiateResponse) shouldBe OK + + And("the response should include an upscan-proxy upload URL") + val responseJson = contentAsJson(initiateResponse) + (responseJson \ "uploadRequest" \ "href").as[String] shouldBe "https://upload-proxy.tax.service.gov.uk/v1/uploads/inbound-bucket" + + And("the response should contain the requested upload fields") + val fields = (responseJson \ "uploadRequest" \ "fields").as[Map[String, String]] + fields.get("x-amz-meta-callback-url") should contain ("https://some-url/callback") + fields.get("error_action_redirect") should contain ("https://some-url/error") + fields.get("success_action_redirect") shouldBe empty + fields.get("Content-Type") shouldBe empty } - private def prepareUploadTests( // scalastyle:ignore - postBodyJson: JsValue, - uri: String, - href: String): Unit = { + "Upscan V1" should { + val requestJson = Json.parse("""{"callbackUrl": "https://some-url/callback"}""") + + behave like upscanInitiate(uri = "/upscan/initiate", requestJson) + } - "include x-amz-meta-consuming-service in the response" in { + "Upscan V2" should { + val requestJson = Json.parse("""|{ + | "callbackUrl": "https://some-url/callback", + | "errorRedirect": "https://some-url/error" + |}""".stripMargin) + + behave like upscanInitiate(uri = "/upscan/v2/initiate", requestJson) + } + + //noinspection ScalaStyle + private def upscanInitiate(uri: String, requestJson: JsValue): Unit = { + "reject requests which do not include a User-Agent header" in { + Given("a request not containing a User-Agent header") + val initiateRequest = FakeRequest(POST, uri, FakeHeaders(Seq((xSessionId, "some-session-id"))), requestJson) + + When("a request is posted to the /initiate endpoint") + val initiateResponse = route(app, initiateRequest).get + + Then("the response should indicate the request is invalid") + status(initiateResponse) shouldBe BAD_REQUEST + } + + "include x-amz-meta-consuming-service to identify the client service" in { Given("a request containing a User-Agent header") - val initiateRequest = FakeRequest( - POST, - uri, - FakeHeaders(Seq((USER_AGENT, SomeConsumingService), (xSessionId, "some-session-id"))), - postBodyJson) + val initiateRequest = FakeRequest(POST, uri, FakeHeaders(Seq((USER_AGENT, SomeConsumingService))), requestJson) When("a request is posted to the /initiate endpoint") val initiateResponse = route(app, initiateRequest).get Then("the response should indicate success") - status(initiateResponse) shouldBe 200 + status(initiateResponse) shouldBe OK - And("the response should include the expected value for x-amz-meta-consuming-service") + And("the response should identify the client service") val responseJson = contentAsJson(initiateResponse) - (responseJson \ "uploadRequest" \ "fields" \ "x-amz-meta-consuming-service") - .as[String] shouldBe SomeConsumingService - - And("the href should be the expected url for the upload") - (responseJson \ "uploadRequest" \ "href") - .as[String] shouldBe href + (responseJson \ "uploadRequest" \ "fields" \ "x-amz-meta-consuming-service").as[String] shouldBe SomeConsumingService } - "include x-amz-meta-session-id in the response" in { + "include x-amz-meta-session-id in the response when a session exists" in { Given("a valid request containing a x-session-id header") - val initiateRequest = FakeRequest( - POST, - uri, - FakeHeaders(Seq((USER_AGENT, SomeConsumingService), (xSessionId, "some-session-id"))), - postBodyJson) + val headers = FakeHeaders(Seq((USER_AGENT, SomeConsumingService), (xSessionId, "some-session-id"))) + val initiateRequest = FakeRequest(POST, uri, headers, requestJson) When("a request is posted to the /initiate endpoint") val initiateResponse = route(app, initiateRequest).get @@ -103,19 +196,14 @@ class PrepareUploadControllerISpec extends AnyWordSpecLike with should.Matchers Then("the response should indicate success") status(initiateResponse) shouldBe 200 - And("the response should include the expected value for x-amz-meta-consuming-service") + And("the response should include the expected value for x-amz-meta-session-id") val responseJson = contentAsJson(initiateResponse) - (responseJson \ "uploadRequest" \ "fields" \ "x-amz-meta-session-id") - .as[String] shouldBe "some-session-id" - - And("the href should be the expected url for the upload") - (responseJson \ "uploadRequest" \ "href") - .as[String] shouldBe href + (responseJson \ "uploadRequest" \ "fields" \ "x-amz-meta-session-id").as[String] shouldBe "some-session-id" } - "set a default x-amz-meta-session-id in the response if no session id passed in" in { + "set a default x-amz-meta-session-id in the response if no session exists" in { Given("a request containing a x-session-id header") - val initiateRequest = FakeRequest(POST, uri, FakeHeaders(Seq((USER_AGENT, SomeConsumingService))), postBodyJson) + val initiateRequest = FakeRequest(POST, uri, FakeHeaders(Seq((USER_AGENT, SomeConsumingService))), requestJson) When("a request is posted to the /initiate endpoint") val initiateResponse = route(app, initiateRequest).get @@ -125,23 +213,30 @@ class PrepareUploadControllerISpec extends AnyWordSpecLike with should.Matchers And("the response should include the expected value for x-amz-meta-consuming-service") val responseJson = contentAsJson(initiateResponse) - (responseJson \ "uploadRequest" \ "fields" \ "x-amz-meta-session-id") - .as[String] shouldBe "n/a" - - And("the href should be the expected url for the upload") - (responseJson \ "uploadRequest" \ "href") - .as[String] shouldBe href + (responseJson \ "uploadRequest" \ "fields" \ "x-amz-meta-session-id").as[String] shouldBe "n/a" } - "reject requests which do not include a User-Agent header" in { - Given("a request not containing a User-Agent header") - val initiateRequest = FakeRequest(POST, uri, FakeHeaders(Seq((xSessionId, "some-session-id"))), postBodyJson) + "include standard upload fields" in { + Given("a request containing a User-Agent header") + val initiateRequest = FakeRequest(POST, uri, FakeHeaders(Seq((USER_AGENT, SomeConsumingService))), requestJson) When("a request is posted to the /initiate endpoint") val initiateResponse = route(app, initiateRequest).get - Then("the response should indicate the request is invalid") - status(initiateResponse) shouldBe BAD_REQUEST + Then("the response should indicate success") + status(initiateResponse) shouldBe OK + + And("the response should contain the standard upload fields") + val responseJson = contentAsJson(initiateResponse) + val reference = (responseJson \ "reference").as[String] + val fields = (responseJson \ "uploadRequest" \ "fields").as[Map[String, String]] + fields.get("key") should contain(reference) + fields.get("acl") should contain("private") + fields.get("x-amz-algorithm") should contain("AWS4-HMAC-SHA256") + fields should contain key "x-amz-date" + fields should contain key "x-amz-credential" + fields should contain key "x-amz-signature" + fields should contain key "policy" } } } diff --git a/test/controllers/model/PrepareUploadRequestV2Spec.scala b/test/controllers/model/PrepareUploadRequestV2Spec.scala new file mode 100644 index 0000000..e7c74f9 --- /dev/null +++ b/test/controllers/model/PrepareUploadRequestV2Spec.scala @@ -0,0 +1,81 @@ +/* + * Copyright 2020 HM Revenue & Customs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package controllers.model + +import org.scalatest.EitherValues +import play.api.libs.json.Json +import test.UnitSpec + + +class PrepareUploadRequestV2Spec extends UnitSpec with EitherValues { + + import PrepareUploadRequestV2Spec._ + + "A v2 upload request" when { + "specifying an upload success redirect URL" should { + "parse as a valid request containing a success redirect URL" in { + val request = s"""|{"callbackUrl":"$CallbackUrl", + | "successRedirect":"$SuccessRedirectUrl", + | "errorRedirect":"$ErrorRedirectUrl"}""".stripMargin + val parseResult = Json.parse(request).validate(PrepareUploadRequestV2.reads(MaxFileSize)) + + parseResult.asEither.map(_.successRedirect).right.value should contain (SuccessRedirectUrl) + } + + "generate upload settings that contain the success redirect URL" in { + val uploadSettings = aV2RequestWith(successRedirectUrl = Some(SuccessRedirectUrl)).toUploadSettings(UploadUrl) + + uploadSettings.successRedirect should contain (SuccessRedirectUrl) + } + } + + "omitting an upload success redirect URL" should { + "parse as a valid request without a success redirect URL" in { + val request = s"""|{"callbackUrl":"$CallbackUrl", + | "errorRedirect":"$ErrorRedirectUrl"}""".stripMargin + val parseResult = Json.parse(request).validate(PrepareUploadRequestV2.reads(MaxFileSize)) + + parseResult.asEither.map(_.successRedirect).right.value shouldBe empty + } + + "generate upload settings without a success redirect URL" in { + val uploadSettings = aV2RequestWith(successRedirectUrl = None).toUploadSettings(UploadUrl) + + uploadSettings.successRedirect shouldBe empty + } + } + } +} + +private object PrepareUploadRequestV2Spec { + val UploadUrl = "https://xxxx/upscan-upload-proxy/bucketName" + val CallbackUrl = "https://myservice.com/callback" + val SuccessRedirectUrl = "https://myservice.com/nextPage" + val ErrorRedirectUrl = "https://myservice.com/errorPage" + val MaxFileSize = 512 + + private val template = PrepareUploadRequestV2( + callbackUrl = CallbackUrl, + successRedirect = None, + errorRedirect = ErrorRedirectUrl, + minimumFileSize = Some(1), + maximumFileSize = Some(99), + expectedContentType = Some("application/pdf")) + + def aV2RequestWith(successRedirectUrl: Option[String]): PrepareUploadRequestV2 = + template.copy(successRedirect = successRedirectUrl) +} \ No newline at end of file From b20244e3da40ce64a5d6ddd3d071a9b920c1fc9d Mon Sep 17 00:00:00 2001 From: Nigel Perkins Date: Wed, 26 Aug 2020 12:23:37 +0100 Subject: [PATCH 56/96] BDOG-1006: v2 errorRedirect url is now optional --- README.md | 33 +- .../model/PrepareUploadRequestV1.scala | 2 +- .../model/PrepareUploadRequestV2.scala | 8 +- .../PrepareUploadControllerISpec.scala | 10 +- .../PrepareUploadControllerSpec.scala | 396 +++++++++--------- .../model/PrepareUploadRequestV2Spec.scala | 79 +++- 6 files changed, 303 insertions(+), 225 deletions(-) diff --git a/README.md b/README.md index 4174567..c343db9 100644 --- a/README.md +++ b/README.md @@ -59,7 +59,7 @@ Once the upload URL has been requested, upload and verification of a file are pe * Consuming service downloads the file using the provided URL or passes this URL on to another service which will make use of the file location. This download URL only remains valid for a limited duration (see [Success](#service__poutcome__success)) * After some time the file is automatically removed from the remote storage. Upscan does NOT keep files indefinitely * If the file fails a check, a notification is sent to the consuming service containing information on the failed check. The file is unavailable for retrieval -* If the consuming service fails to respond to the callback request (e.g. the consuming service is down, the consuming service answered with an HTTP status code other than 2xx), the callback will be retried up to a maximum of 30 retries. The time interval between the retries is 60 seconds +* If the consuming service fails to respond to the callback request (e.g. the consuming service is down, the consuming service answered with an HTTP status code other than 2xx), the callback will be retried up to a maximum of 30 retries. The time interval between the retries is 60 seconds. Configuration of these values is here (https://github.com/hmrc/upscan-infrastructure/blob/master/modules/sqs/main.tf) Please view the [Upscan Service & Flow Overview in Confluence](https://confluence.tools.tax.service.gov.uk/pages/viewpage.action?pageId=101663507) for a more visual representation. @@ -72,7 +72,7 @@ Please view the [Upscan Service & Flow Overview in Confluence](https://confluenc The consuming service makes a POST request to `/upscan/initiate` or `upscan/v2/initiate`. This request must contain a `User-Agent` header that can be used to identify the service, but an allow list of authorised services is no longer cross-checked. -The service must also provide a callbackUrl for asynchronous notification of the outcome of an upload. The callback will be made from inside the MDTP environment. Hence, the callback URL should comprise the MDTP internal callback address and not the public domain address. `upscan/v2/initiate` additionally requires an errorRedirect url. See next section for specifics. +The service must also provide a callbackUrl for asynchronous notification of the outcome of an upload. The callback will be made from inside the MDTP environment. Hence, the callback URL should comprise the MDTP internal callback address and not the public domain address. **Note:** `callbackUrl` must use the `https` protocol. (Although this rule is relaxed when testing locally with [upscan-stub](https://github.com/hmrc/upscan-stub) rather than [upscan-initiate](https://github.com/hmrc/upscan-initiate). @@ -118,7 +118,7 @@ Example `upscan/v2/initiate` request: |--------------|-----------|--------| |callbackUrl |Url that will be called to report the outcome of file checking and upload, including retrieval details if successful. Notification format is detailed further down in this file. Must be https.| yes| |successRedirect|Url to redirect to after file has been successfully uploaded.|no| -|errorRedirect|Url to redirect to if error encountered during upload.|yes| +|errorRedirect|Url to redirect to if error encountered during upload.|no| |minimumFileSize|Minimum file size (in Bytes). Default is 0.|no| |maximumFileSize|Maximum file size (in Bytes). Cannot be greater than 100MB. Default is 100MB.|no| |expectedContentType|MIME type describing the upload contents.|no| @@ -163,22 +163,39 @@ S3 will return errors in `application/xml` in the following format: ``` -Example [upscan-upload-proxy](https://github.com/hmrc/upscan-upload-proxy) error redirect response: +For v2, how such an error is returned depends upon whether the `error_action_redirect` form field was set. -Redirect Response: +If set, the [upscan-upload-proxy](https://github.com/hmrc/upscan-upload-proxy) will redirect to the specified URL. +Details of the error will be supplied to this URL as query parameters, with the names `errorCode`, `errorMessage`, `errorResource` and `errorRequestId`. + +The query parameter named `key` contains the globally unique file reference that was allocated by the initiate request +to identify the upload. ``` HTTP Response Code: 303 -Header ("Location" -> "https://myservice.com/errorPage?key=11370e18-6e24-453e-b45a-76d3e32ea33d&errorCode=NoSuchKey&errorMessage=The+resource+you+requested+does+not+exist&errorResource=/mybucket/myfoto.jpg&errorRequestId=4442587FB7D0A2F9") +Header ("Location" -> "https://myservice.com/errorPage?key=11370e18-6e24-453e-b45a-76d3e32ea33d&errorCode=NoSuchKey&errorMessage=The+resource+you+requested+does+not+exist&errorResource=/mybucket/myfoto.jpg&errorRequestId=4442587FB7D0A2F9") ``` +If a redirect URL is not set, the proxy will respond with the failure status code. +The details of the error along with the key will be available from the JSON body that has the following structure: + +``` +{ + "key": "11370e18-6e24-453e-b45a-76d3e32ea33d", + "errorCode": "NoSuchKey", + "errorMessage": "The resource you requested does not exist", + "errorResource": "/mybucket/myfoto.jpg", + "errorRequestId": "4442587FB7D0A2F9" +} +``` -Note that this Location header comprises the errorRedirect URL supplemented with additional information about the error by way of query parameters. The query parameter named "key" contains the globally unique file reference that was allocated by the initiate request to identify the upload. +All error fields are optional. +If an `error_action_redirect` is specified that does not represent a valid URL, the response will Bad Request: Json Error Response (can not redirect): ```json -{"message":"Bad request"} +{"message":"Unable to build valid redirect URL for error action"} ``` diff --git a/app/controllers/model/PrepareUploadRequestV1.scala b/app/controllers/model/PrepareUploadRequestV1.scala index f299d34..7abb64a 100644 --- a/app/controllers/model/PrepareUploadRequestV1.scala +++ b/app/controllers/model/PrepareUploadRequestV1.scala @@ -46,7 +46,7 @@ object PrepareUploadRequestV1 { def reads(maxFileSize: Int): Reads[PrepareUploadRequestV1] = ((JsPath \ "callbackUrl").read[String] and (JsPath \ "minimumFileSize").readNullable[Int](min(0)) and - (JsPath \ "maximumFileSize").readNullable[Int](min(0) keepAnd max(maxFileSize + 1)) and + (JsPath \ "maximumFileSize").readNullable[Int](min(0) keepAnd max(maxFileSize)) and (JsPath \ "expectedContentType").readNullable[String] and (JsPath \ "successRedirect").readNullable[String])(PrepareUploadRequestV1.apply _) .filter(JsonValidationError("Maximum file size must be equal or greater than minimum file size"))(request => diff --git a/app/controllers/model/PrepareUploadRequestV2.scala b/app/controllers/model/PrepareUploadRequestV2.scala index 0a88c5c..9622758 100644 --- a/app/controllers/model/PrepareUploadRequestV2.scala +++ b/app/controllers/model/PrepareUploadRequestV2.scala @@ -24,7 +24,7 @@ import services.model.UploadSettings case class PrepareUploadRequestV2( callbackUrl: String, successRedirect: Option[String], - errorRedirect: String, + errorRedirect: Option[String], minimumFileSize: Option[Int], maximumFileSize: Option[Int], expectedContentType: Option[String]) @@ -37,7 +37,7 @@ case class PrepareUploadRequestV2( maximumFileSize = maximumFileSize, expectedContentType = expectedContentType, successRedirect = successRedirect, - errorRedirect = Some(errorRedirect) + errorRedirect = errorRedirect ) } @@ -46,9 +46,9 @@ object PrepareUploadRequestV2 { def reads(maxFileSize: Int): Reads[PrepareUploadRequestV2] = ((JsPath \ "callbackUrl").read[String] and (JsPath \ "successRedirect").readNullable[String] and - (JsPath \ "errorRedirect").read[String] and + (JsPath \ "errorRedirect").readNullable[String] and (JsPath \ "minimumFileSize").readNullable[Int](min(0)) and - (JsPath \ "maximumFileSize").readNullable[Int](min(0) keepAnd max(maxFileSize + 1)) and + (JsPath \ "maximumFileSize").readNullable[Int](min(0) keepAnd max(maxFileSize)) and (JsPath \ "expectedContentType").readNullable[String])(PrepareUploadRequestV2.apply _) .filter(JsonValidationError("Maximum file size must be equal or greater than minimum file size"))(request => request.minimumFileSize.getOrElse(0) <= request.maximumFileSize.getOrElse(maxFileSize)) diff --git a/it/controllers/PrepareUploadControllerISpec.scala b/it/controllers/PrepareUploadControllerISpec.scala index b46cc3e..0e63738 100644 --- a/it/controllers/PrepareUploadControllerISpec.scala +++ b/it/controllers/PrepareUploadControllerISpec.scala @@ -116,8 +116,7 @@ class PrepareUploadControllerISpec extends AnyWordSpecLike with should.Matchers "PrepareUploadController prepareUploadV2 with only mandatory request values" in { val postBodyJson = Json.parse("""|{ - | "callbackUrl": "https://some-url/callback", - | "errorRedirect": "https://some-url/error" + | "callbackUrl": "https://some-url/callback" |}""".stripMargin) Given("a request containing a User-Agent header") @@ -137,7 +136,7 @@ class PrepareUploadControllerISpec extends AnyWordSpecLike with should.Matchers And("the response should contain the requested upload fields") val fields = (responseJson \ "uploadRequest" \ "fields").as[Map[String, String]] fields.get("x-amz-meta-callback-url") should contain ("https://some-url/callback") - fields.get("error_action_redirect") should contain ("https://some-url/error") + fields.get("error_action_redirect") shouldBe empty fields.get("success_action_redirect") shouldBe empty fields.get("Content-Type") shouldBe empty } @@ -149,10 +148,7 @@ class PrepareUploadControllerISpec extends AnyWordSpecLike with should.Matchers } "Upscan V2" should { - val requestJson = Json.parse("""|{ - | "callbackUrl": "https://some-url/callback", - | "errorRedirect": "https://some-url/error" - |}""".stripMargin) + val requestJson = Json.parse("""{"callbackUrl": "https://some-url/callback"}""") behave like upscanInitiate(uri = "/upscan/v2/initiate", requestJson) } diff --git a/test/controllers/PrepareUploadControllerSpec.scala b/test/controllers/PrepareUploadControllerSpec.scala index 5428c96..0780b8e 100644 --- a/test/controllers/PrepareUploadControllerSpec.scala +++ b/test/controllers/PrepareUploadControllerSpec.scala @@ -16,7 +16,7 @@ package controllers -import java.time.{Clock, Instant} +import java.time.Clock import akka.actor.ActorSystem import akka.stream.ActorMaterializer @@ -25,7 +25,7 @@ import controllers.model.{PreparedUploadResponse, Reference, UploadFormTemplate} import org.scalatest.GivenWhenThen import play.api.http.HeaderNames.USER_AGENT import play.api.http.Status.{BAD_REQUEST, OK} -import play.api.libs.json.{JsObject, JsValue, Json} +import play.api.libs.json.{JsValue, Json} import play.api.mvc.Action import play.api.mvc.Results.Ok import play.api.test.Helpers.{contentAsString, status} @@ -33,7 +33,7 @@ import play.api.test.{FakeRequest, Helpers, StubControllerComponentsFactory} import services.PrepareUploadService import services.model.UploadSettings import test.UnitSpec - +import Helpers.contentAsJson import scala.concurrent.Future import scala.concurrent.duration._ import scala.language.postfixOps @@ -41,216 +41,254 @@ import scala.language.postfixOps class PrepareUploadControllerSpec extends UnitSpec with StubControllerComponentsFactory with GivenWhenThen { private implicit val actorSystem: ActorSystem = ActorSystem() - private implicit val materializer: ActorMaterializer = ActorMaterializer() - private implicit val timeout: akka.util.Timeout = 10 seconds - private val clock: Clock = Clock.systemDefaultZone() + private val clock = Clock.fixed(Clock.systemDefaultZone().instant(), Clock.systemDefaultZone().getZone) - "PrepareUploadController prepareUploadV1" should { - - behave like prepareUploadTests(_.prepareUploadV1()) + private trait WithGlobalFileSizeLimitFixture { + val GlobalFileSizeLimit = 1024 + val prepareUploadService = mock[PrepareUploadService] + when(prepareUploadService.globalFileSizeLimit).thenReturn(GlobalFileSizeLimit) } - "PrepareUploadController prepareUploadV2" should { - - val extraRequestFields = Json - .obj("successRedirect" -> "https://www.example.com/nextpage", "errorRedirect" -> "https://www.example.com/error") - - val extraResponseFields = Json.obj( - "success_action_redirect" -> "https://www.example.com/nextpage", - "error_action_redirect" -> "https://www.example.com/error") - - behave like prepareUploadTests(_.prepareUploadV2(), extraRequestFields, extraResponseFields) + private trait WithServiceConfiguration { + val config = mock[ServiceConfiguration] } - private def prepareUploadTests( // scalastyle:ignore - prepareUploadAction: PrepareUploadController => Action[JsValue], - extraRequestFields: JsObject = JsObject(Seq()), - extraResponseFields: JsObject = JsObject(Seq())) { - - val config = mock[ServiceConfiguration] + private trait WithAllowedCallbackProtocol { this: WithServiceConfiguration => when(config.allowedCallbackProtocols).thenReturn(List("https")) + } - "build and return upload URL if valid request with all data" in { - val controller = new PrepareUploadController(prepareUploadServiceWithUpload, config, clock, stubControllerComponents()) - - Given("there is a valid upload request with all data") - - val request: FakeRequest[JsValue] = FakeRequest() - .withHeaders( - (USER_AGENT, "SOME-USER-AGENT"), - ("x-session-id", "some-session-id"), - ("x-request-id", "some-request-id")) - .withBody( - Json.obj( - "id" -> "1", - "callbackUrl" -> "https://www.example.com", - "minimumFileSize" -> 0, - "maximumFileSize" -> 1024) ++ extraRequestFields) - - When("upload initiation has been requested") + private trait WithInboundBucketName { this: WithServiceConfiguration => + when(config.inboundBucketName).thenReturn("inbound-bucket") + } - val result = prepareUploadAction(controller)(request) + private trait WithUploadProxyUrl { this: WithServiceConfiguration => + when(config.uploadProxyUrl).thenReturn("https://upload-proxy.com") + } - Then("service returns valid response with reference and template of upload form") - - withClue(Helpers.contentAsString(result)) { status(result) shouldBe OK } - val json = Helpers.contentAsJson(result) - json shouldBe Json.obj( - "reference" -> "TEST", - "uploadRequest" -> Json.obj( - "href" -> "https://www.example.com", - "fields" -> (Json.obj( - "minFileSize" -> "0", - "maxFileSize" -> "1024", - "sessionId" -> "some-session-id", - "requestId" -> "some-request-id") ++ extraResponseFields) - ) - ) + private trait WithAllowedCallbackProtocolFixture extends WithServiceConfiguration with WithAllowedCallbackProtocol + private trait WithV1SuccessFixture extends WithAllowedCallbackProtocolFixture with WithInboundBucketName + private trait WithV2SuccessFixture extends WithV1SuccessFixture with WithUploadProxyUrl + private trait WithV1BadRequestFixture extends WithServiceConfiguration with WithInboundBucketName + private trait WithV2BadRequestFixture extends WithV1BadRequestFixture with WithUploadProxyUrl + + "V1 initiate with minimal request settings" should { + val minimalRequestBody = Json.obj("callbackUrl" -> "https://www.example.com") + val expectedUploadSettings = UploadSettings( + uploadUrl = "https://inbound-bucket.s3.amazonaws.com", + callbackUrl = "https://www.example.com", + minimumFileSize = None, + maximumFileSize = None, + expectedContentType = None, + successRedirect = None, + errorRedirect = None + ) + + val _ = new WithV1SuccessFixture { + behave like successfulInitiate(config, _.prepareUploadV1(), minimalRequestBody, requestId = None, sessionId = None, + expectedUploadSettings) } + } - "build and return upload URL if valid request with redirect on success url" in { - val controller = new PrepareUploadController(prepareUploadServiceWithUpload, config, clock, stubControllerComponents()) - - Given("there is a valid upload request with all data") - - val request: FakeRequest[JsValue] = FakeRequest() - .withHeaders( - (USER_AGENT, "SOME-USER-AGENT"), - ("x-session-id", "some-session-id"), - ("x-request-id", "some-request-id")) - .withBody( - Json.obj( - "id" -> "1", - "callbackUrl" -> "https://www.example.com", - "successRedirect" -> "https://www.example.com/nextpage", - "minimumFileSize" -> 0, - "maximumFileSize" -> 1024) ++ extraRequestFields) + "V1 initiate with all request settings" should { + val maximalRequestBody = Json.obj( + "callbackUrl" -> "https://www.example.com", + "minimumFileSize" -> 1, + "maximumFileSize" -> 1024, + "expectedContentType" -> "application/pdf", + "successRedirect" -> "https://www.example.com/success" + ) + val expectedUploadSettings = UploadSettings( + uploadUrl = "https://inbound-bucket.s3.amazonaws.com", + callbackUrl = "https://www.example.com", + minimumFileSize = Some(1), + maximumFileSize = Some(1024), + expectedContentType = Some("application/pdf"), + successRedirect = Some("https://www.example.com/success"), + errorRedirect = None + ) + + val _ = new WithV1SuccessFixture { + behave like successfulInitiate(config, _.prepareUploadV1(), maximalRequestBody, requestId = Some("a-request-id"), + sessionId = Some("a-session-id"), expectedUploadSettings) + } + } - When("upload initiation has been requested") + "V2 initiate with minimal request settings" should { + val minimalRequestBody = Json.obj("callbackUrl" -> "https://www.example.com") + val expectedUploadSettings = UploadSettings( + uploadUrl = "https://upload-proxy.com/v1/uploads/inbound-bucket", + callbackUrl = "https://www.example.com", + minimumFileSize = None, + maximumFileSize = None, + expectedContentType = None, + successRedirect = None, + errorRedirect = None + ) + + val _ = new WithV2SuccessFixture { + behave like successfulInitiate(config, _.prepareUploadV2(), minimalRequestBody, requestId = None, sessionId = None, + expectedUploadSettings) + } + } - val result = prepareUploadAction(controller)(request) + "V2 initiate with all request settings" should { + val maximalRequestBody = Json.obj( + "callbackUrl" -> "https://www.example.com", + "minimumFileSize" -> 1, + "maximumFileSize" -> 1024, + "expectedContentType" -> "application/pdf", + "successRedirect" -> "https://www.example.com/success", + "errorRedirect" -> "https://www.example.com/error" + ) + val expectedUploadSettings = UploadSettings( + uploadUrl = "https://upload-proxy.com/v1/uploads/inbound-bucket", + callbackUrl = "https://www.example.com", + minimumFileSize = Some(1), + maximumFileSize = Some(1024), + expectedContentType = Some("application/pdf"), + successRedirect = Some("https://www.example.com/success"), + errorRedirect = Some("https://www.example.com/error") + ) + + val _ = new WithV2SuccessFixture { + behave like successfulInitiate(config, _.prepareUploadV2(), maximalRequestBody, requestId = Some("a-request-id"), + sessionId = Some("a-session-id"), expectedUploadSettings) + } + } - Then("service returns valid response with reference and template of upload form") - - withClue(Helpers.contentAsString(result)) { status(result) shouldBe OK } - val json = Helpers.contentAsJson(result) - json shouldBe Json.obj( - "reference" -> "TEST", - "uploadRequest" -> Json.obj( - "href" -> "https://www.example.com", - "fields" -> (Json.obj( - "minFileSize" -> "0", - "maxFileSize" -> "1024", - "sessionId" -> "some-session-id", - "requestId" -> "some-request-id", - "success_action_redirect" -> "https://www.example.com/nextpage" - ) ++ extraResponseFields) + //noinspection ScalaStyle + private def successfulInitiate(config: ServiceConfiguration, + actionUnderTest: PrepareUploadController => Action[JsValue], + requestBody: JsValue, + requestId: Option[String], + sessionId: Option[String], + expectedUploadSettings: UploadSettings): Unit = { + + "prepare upload request" in new WithGlobalFileSizeLimitFixture { + val controller = new PrepareUploadController(prepareUploadService, config, clock, stubControllerComponents()) + + Given("a valid initiate request") + val headers = (USER_AGENT, "SOME-USER-AGENT") +: Seq( + requestId.map(Tuple2("x-request-id", _)), + sessionId.map(Tuple2("x-session-id", _)) + ).flatten + val request = FakeRequest().withHeaders(headers: _*).withBody(requestBody) + + val expectedUploadResponse = PreparedUploadResponse( + reference = Reference("TEST"), + uploadRequest = UploadFormTemplate( + href = expectedUploadSettings.uploadUrl, + fields = Map("a" -> "b", "x" -> "y") ) ) - } - "build and return upload URL if valid request with minimal data including session id and request id" in { - val controller = new PrepareUploadController(prepareUploadServiceWithUpload, config, clock, stubControllerComponents()) + when(prepareUploadService.prepareUpload( + expectedUploadSettings, + "SOME-USER-AGENT", + requestId.getOrElse("n/a"), + sessionId.getOrElse("n/a"), + clock.instant) + ).thenReturn(expectedUploadResponse) - Given("there is a valid upload request with minimal data") + When("upload initiation is requested") + val result = actionUnderTest(controller)(request) - val request: FakeRequest[JsValue] = FakeRequest() - .withHeaders( - (USER_AGENT, "SOME-USER-AGENT"), - ("x-session-id", "some-session-id"), - ("x-request-id", "some-request-id")) - .withBody(Json.obj("callbackUrl" -> "https://www.example.com") ++ extraRequestFields) - - When("upload initiation has been requested") + Then("a response containing a reference and template of the upload form is returned") + status(result) shouldBe OK - val result = prepareUploadAction(controller)(request) + val expectedFieldsAsJson = expectedUploadResponse.uploadRequest.fields.toSeq.map { kvPair => + s""""${kvPair._1}": "${kvPair._2}"""" + }.mkString("{", ",", "}") + + contentAsJson(result) shouldBe Json.parse( + s"""|{ + | "reference": "${expectedUploadResponse.reference.value}", + | "uploadRequest": { + | "href": "${expectedUploadResponse.uploadRequest.href}", + | "fields": $expectedFieldsAsJson + | } + |}""".stripMargin) + } + } - Then("service returns valid response with reference and template of upload form") + "V1 bad initiate request" should { + val _ = new WithV1BadRequestFixture { + behave like badRequestInitiate(config, _.prepareUploadV1()) + } + } - withClue(Helpers.contentAsString(result)) { status(result) shouldBe OK } - val json = Helpers.contentAsJson(result) - json shouldBe Json.obj( - "reference" -> "TEST", - "uploadRequest" -> Json.obj( - "href" -> "https://www.example.com", - "fields" -> (Json - .obj("sessionId" -> "some-session-id", "requestId" -> "some-request-id") ++ extraResponseFields) - ) - ) + "V2 bad initiate request" should { + val _ = new WithV2BadRequestFixture { + behave like badRequestInitiate(config, _.prepareUploadV2()) } + } - "build and return upload URL if valid request with minimal data excluding session id and request id" in { - val controller = new PrepareUploadController(prepareUploadServiceWithUpload, config, clock, stubControllerComponents()) + private def badRequestInitiate(// scalastyle:ignore + config: ServiceConfiguration, + prepareUploadAction: PrepareUploadController => Action[JsValue]) { - Given("there is a valid upload request with minimal data") + "return a bad request error when the request has the wrong structure" in new WithGlobalFileSizeLimitFixture { + val controller = new PrepareUploadController(prepareUploadService, config, clock, stubControllerComponents()) - val request: FakeRequest[JsValue] = FakeRequest() - .withHeaders((USER_AGENT, "SOME-USER-AGENT")) - .withBody(Json.obj("callbackUrl" -> "https://www.example.com") ++ extraRequestFields) + Given("there is an invalid upload request") + val request = FakeRequest().withHeaders( + (USER_AGENT, "SOME-USER-AGENT"), + ("x-session-id", "some-session-id"), + ("x-request-id", "some-request-id") + ).withBody(Json.obj("invalid" -> "body")) When("upload initiation has been requested") - val result = prepareUploadAction(controller)(request) - Then("service returns valid response with reference and template of upload form") - - withClue(Helpers.contentAsString(result)) { status(result) shouldBe OK } - val json = Helpers.contentAsJson(result) - json shouldBe Json.obj( - "reference" -> "TEST", - "uploadRequest" -> Json.obj( - "href" -> "https://www.example.com", - "fields" -> (Json.obj("sessionId" -> "n/a", "requestId" -> "n/a") ++ extraResponseFields) - ) - ) + Then("service returns error response") + withClue(Helpers.contentAsString(result)) { status(result) shouldBe BAD_REQUEST } } - "return a bad request error if invalid request - wrong structure" in { - val controller = new PrepareUploadController(prepareUploadServiceNoUpload, config, clock, stubControllerComponents()) + "return a bad request error when the maximum file size is in excess of the global maximum" in new WithGlobalFileSizeLimitFixture { + val controller = new PrepareUploadController(prepareUploadService, config, clock, stubControllerComponents()) Given("there is an invalid upload request") - - val request: FakeRequest[JsValue] = FakeRequest() - .withHeaders( - (USER_AGENT, "SOME-USER-AGENT"), - ("x-session-id", "some-session-id"), - ("x-request-id", "some-request-id")) - .withBody(Json.obj("invalid" -> "body")) + val request = FakeRequest().withHeaders( + (USER_AGENT, "SOME-USER-AGENT"), + ("x-session-id", "some-session-id") + ).withBody(Json.obj( + "callbackUrl" -> "https://www.example.com", + "maximumFileSize" -> (GlobalFileSizeLimit + 1)) + ) When("upload initiation has been requested") - val result = prepareUploadAction(controller)(request) Then("service returns error response") - - withClue(Helpers.contentAsString(result)) { status(result) shouldBe BAD_REQUEST } + withClue(Helpers.contentAsString(result)) { + status(result) shouldBe BAD_REQUEST + } } - "return a bad request error if invalid request - incorrect maximum file size " in { - val controller = new PrepareUploadController(prepareUploadServiceNoUpload, config, clock, stubControllerComponents()) + "return a bad request error when no user agent is supplied" in new WithGlobalFileSizeLimitFixture { + val controller = new PrepareUploadController(prepareUploadService, config, clock, stubControllerComponents()) - Given("there is an invalid upload request") - - val request: FakeRequest[JsValue] = FakeRequest() - .withHeaders((USER_AGENT, "SOME-USER-AGENT"), ("x-session-id", "some-session-id")) - .withBody(Json.obj("callbackUrl" -> "https://www.example.com", "maximumFileSize" -> 2048)) + Given("there is an upload request without a user agent header") + val request = FakeRequest().withHeaders( + ("x-session-id", "some-session-id") + ).withBody(Json.obj("callbackUrl" -> "https://www.example.com")) When("upload initiation has been requested") - val result = prepareUploadAction(controller)(request) Then("service returns error response") - withClue(Helpers.contentAsString(result)) { status(result) shouldBe BAD_REQUEST } } + } - "allow https callback urls" in { - val controller = new PrepareUploadController(prepareUploadServiceNoUpload, config, clock, stubControllerComponents()) + "withAllowedCallbackProtocol" should { + "allow https callback urls" in new WithGlobalFileSizeLimitFixture with WithAllowedCallbackProtocolFixture { + val controller = new PrepareUploadController(prepareUploadService, config, clock, stubControllerComponents()) val result = controller.withAllowedCallbackProtocol("https://my.callback.url") { Future.successful(Ok) @@ -259,54 +297,26 @@ class PrepareUploadControllerSpec extends UnitSpec with StubControllerComponents status(result) shouldBe OK } - "disallow http callback urls" in { - val controller = new PrepareUploadController(prepareUploadServiceNoUpload, config, clock, stubControllerComponents()) + "disallow http callback urls" in new WithGlobalFileSizeLimitFixture with WithAllowedCallbackProtocolFixture { + val controller = new PrepareUploadController(prepareUploadService, config, clock, stubControllerComponents()) val result = controller.withAllowedCallbackProtocol("http://my.callback.url") { Future.failed(new RuntimeException("This block should not have been invoked.")) } - status(result) shouldBe BAD_REQUEST + status(result) shouldBe BAD_REQUEST contentAsString(result) should include("Invalid callback url protocol") } - "disallow invalidly formatted callback urls" in { - val controller = new PrepareUploadController(prepareUploadServiceNoUpload, config, clock, stubControllerComponents()) + "disallow invalidly formatted callback urls" in new WithGlobalFileSizeLimitFixture with WithAllowedCallbackProtocolFixture { + val controller = new PrepareUploadController(prepareUploadService, config, clock, stubControllerComponents()) val result = controller.withAllowedCallbackProtocol("123") { Future.failed(new RuntimeException("This block should not have been invoked.")) } - status(result) shouldBe BAD_REQUEST + + status(result) shouldBe BAD_REQUEST contentAsString(result) should include("Invalid callback url format") } } - - private def prepareUploadServiceNoUpload: PrepareUploadService = { - val service = mock[PrepareUploadService] - when(service.globalFileSizeLimit).thenReturn(1024) - } - - private def prepareUploadServiceWithUpload: PrepareUploadService = { - val service = mock[PrepareUploadService] - when(service.globalFileSizeLimit).thenReturn(1024) - when(service.prepareUpload(any[UploadSettings], any[String], any[String], any[String], any[Instant])) - .thenAnswer{ (settings: UploadSettings, _: String, requestId: String, sessionId: String, _: Instant) => - PreparedUploadResponse( - Reference("TEST"), - UploadFormTemplate( - settings.callbackUrl, - Map.empty ++ - settings.minimumFileSize.map(s => Map("minFileSize" -> s.toString).head) ++ - settings.maximumFileSize.map(s => Map("maxFileSize" -> s.toString).head) ++ - Map("sessionId" -> sessionId) ++ - Map("requestId" -> requestId) ++ - settings.successRedirect.map(url => Map("success_action_redirect" -> url)).getOrElse(Map.empty) ++ - settings.errorRedirect.map(url => Map("error_action_redirect" -> url)).getOrElse(Map.empty) - ) - ) - } - - service - } - } diff --git a/test/controllers/model/PrepareUploadRequestV2Spec.scala b/test/controllers/model/PrepareUploadRequestV2Spec.scala index e7c74f9..edeb99a 100644 --- a/test/controllers/model/PrepareUploadRequestV2Spec.scala +++ b/test/controllers/model/PrepareUploadRequestV2Spec.scala @@ -29,15 +29,14 @@ class PrepareUploadRequestV2Spec extends UnitSpec with EitherValues { "specifying an upload success redirect URL" should { "parse as a valid request containing a success redirect URL" in { val request = s"""|{"callbackUrl":"$CallbackUrl", - | "successRedirect":"$SuccessRedirectUrl", - | "errorRedirect":"$ErrorRedirectUrl"}""".stripMargin + | "successRedirect":"$SuccessRedirectUrl"}""".stripMargin val parseResult = Json.parse(request).validate(PrepareUploadRequestV2.reads(MaxFileSize)) parseResult.asEither.map(_.successRedirect).right.value should contain (SuccessRedirectUrl) } "generate upload settings that contain the success redirect URL" in { - val uploadSettings = aV2RequestWith(successRedirectUrl = Some(SuccessRedirectUrl)).toUploadSettings(UploadUrl) + val uploadSettings = aV2RequestWithSuccessRedirectUrlOf(Some(SuccessRedirectUrl)).toUploadSettings(UploadUrl) uploadSettings.successRedirect should contain (SuccessRedirectUrl) } @@ -45,19 +44,72 @@ class PrepareUploadRequestV2Spec extends UnitSpec with EitherValues { "omitting an upload success redirect URL" should { "parse as a valid request without a success redirect URL" in { - val request = s"""|{"callbackUrl":"$CallbackUrl", - | "errorRedirect":"$ErrorRedirectUrl"}""".stripMargin + val request = s"""{"callbackUrl":"$CallbackUrl"}""" val parseResult = Json.parse(request).validate(PrepareUploadRequestV2.reads(MaxFileSize)) parseResult.asEither.map(_.successRedirect).right.value shouldBe empty } "generate upload settings without a success redirect URL" in { - val uploadSettings = aV2RequestWith(successRedirectUrl = None).toUploadSettings(UploadUrl) + val uploadSettings = aV2RequestWithSuccessRedirectUrlOf(None).toUploadSettings(UploadUrl) uploadSettings.successRedirect shouldBe empty } } + + "specifying an error redirect URL" should { + "parse as a valid request containing an error redirect URL" in { + val request = s"""|{"callbackUrl":"$CallbackUrl", + | "errorRedirect":"$ErrorRedirectUrl"}""".stripMargin + val parseResult = Json.parse(request).validate(PrepareUploadRequestV2.reads(MaxFileSize)) + + parseResult.asEither.map(_.errorRedirect).right.value should contain (ErrorRedirectUrl) + } + + "generate upload settings that contain the error redirect URL" in { + val uploadSettings = aV2RequestWithErrorRedirectUrlOf(Some(ErrorRedirectUrl)).toUploadSettings(UploadUrl) + + uploadSettings.errorRedirect should contain (ErrorRedirectUrl) + } + } + + "omitting an error redirect URL" should { + "parse as a valid request without an error redirect URL" in { + val request = s"""{"callbackUrl":"$CallbackUrl"}""" + val parseResult = Json.parse(request).validate(PrepareUploadRequestV2.reads(MaxFileSize)) + + parseResult.asEither.map(_.errorRedirect).right.value shouldBe empty + } + + "generate upload settings without an error redirect URL" in { + val uploadSettings = aV2RequestWithErrorRedirectUrlOf(None).toUploadSettings(UploadUrl) + + uploadSettings.errorRedirect shouldBe empty + } + } + + "specifying a maximum file size" should { + "be accepted when equal to the global maximum" in { + val request = s"""{"callbackUrl":"$CallbackUrl", "maximumFileSize": $MaxFileSize}""" + val parseResult = Json.parse(request).validate(PrepareUploadRequestV2.reads(MaxFileSize)) + + parseResult.asEither.map(_.maximumFileSize).right.value should contain (MaxFileSize) + } + + "be rejected when greater than the global maximum" in { + val request = s"""{"callbackUrl":"$CallbackUrl", "maximumFileSize": ${MaxFileSize + 1}}""" + val parseResult = Json.parse(request).validate(PrepareUploadRequestV2.reads(MaxFileSize)) + + parseResult.isError shouldBe true + } + + "be rejected when less than the specified minimumFileSize" in { + val request = s"""{"callbackUrl":"$CallbackUrl", "minimumFileSize": $MaxFileSize, "maximumFileSize": ${MaxFileSize - 1}}""" + val parseResult = Json.parse(request).validate(PrepareUploadRequestV2.reads(MaxFileSize)) + + parseResult.isError shouldBe true + } + } } } @@ -71,11 +123,14 @@ private object PrepareUploadRequestV2Spec { private val template = PrepareUploadRequestV2( callbackUrl = CallbackUrl, successRedirect = None, - errorRedirect = ErrorRedirectUrl, - minimumFileSize = Some(1), - maximumFileSize = Some(99), - expectedContentType = Some("application/pdf")) + errorRedirect = None, + minimumFileSize = None, + maximumFileSize = None, + expectedContentType = None) + + def aV2RequestWithSuccessRedirectUrlOf(url: Option[String]): PrepareUploadRequestV2 = + template.copy(successRedirect = url) - def aV2RequestWith(successRedirectUrl: Option[String]): PrepareUploadRequestV2 = - template.copy(successRedirect = successRedirectUrl) + def aV2RequestWithErrorRedirectUrlOf(url: Option[String]): PrepareUploadRequestV2 = + template.copy(errorRedirect = url) } \ No newline at end of file From f36ba0c553452b65e40cc1e23dca1f9c56afa4d6 Mon Sep 17 00:00:00 2001 From: Nigel Perkins Date: Wed, 2 Sep 2020 11:51:50 +0100 Subject: [PATCH 57/96] BDOG-1006: housekeeping - remove sbt-artifactory & uplift versions --- README.md | 3 +-- build.sbt | 4 ++-- project/AppDependencies.scala | 4 ++-- project/build.properties | 2 +- project/plugins.sbt | 8 +++----- 5 files changed, 9 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index c343db9..1ef0284 100644 --- a/README.md +++ b/README.md @@ -191,8 +191,7 @@ The details of the error along with the key will be available from the JSON body All error fields are optional. -If an `error_action_redirect` is specified that does not represent a valid URL, the response will Bad Request: -Json Error Response (can not redirect): +If an `error_action_redirect` is specified that does not represent a valid URL, the response will be a Bad Request: ```json {"message":"Unable to build valid redirect URL for error action"} diff --git a/build.sbt b/build.sbt index 0a80553..0474c47 100644 --- a/build.sbt +++ b/build.sbt @@ -20,7 +20,7 @@ lazy val scoverageSettings = Seq( ) lazy val microservice = Project(appName, file(".")) - .enablePlugins(Seq(play.sbt.PlayScala, SbtAutoBuildPlugin, SbtGitVersioning, SbtDistributablesPlugin, SbtArtifactory): _*) + .enablePlugins(Seq(play.sbt.PlayScala, SbtAutoBuildPlugin, SbtGitVersioning, SbtDistributablesPlugin): _*) .disablePlugins(JUnitXmlReportPlugin) //Required to prevent https://github.com/scalatest/scalatest/issues/1427 .settings(scoverageSettings: _*) .settings(majorVersion := 0) @@ -29,7 +29,7 @@ lazy val microservice = Project(appName, file(".")) .settings(libraryDependencies ++= AppDependencies()) .settings(resolvers += Resolver.jcenterRepo) .settings(scalacOptions += "-target:jvm-1.8") - .settings(scalaVersion := "2.12.10") + .settings(scalaVersion := "2.12.12") .settings(Test / parallelExecution := false) .configs(IntegrationTest) .settings(integrationTestSettings(): _*) \ No newline at end of file diff --git a/project/AppDependencies.scala b/project/AppDependencies.scala index 7ba2c0f..fa25431 100644 --- a/project/AppDependencies.scala +++ b/project/AppDependencies.scala @@ -4,8 +4,8 @@ object AppDependencies { import play.core.PlayVersion private val compile = Seq( - "uk.gov.hmrc" %% "bootstrap-backend-play-27" % "2.14.0", - "com.typesafe.play" %% "play-json" % PlayVersion.current, + "uk.gov.hmrc" %% "bootstrap-backend-play-27" % "2.24.0", + "com.typesafe.play" %% "play-json" % "2.7.4", "com.amazonaws" % "aws-java-sdk-s3" % "1.11.769", "org.apache.commons" % "commons-lang3" % "3.10" ) diff --git a/project/build.properties b/project/build.properties index 35b49ae..d7ec7ac 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1 +1 @@ -sbt.version=1.3.8 \ No newline at end of file +sbt.version=1.3.13 \ No newline at end of file diff --git a/project/plugins.sbt b/project/plugins.sbt index a5866e3..36404e7 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -5,13 +5,11 @@ resolvers += Resolver.bintrayRepo("hmrc", "releases") addSbtPlugin("com.github.gseitz" % "sbt-release" % "1.0.13") -addSbtPlugin("com.typesafe.play" % "sbt-plugin" % "2.7.4") +addSbtPlugin("com.typesafe.play" % "sbt-plugin" % "2.7.5") -addSbtPlugin("uk.gov.hmrc" % "sbt-auto-build" % "2.6.0") +addSbtPlugin("uk.gov.hmrc" % "sbt-auto-build" % "2.9.0") -addSbtPlugin("uk.gov.hmrc" % "sbt-settings" % "4.1.0") - -addSbtPlugin("uk.gov.hmrc" % "sbt-artifactory" % "1.2.0") +addSbtPlugin("uk.gov.hmrc" % "sbt-settings" % "4.5.0") addSbtPlugin("uk.gov.hmrc" % "sbt-git-versioning" % "2.1.0") From ee06e7067e87d652063ff05c36ba36ed8299b579 Mon Sep 17 00:00:00 2001 From: Nigel Perkins Date: Fri, 4 Sep 2020 16:00:12 +0100 Subject: [PATCH 58/96] BDOG-1006: further housekeeping --- conf/application.conf | 6 +++--- .../model/PrepareUploadRequestV2Spec.scala | 16 ++++++++++++++++ 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/conf/application.conf b/conf/application.conf index 5744f1c..c2efd96 100644 --- a/conf/application.conf +++ b/conf/application.conf @@ -35,15 +35,15 @@ play.modules.enabled += "uk.gov.hmrc.play.bootstrap.AuditModule" play.modules.enabled += "uk.gov.hmrc.play.bootstrap.graphite.GraphiteMetricsModule" # Provides an implementation and configures all filters required by a Platform frontend microservice. -play.modules.enabled += "uk.gov.hmrc.play.bootstrap.MicroserviceModule" +play.modules.enabled += "uk.gov.hmrc.play.bootstrap.backend.BackendModule" play.modules.enabled += "UpscanInitiateModule" play.modules.enabled += "connectors.s3.S3Module" -play.http.filters = "uk.gov.hmrc.play.bootstrap.filters.MicroserviceFilters" +play.http.filters = "uk.gov.hmrc.play.bootstrap.backend.filters.BackendFilters" # Json error handler -play.http.errorHandler = "uk.gov.hmrc.play.bootstrap.http.JsonErrorHandler" +play.http.errorHandler = "uk.gov.hmrc.play.bootstrap.backend.http.JsonErrorHandler" # Secret key # ~~~~~ diff --git a/test/controllers/model/PrepareUploadRequestV2Spec.scala b/test/controllers/model/PrepareUploadRequestV2Spec.scala index edeb99a..b0aa577 100644 --- a/test/controllers/model/PrepareUploadRequestV2Spec.scala +++ b/test/controllers/model/PrepareUploadRequestV2Spec.scala @@ -88,6 +88,22 @@ class PrepareUploadRequestV2Spec extends UnitSpec with EitherValues { } } + "specifying a minimum file size" should { + "be rejected when negative" in { + val request = s"""{"callbackUrl":"$CallbackUrl", "minimumFileSize": -1}""" + val parseResult = Json.parse(request).validate(PrepareUploadRequestV2.reads(MaxFileSize)) + + parseResult.isError shouldBe true + } + + "be accepted when zero" in { + val request = s"""{"callbackUrl":"$CallbackUrl", "minimumFileSize": 0}""" + val parseResult = Json.parse(request).validate(PrepareUploadRequestV2.reads(MaxFileSize)) + + parseResult.asEither.map(_.minimumFileSize).right.value should contain (0) + } + } + "specifying a maximum file size" should { "be accepted when equal to the global maximum" in { val request = s"""{"callbackUrl":"$CallbackUrl", "maximumFileSize": $MaxFileSize}""" From 9564cdc9ae35f3744687cce0e3ca28253b313fa8 Mon Sep 17 00:00:00 2001 From: anshulbajpai <291429+anshulbajpai@users.noreply.github.com> Date: Fri, 11 Sep 2020 10:06:33 +0100 Subject: [PATCH 59/96] BDO-782: Changed file size type all over to accomodate higher values. --- app/config/ServiceConfiguration.scala | 20 +++++++++---------- app/connectors/model/ContentLengthRange.scala | 2 +- .../model/PrepareUploadRequestV1.scala | 12 +++++------ .../model/PrepareUploadRequestV2.scala | 12 +++++------ app/services/PrepareUploadService.scala | 4 ++-- app/services/model/UploadSettings.scala | 4 ++-- 6 files changed, 27 insertions(+), 27 deletions(-) diff --git a/app/config/ServiceConfiguration.scala b/app/config/ServiceConfiguration.scala index 74756b5..b1a7b16 100644 --- a/app/config/ServiceConfiguration.scala +++ b/app/config/ServiceConfiguration.scala @@ -32,32 +32,32 @@ trait ServiceConfiguration { def accessKeyId: String def secretAccessKey: String def fileExpirationPeriod: java.time.Duration - def globalFileSizeLimit: Int + def globalFileSizeLimit: Long def allowedCallbackProtocols: Seq[String] } class PlayBasedServiceConfiguration @Inject()(configuration: Configuration) extends ServiceConfiguration { - override def region: String = getRequired(configuration.getOptional[String](_), "aws.s3.region") + override val region: String = getRequired(configuration.getOptional[String](_), "aws.s3.region") - override def uploadProxyUrl: String = getRequired(configuration.getOptional[String](_), "uploadProxy.url") + override val uploadProxyUrl: String = getRequired(configuration.getOptional[String](_), "uploadProxy.url") - override def inboundBucketName: String = getRequired(configuration.getOptional[String](_), "aws.s3.bucket.inbound") + override val inboundBucketName: String = getRequired(configuration.getOptional[String](_), "aws.s3.bucket.inbound") - override def fileExpirationPeriod: Duration = { + override val fileExpirationPeriod: Duration = { val readAsMillis: String => Option[Long] = configuration.getOptional[scala.concurrent.duration.Duration](_).map(_.toMillis) Duration.ofMillis(getRequired(readAsMillis, "aws.s3.upload.link.validity.duration")) } - override def accessKeyId: String = getRequired(configuration.getOptional[String](_), "aws.accessKeyId") + override val accessKeyId: String = getRequired(configuration.getOptional[String](_), "aws.accessKeyId") - override def secretAccessKey: String = getRequired(configuration.getOptional[String](_), "aws.secretAccessKey") + override val secretAccessKey: String = getRequired(configuration.getOptional[String](_), "aws.secretAccessKey") - override def sessionToken: Option[String] = configuration.getOptional[String]("aws.sessionToken") + override val sessionToken: Option[String] = configuration.getOptional[String]("aws.sessionToken") - override def globalFileSizeLimit: Int = getRequired(configuration.getOptional[Int](_), "global.file.size.limit") + override val globalFileSizeLimit: Long = getRequired(configuration.getOptional[Long](_), "global.file.size.limit") - override def allowedCallbackProtocols: Seq[String] = + override val allowedCallbackProtocols: Seq[String] = commaSeparatedList(configuration.getOptional[String]("callbackValidation.allowedProtocols")) private def getRequired[T](read: String => Option[T], path: String): T = diff --git a/app/connectors/model/ContentLengthRange.scala b/app/connectors/model/ContentLengthRange.scala index 44c7f4a..b7031d8 100644 --- a/app/connectors/model/ContentLengthRange.scala +++ b/app/connectors/model/ContentLengthRange.scala @@ -16,4 +16,4 @@ package connectors.model -case class ContentLengthRange(min: Int, max: Int) +case class ContentLengthRange(min: Long, max: Long) diff --git a/app/controllers/model/PrepareUploadRequestV1.scala b/app/controllers/model/PrepareUploadRequestV1.scala index 7abb64a..5cb5c5a 100644 --- a/app/controllers/model/PrepareUploadRequestV1.scala +++ b/app/controllers/model/PrepareUploadRequestV1.scala @@ -23,8 +23,8 @@ import services.model.UploadSettings case class PrepareUploadRequestV1( callbackUrl: String, - minimumFileSize: Option[Int], - maximumFileSize: Option[Int], + minimumFileSize: Option[Long], + maximumFileSize: Option[Long], expectedContentType: Option[String], successRedirect: Option[String]) extends PrepareUpload { @@ -43,13 +43,13 @@ case class PrepareUploadRequestV1( object PrepareUploadRequestV1 { - def reads(maxFileSize: Int): Reads[PrepareUploadRequestV1] = + def reads(maxFileSize: Long): Reads[PrepareUploadRequestV1] = ((JsPath \ "callbackUrl").read[String] and - (JsPath \ "minimumFileSize").readNullable[Int](min(0)) and - (JsPath \ "maximumFileSize").readNullable[Int](min(0) keepAnd max(maxFileSize)) and + (JsPath \ "minimumFileSize").readNullable[Long](min(0)) and + (JsPath \ "maximumFileSize").readNullable[Long](min(0L) keepAnd max(maxFileSize)) and (JsPath \ "expectedContentType").readNullable[String] and (JsPath \ "successRedirect").readNullable[String])(PrepareUploadRequestV1.apply _) .filter(JsonValidationError("Maximum file size must be equal or greater than minimum file size"))(request => - request.minimumFileSize.getOrElse(0) <= request.maximumFileSize.getOrElse(maxFileSize)) + request.minimumFileSize.getOrElse(0L) <= request.maximumFileSize.getOrElse(maxFileSize)) } diff --git a/app/controllers/model/PrepareUploadRequestV2.scala b/app/controllers/model/PrepareUploadRequestV2.scala index 9622758..8775d6d 100644 --- a/app/controllers/model/PrepareUploadRequestV2.scala +++ b/app/controllers/model/PrepareUploadRequestV2.scala @@ -25,8 +25,8 @@ case class PrepareUploadRequestV2( callbackUrl: String, successRedirect: Option[String], errorRedirect: Option[String], - minimumFileSize: Option[Int], - maximumFileSize: Option[Int], + minimumFileSize: Option[Long], + maximumFileSize: Option[Long], expectedContentType: Option[String]) extends PrepareUpload { @@ -43,13 +43,13 @@ case class PrepareUploadRequestV2( object PrepareUploadRequestV2 { - def reads(maxFileSize: Int): Reads[PrepareUploadRequestV2] = + def reads(maxFileSize: Long): Reads[PrepareUploadRequestV2] = ((JsPath \ "callbackUrl").read[String] and (JsPath \ "successRedirect").readNullable[String] and (JsPath \ "errorRedirect").readNullable[String] and - (JsPath \ "minimumFileSize").readNullable[Int](min(0)) and - (JsPath \ "maximumFileSize").readNullable[Int](min(0) keepAnd max(maxFileSize)) and + (JsPath \ "minimumFileSize").readNullable[Long](min(0)) and + (JsPath \ "maximumFileSize").readNullable[Long](min(0L) keepAnd max(maxFileSize)) and (JsPath \ "expectedContentType").readNullable[String])(PrepareUploadRequestV2.apply _) .filter(JsonValidationError("Maximum file size must be equal or greater than minimum file size"))(request => - request.minimumFileSize.getOrElse(0) <= request.maximumFileSize.getOrElse(maxFileSize)) + request.minimumFileSize.getOrElse(0L) <= request.maximumFileSize.getOrElse(maxFileSize)) } diff --git a/app/services/PrepareUploadService.scala b/app/services/PrepareUploadService.scala index 18a7b79..2573504 100644 --- a/app/services/PrepareUploadService.scala +++ b/app/services/PrepareUploadService.scala @@ -77,7 +77,7 @@ class PrepareUploadService @Inject()( sessionId: String, receivedAt: Instant): UploadFormTemplate = { - val minFileSize = settings.minimumFileSize.getOrElse(0) + val minFileSize = settings.minimumFileSize.getOrElse(0L) val maxFileSize = settings.maximumFileSize.getOrElse(globalFileSizeLimit) require(minFileSize >= 0, "Minimum file size is less than 0") @@ -108,5 +108,5 @@ class PrepareUploadService @Inject()( UploadFormTemplate(endpoint, form) } - def globalFileSizeLimit: Int = configuration.globalFileSizeLimit + def globalFileSizeLimit: Long = configuration.globalFileSizeLimit } diff --git a/app/services/model/UploadSettings.scala b/app/services/model/UploadSettings.scala index db6b97f..63123ca 100644 --- a/app/services/model/UploadSettings.scala +++ b/app/services/model/UploadSettings.scala @@ -19,8 +19,8 @@ package services.model case class UploadSettings( uploadUrl: String, callbackUrl: String, - minimumFileSize: Option[Int], - maximumFileSize: Option[Int], + minimumFileSize: Option[Long], + maximumFileSize: Option[Long], expectedContentType: Option[String], successRedirect: Option[String], errorRedirect: Option[String]) From aa2ca4c4cffe8c65a762bc3cc3e150f425103e77 Mon Sep 17 00:00:00 2001 From: Nigel Perkins Date: Thu, 3 Dec 2020 10:32:02 +0000 Subject: [PATCH 60/96] BDOG-1169: remove Travis --- .travis.yml | 14 -------------- README.md | 2 +- 2 files changed, 1 insertion(+), 15 deletions(-) delete mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 3a0a52c..0000000 --- a/.travis.yml +++ /dev/null @@ -1,14 +0,0 @@ - -sudo: false -language: scala -scala: -- 2.11.12 -jdk: -- oraclejdk8 -cache: - directories: - - '$HOME/.ivy2/cache' -branches: - except: - - master - \ No newline at end of file diff --git a/README.md b/README.md index 1ef0284..ebe579f 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ Microservice for initiating the upload of files created externally to HMRC estate. These could be from members of the public or third-party services. This service is not for transfer of files from one HMRC service to another. See the Transmission Service as documented in Confluence for this use-case. -[![Build Status](https://travis-ci.org/hmrc/upscan-initiate.svg)](https://travis-ci.org/hmrc/upscan-initiate) [ ![Download](https://api.bintray.com/packages/hmrc/releases/upscan-initiate/images/download.svg) ](https://bintray.com/hmrc/releases/upscan-initiate/_latestVersion) +[ ![Download](https://api.bintray.com/packages/hmrc/releases/upscan-initiate/images/download.svg) ](https://bintray.com/hmrc/releases/upscan-initiate/_latestVersion) # TLDR From 41c65a5cb61e74920a08a7501ad359a9eabfc110 Mon Sep 17 00:00:00 2001 From: Nigel Perkins Date: Mon, 14 Dec 2020 16:42:13 +0000 Subject: [PATCH 61/96] BDOG-632: improve logging of upload key / reference --- README.md | 19 +++++++++++-------- app/controllers/PrepareUploadController.scala | 3 +-- app/services/PrepareUploadService.scala | 5 ++--- conf/application-json-logger.xml | 2 +- conf/application.conf | 2 +- conf/logback.xml | 4 +++- 6 files changed, 19 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index ebe579f..4006885 100644 --- a/README.md +++ b/README.md @@ -78,14 +78,15 @@ The service must also provide a callbackUrl for asynchronous notification of the (Although this rule is relaxed when testing locally with [upscan-stub](https://github.com/hmrc/upscan-stub) rather than [upscan-initiate](https://github.com/hmrc/upscan-initiate). In this stubbed scenario a `callbackUrl` referring to localhost may still specify `http` as the protocol.) -Session-ID / Request-ID headers will be used to link the file with the user's journey. +Session-ID / Request-ID headers are used to link the file with the user's journey. *Note:* If you are using `[http-verbs](https://github.com/hmrc/http-verbs)` to call Upscan, all the headers will be set automatically (See: [HttpVerb.scala](https://github.com/hmrc/http-verbs/blob/2807dc65f64009bd7ce1f14b38b356e06dd23512/src/main/scala/uk/gov/hmrc/http/HttpVerb.scala#L53)) The service replies with a pre-filled template for the upload of the file. -The JSON response also contains a globally unique file reference for the upload. This reference can be used by the Upscan service team to view the progress and result of the journey through the different Upscan components. The consuming service can use this reference to correlate the subsequent upload result with this upscan initiation. - +The JSON response contains a globally unique identifier for the upload (known as both _reference_ and _key_). +This identifier can be used by the Upscan service team to view the progress and result of the journey through the different Upscan components. +The consuming service can use this identifier to correlate the subsequent file processing outcome with this upload initiation (see [File processing outcome](#service__poutcome)). ### POST upscan/v2/initiate @@ -133,7 +134,7 @@ Example response "fields": { "Content-Type": "application/xml", "acl": "private", - "key": "xxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx", + "key": "11370e18-6e24-453e-b45a-76d3e32ea33d", "policy": "xxxxxxxx==", "x-amz-algorithm": "AWS4-HMAC-SHA256", "x-amz-credential": "ASIAxxxxxxxxx/20180202/eu-west-2/s3/aws4_request", @@ -168,8 +169,8 @@ For v2, how such an error is returned depends upon whether the `error_action_red If set, the [upscan-upload-proxy](https://github.com/hmrc/upscan-upload-proxy) will redirect to the specified URL. Details of the error will be supplied to this URL as query parameters, with the names `errorCode`, `errorMessage`, `errorResource` and `errorRequestId`. -The query parameter named `key` contains the globally unique file reference that was allocated by the initiate request -to identify the upload. +The query parameter named `key` contains the globally unique identifier that was allocated by the initiate request +to identify the upload (and was returned as `reference` in the initiate response). ``` HTTP Response Code: 303 @@ -231,7 +232,6 @@ Example request: |expectedContentType|MIME type describing the upload contents.|no| - Example Response: ```json @@ -242,7 +242,7 @@ Example Response: "fields": { "Content-Type": "application/xml", "acl": "private", - "key": "xxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx", + "key": "11370e18-6e24-453e-b45a-76d3e32ea33d", "policy": "xxxxxxxx==", "x-amz-algorithm": "AWS4-HMAC-SHA256", "x-amz-credential": "ASIAxxxxxxxxx/20180202/eu-west-2/s3/aws4_request", @@ -285,6 +285,9 @@ If the POST is successful, the service returns a HTTP 204 response with an empty ### File processing outcome +A callback notification provides details of the file processing outcome. +This contains a _reference_ that correlates with that contained in the upload initiation request (see [Requesting a URL to upload to](#service__request)). + #### Success When a file is successfully uploaded it is processed by [upscan-verify](https://github.com/hmrc/upscan-verify) to check for viruses & that it is of an allowed file type. diff --git a/app/controllers/PrepareUploadController.scala b/app/controllers/PrepareUploadController.scala index 1d6ec30..a8e1e27 100644 --- a/app/controllers/PrepareUploadController.scala +++ b/app/controllers/PrepareUploadController.scala @@ -63,10 +63,9 @@ class PrepareUploadController @Inject()( requireUserAgent[JsValue] { (_, consumingService) => withJsonBody[T] { prepareUploadRequest: T => withAllowedCallbackProtocol(prepareUploadRequest.callbackUrl) { - logger.debug(s"Processing request: [$prepareUploadRequest].") - val sessionId = hc(request).sessionId.map(_.value).getOrElse("n/a") val requestId = hc(request).requestId.map(_.value).getOrElse("n/a") + logger.debug(s"Processing request: [$prepareUploadRequest] with requestId: [$requestId] sessionId: [$sessionId] from: [$consumingService].") val result: PreparedUploadResponse = prepareUploadService .prepareUpload( diff --git a/app/services/PrepareUploadService.scala b/app/services/PrepareUploadService.scala index 2573504..23bb2aa 100644 --- a/app/services/PrepareUploadService.scala +++ b/app/services/PrepareUploadService.scala @@ -57,9 +57,8 @@ class PrepareUploadService @Inject()( try { MDC.put("file-reference", reference.toString) - logger.info( - s"Generated file-reference: [$reference], for settings: [$settings], with expiration at: [$expiration].") - + logger.info(s"Allocated key=[${reference.value}] to uploadRequest with requestId=[$requestId] sessionId=[$sessionId] from [$consumingService].") + logger.debug(s"Prepared upload response [$result].") metrics.defaultRegistry.counter("uploadInitiated").inc() result diff --git a/conf/application-json-logger.xml b/conf/application-json-logger.xml index 5d7f9dc..332df82 100644 --- a/conf/application-json-logger.xml +++ b/conf/application-json-logger.xml @@ -9,7 +9,7 @@ - + diff --git a/conf/application.conf b/conf/application.conf index c2efd96..0954f40 100644 --- a/conf/application.conf +++ b/conf/application.conf @@ -96,7 +96,7 @@ callbackValidation.allowedProtocols = "https" global.file.size.limit = 104857600 auditing { - enabled = true + enabled = false traceRequests = true consumer { baseUri { diff --git a/conf/logback.xml b/conf/logback.xml index dac3fe2..5c4ccee 100644 --- a/conf/logback.xml +++ b/conf/logback.xml @@ -44,9 +44,11 @@ - + + + From 6a45add0c912ce2ef0b8bb43f78744453981b3df Mon Sep 17 00:00:00 2001 From: Nigel Perkins Date: Fri, 18 Dec 2020 08:55:32 +0000 Subject: [PATCH 62/96] BDOG-632: dependency updates --- project/AppDependencies.scala | 10 +++++----- project/plugins.sbt | 8 ++++---- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/project/AppDependencies.scala b/project/AppDependencies.scala index fa25431..0c7914b 100644 --- a/project/AppDependencies.scala +++ b/project/AppDependencies.scala @@ -4,18 +4,18 @@ object AppDependencies { import play.core.PlayVersion private val compile = Seq( - "uk.gov.hmrc" %% "bootstrap-backend-play-27" % "2.24.0", + "uk.gov.hmrc" %% "bootstrap-backend-play-27" % "3.2.0", "com.typesafe.play" %% "play-json" % "2.7.4", - "com.amazonaws" % "aws-java-sdk-s3" % "1.11.769", - "org.apache.commons" % "commons-lang3" % "3.10" + "com.amazonaws" % "aws-java-sdk-s3" % "1.11.921", + "org.apache.commons" % "commons-lang3" % "3.11" ) private val test = Seq( "com.typesafe.play" %% "play-test" % PlayVersion.current % s"$Test,$IntegrationTest", - "org.scalatest" %% "scalatest" % "3.1.1" % s"$Test,$IntegrationTest", + "org.scalatest" %% "scalatest" % "3.1.4" % s"$Test,$IntegrationTest", "org.scalatestplus.play" %% "scalatestplus-play" % "4.0.3" % s"$Test,$IntegrationTest", "com.vladsch.flexmark" % "flexmark-all" % "0.35.10" % s"$Test,$IntegrationTest", - "org.mockito" %% "mockito-scala-scalatest" % "1.13.10" % Test + "org.mockito" %% "mockito-scala-scalatest" % "1.15.1" % Test ) def apply(): Seq[ModuleID] = compile ++ test diff --git a/project/plugins.sbt b/project/plugins.sbt index 36404e7..ad7abdf 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -7,13 +7,13 @@ addSbtPlugin("com.github.gseitz" % "sbt-release" % "1.0.13") addSbtPlugin("com.typesafe.play" % "sbt-plugin" % "2.7.5") -addSbtPlugin("uk.gov.hmrc" % "sbt-auto-build" % "2.9.0") +addSbtPlugin("uk.gov.hmrc" % "sbt-auto-build" % "2.12.0") -addSbtPlugin("uk.gov.hmrc" % "sbt-settings" % "4.5.0") +addSbtPlugin("uk.gov.hmrc" % "sbt-settings" % "4.6.0") -addSbtPlugin("uk.gov.hmrc" % "sbt-git-versioning" % "2.1.0") +addSbtPlugin("uk.gov.hmrc" % "sbt-git-versioning" % "2.2.0") -addSbtPlugin("uk.gov.hmrc" % "sbt-distributables" % "2.0.0") +addSbtPlugin("uk.gov.hmrc" % "sbt-distributables" % "2.1.0") addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.6.1") From eca42f8fb3a44195e2df43accccff6d7a70af267 Mon Sep 17 00:00:00 2001 From: Nigel Perkins Date: Tue, 22 Dec 2020 15:38:11 +0000 Subject: [PATCH 63/96] BDOG-632: changes in response to review comments --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 4006885..a2942c5 100644 --- a/README.md +++ b/README.md @@ -85,8 +85,9 @@ Session-ID / Request-ID headers are used to link the file with the user's journe The service replies with a pre-filled template for the upload of the file. The JSON response contains a globally unique identifier for the upload (known as both _reference_ and _key_). -This identifier can be used by the Upscan service team to view the progress and result of the journey through the different Upscan components. The consuming service can use this identifier to correlate the subsequent file processing outcome with this upload initiation (see [File processing outcome](#service__poutcome)). +This identifier can also be used to track the journey of an upload through the various Upscan services. If you require support from the Upscan team in relation to a particular +upload, please provide this reference. ### POST upscan/v2/initiate From 5b831a41043557ee168cb64c1a4b35e2dad84bcf Mon Sep 17 00:00:00 2001 From: Anshul Bajpai <291429+anshulbajpai@users.noreply.github.com> Date: Thu, 24 Dec 2020 11:50:58 +0000 Subject: [PATCH 64/96] BDOG-1154: Remove transmission service reference --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ebe579f..c8271c7 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # upscan-initiate Microservice for initiating the upload of files created externally to HMRC estate. These could be from members of the public or third-party services. -This service is not for transfer of files from one HMRC service to another. See the Transmission Service as documented in Confluence for this use-case. +This service is not for transfer of files from one HMRC service to another. Please speak to us for advice on file transfer integration. [ ![Download](https://api.bintray.com/packages/hmrc/releases/upscan-initiate/images/download.svg) ](https://bintray.com/hmrc/releases/upscan-initiate/_latestVersion) From f435ba0a9910dea979d4f759c13c7698a38bf07d Mon Sep 17 00:00:00 2001 From: Nigel Perkins Date: Tue, 9 Feb 2021 16:38:33 +0000 Subject: [PATCH 65/96] BDOG-1230: update example file OK notification with file size --- README.md | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index fa7e84a..d44cc89 100644 --- a/README.md +++ b/README.md @@ -297,25 +297,27 @@ If these checks pass, the file is made available for retrieval & the Upscan serv ```json { - "reference" : "11370e18-6e24-453e-b45a-76d3e32ea33d", - "fileStatus" : "READY", - "downloadUrl" : "https://bucketName.s3.eu-west-2.amazonaws.com?1235676", + "reference": "11370e18-6e24-453e-b45a-76d3e32ea33d", + "downloadUrl": "https://bucketName.s3.eu-west-2.amazonaws.com?1235676", + "fileStatus": "READY", "uploadDetails": { + "fileName": "test.pdf", + "fileMimeType": "application/pdf", "uploadTimestamp": "2018-04-24T09:30:00Z", "checksum": "396f101dd52e8b2ace0dcf5ed09b1d1f030e608938510ce46e7a5c7a4e775100", - "fileName": "test.pdf", - "fileMimeType": "application/pdf" + "size": 987 } } ``` Note the block entitled 'uploadDetails', see the Confluence page 'Upscan & Non-Repudiation Service' in the Platform Services space usage of this information: -- `uploadTimestamp` - The timestamp of the file upload -- `checksum` - The SHA256 hash of the uploaded file - `fileName` - File name as it was provided by the user - `fileMimeType` - Detected MIME type of the file. Please note that this refers to actual contents -of the file, not to the name (if user uploads PDF document named `data.png`, it will be detected as a `application/pdf`) +of the file, not to the name (if user uploads PDF document named `data.png`, it will be detected as a `application/pdf`) +- `uploadTimestamp` - The timestamp of the file upload +- `checksum` - The SHA256 hash of the uploaded file +- `size` - The size of the uploaded file (in bytes) The downloadUrl will expire after 1 day by default. This can be configured on a per-consuming service basis. For example, to limit to 1 hour add the following configuration (substituting the appropriate `User-Agent` service identifier) to [upscan-notify.conf](https://github.com/hmrc/app-config-base/blob/master/upscan-notify.conf): From 15f3baf8b5f6a8a674c71c6502b9cad5dbdfeb07 Mon Sep 17 00:00:00 2001 From: anshulbajpai <291429+anshulbajpai@users.noreply.github.com> Date: Mon, 28 Jun 2021 14:38:30 +0100 Subject: [PATCH 66/96] NIJ-Update documentation to list all causes for REJECTED failure type. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d44cc89..db99c62 100644 --- a/README.md +++ b/README.md @@ -341,7 +341,7 @@ consuming-services { The list of failure reasons is as follows: - `QUARANTINE` - the file has failed virus scanning -- `REJECTED` - the file is not of an allowed file type +- `REJECTED` - the [file's detected mime-type is not allowed for the service](https://github.com/hmrc/upscan-app-config/blob/master/production/verify.yaml) or the [file's extension is not allowed for the detected mime-type](https://github.com/hmrc/upscan-verify/blob/master/conf/extensionsAllowList.conf) - `UNKNOWN` - there is some other problem with the file These reasons form one of the following JSON responses sent to the callback URL: From b204cc3a1e378825cf22d3770e03a7c1d9b3b1fb Mon Sep 17 00:00:00 2001 From: chotai Date: Tue, 24 Aug 2021 12:28:54 +0100 Subject: [PATCH 67/96] BDOG-1471 Update bootstrap-play version to 28 --- app/UpscanInitiateModule.scala | 2 +- app/config/ServiceConfiguration.scala | 2 +- app/connectors/model/AwsCredentials.scala | 2 +- app/connectors/model/ContentLengthRange.scala | 2 +- app/connectors/model/UploadFormGenerator.scala | 2 +- app/connectors/model/UploadParameters.scala | 2 +- app/connectors/s3/PolicySigner.scala | 2 +- app/connectors/s3/S3Module.scala | 2 +- app/connectors/s3/S3UploadFormGenerator.scala | 2 +- .../s3/S3UploadFormGeneratorProvider.scala | 2 +- app/controllers/PrepareUploadController.scala | 2 +- app/controllers/model/PrepareUpload.scala | 2 +- .../model/PrepareUploadRequestV1.scala | 2 +- .../model/PrepareUploadRequestV2.scala | 2 +- .../model/PreparedUploadResponse.scala | 2 +- app/controllers/model/Reference.scala | 2 +- app/controllers/model/UploadFormTemplate.scala | 2 +- app/services/PrepareUploadService.scala | 2 +- app/services/model/UploadSettings.scala | 2 +- app/utils/UserAgentFilter.scala | 2 +- build.sbt | 2 +- conf/application.conf | 2 +- .../PrepareUploadControllerISpec.scala | 16 ++++++++++++++++ project/AppDependencies.scala | 15 ++++++--------- project/build.properties | 2 +- project/plugins.sbt | 15 +++++---------- test/connectors/s3/PolicySignerSpec.scala | 2 +- .../s3/S3UploadFormGeneratorSpec.scala | 2 +- .../PrepareUploadControllerSpec.scala | 2 +- .../model/PrepareUploadRequestV2Spec.scala | 2 +- test/model/PrepareUploadServiceSpec.scala | 2 +- test/test/UnitSpec.scala | 2 +- test/utils/UserAgentFilterSpec.scala | 2 +- 33 files changed, 57 insertions(+), 49 deletions(-) diff --git a/app/UpscanInitiateModule.scala b/app/UpscanInitiateModule.scala index 5bafce1..ec68e34 100644 --- a/app/UpscanInitiateModule.scala +++ b/app/UpscanInitiateModule.scala @@ -1,5 +1,5 @@ /* - * Copyright 2020 HM Revenue & Customs + * Copyright 2021 HM Revenue & Customs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/app/config/ServiceConfiguration.scala b/app/config/ServiceConfiguration.scala index b1a7b16..0591852 100644 --- a/app/config/ServiceConfiguration.scala +++ b/app/config/ServiceConfiguration.scala @@ -1,5 +1,5 @@ /* - * Copyright 2020 HM Revenue & Customs + * Copyright 2021 HM Revenue & Customs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/app/connectors/model/AwsCredentials.scala b/app/connectors/model/AwsCredentials.scala index e55897a..e794034 100644 --- a/app/connectors/model/AwsCredentials.scala +++ b/app/connectors/model/AwsCredentials.scala @@ -1,5 +1,5 @@ /* - * Copyright 2020 HM Revenue & Customs + * Copyright 2021 HM Revenue & Customs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/app/connectors/model/ContentLengthRange.scala b/app/connectors/model/ContentLengthRange.scala index b7031d8..360ff9b 100644 --- a/app/connectors/model/ContentLengthRange.scala +++ b/app/connectors/model/ContentLengthRange.scala @@ -1,5 +1,5 @@ /* - * Copyright 2020 HM Revenue & Customs + * Copyright 2021 HM Revenue & Customs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/app/connectors/model/UploadFormGenerator.scala b/app/connectors/model/UploadFormGenerator.scala index 42a5165..052499d 100644 --- a/app/connectors/model/UploadFormGenerator.scala +++ b/app/connectors/model/UploadFormGenerator.scala @@ -1,5 +1,5 @@ /* - * Copyright 2020 HM Revenue & Customs + * Copyright 2021 HM Revenue & Customs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/app/connectors/model/UploadParameters.scala b/app/connectors/model/UploadParameters.scala index 2e5f400..c721ad3 100644 --- a/app/connectors/model/UploadParameters.scala +++ b/app/connectors/model/UploadParameters.scala @@ -1,5 +1,5 @@ /* - * Copyright 2020 HM Revenue & Customs + * Copyright 2021 HM Revenue & Customs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/app/connectors/s3/PolicySigner.scala b/app/connectors/s3/PolicySigner.scala index f9073cc..10d7378 100644 --- a/app/connectors/s3/PolicySigner.scala +++ b/app/connectors/s3/PolicySigner.scala @@ -1,5 +1,5 @@ /* - * Copyright 2020 HM Revenue & Customs + * Copyright 2021 HM Revenue & Customs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/app/connectors/s3/S3Module.scala b/app/connectors/s3/S3Module.scala index 391c8c2..2e3c290 100644 --- a/app/connectors/s3/S3Module.scala +++ b/app/connectors/s3/S3Module.scala @@ -1,5 +1,5 @@ /* - * Copyright 2020 HM Revenue & Customs + * Copyright 2021 HM Revenue & Customs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/app/connectors/s3/S3UploadFormGenerator.scala b/app/connectors/s3/S3UploadFormGenerator.scala index 9055df5..0767c91 100644 --- a/app/connectors/s3/S3UploadFormGenerator.scala +++ b/app/connectors/s3/S3UploadFormGenerator.scala @@ -1,5 +1,5 @@ /* - * Copyright 2020 HM Revenue & Customs + * Copyright 2021 HM Revenue & Customs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/app/connectors/s3/S3UploadFormGeneratorProvider.scala b/app/connectors/s3/S3UploadFormGeneratorProvider.scala index 40bddd4..9ecf5ea 100644 --- a/app/connectors/s3/S3UploadFormGeneratorProvider.scala +++ b/app/connectors/s3/S3UploadFormGeneratorProvider.scala @@ -1,5 +1,5 @@ /* - * Copyright 2020 HM Revenue & Customs + * Copyright 2021 HM Revenue & Customs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/app/controllers/PrepareUploadController.scala b/app/controllers/PrepareUploadController.scala index a8e1e27..265fd8a 100644 --- a/app/controllers/PrepareUploadController.scala +++ b/app/controllers/PrepareUploadController.scala @@ -1,5 +1,5 @@ /* - * Copyright 2020 HM Revenue & Customs + * Copyright 2021 HM Revenue & Customs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/app/controllers/model/PrepareUpload.scala b/app/controllers/model/PrepareUpload.scala index 4cea64c..9547cf2 100644 --- a/app/controllers/model/PrepareUpload.scala +++ b/app/controllers/model/PrepareUpload.scala @@ -1,5 +1,5 @@ /* - * Copyright 2020 HM Revenue & Customs + * Copyright 2021 HM Revenue & Customs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/app/controllers/model/PrepareUploadRequestV1.scala b/app/controllers/model/PrepareUploadRequestV1.scala index 5cb5c5a..e0ef54e 100644 --- a/app/controllers/model/PrepareUploadRequestV1.scala +++ b/app/controllers/model/PrepareUploadRequestV1.scala @@ -1,5 +1,5 @@ /* - * Copyright 2020 HM Revenue & Customs + * Copyright 2021 HM Revenue & Customs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/app/controllers/model/PrepareUploadRequestV2.scala b/app/controllers/model/PrepareUploadRequestV2.scala index 8775d6d..53ce8de 100644 --- a/app/controllers/model/PrepareUploadRequestV2.scala +++ b/app/controllers/model/PrepareUploadRequestV2.scala @@ -1,5 +1,5 @@ /* - * Copyright 2020 HM Revenue & Customs + * Copyright 2021 HM Revenue & Customs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/app/controllers/model/PreparedUploadResponse.scala b/app/controllers/model/PreparedUploadResponse.scala index bde5e80..c28d3f2 100644 --- a/app/controllers/model/PreparedUploadResponse.scala +++ b/app/controllers/model/PreparedUploadResponse.scala @@ -1,5 +1,5 @@ /* - * Copyright 2020 HM Revenue & Customs + * Copyright 2021 HM Revenue & Customs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/app/controllers/model/Reference.scala b/app/controllers/model/Reference.scala index ed14653..c517fdf 100644 --- a/app/controllers/model/Reference.scala +++ b/app/controllers/model/Reference.scala @@ -1,5 +1,5 @@ /* - * Copyright 2020 HM Revenue & Customs + * Copyright 2021 HM Revenue & Customs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/app/controllers/model/UploadFormTemplate.scala b/app/controllers/model/UploadFormTemplate.scala index 8db2618..42cdb70 100644 --- a/app/controllers/model/UploadFormTemplate.scala +++ b/app/controllers/model/UploadFormTemplate.scala @@ -1,5 +1,5 @@ /* - * Copyright 2020 HM Revenue & Customs + * Copyright 2021 HM Revenue & Customs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/app/services/PrepareUploadService.scala b/app/services/PrepareUploadService.scala index 23bb2aa..fe0f66a 100644 --- a/app/services/PrepareUploadService.scala +++ b/app/services/PrepareUploadService.scala @@ -1,5 +1,5 @@ /* - * Copyright 2020 HM Revenue & Customs + * Copyright 2021 HM Revenue & Customs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/app/services/model/UploadSettings.scala b/app/services/model/UploadSettings.scala index 63123ca..40258fa 100644 --- a/app/services/model/UploadSettings.scala +++ b/app/services/model/UploadSettings.scala @@ -1,5 +1,5 @@ /* - * Copyright 2020 HM Revenue & Customs + * Copyright 2021 HM Revenue & Customs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/app/utils/UserAgentFilter.scala b/app/utils/UserAgentFilter.scala index 02d75f0..c49e654 100644 --- a/app/utils/UserAgentFilter.scala +++ b/app/utils/UserAgentFilter.scala @@ -1,5 +1,5 @@ /* - * Copyright 2020 HM Revenue & Customs + * Copyright 2021 HM Revenue & Customs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/build.sbt b/build.sbt index 0474c47..914cc16 100644 --- a/build.sbt +++ b/build.sbt @@ -20,7 +20,7 @@ lazy val scoverageSettings = Seq( ) lazy val microservice = Project(appName, file(".")) - .enablePlugins(Seq(play.sbt.PlayScala, SbtAutoBuildPlugin, SbtGitVersioning, SbtDistributablesPlugin): _*) + .enablePlugins(play.sbt.PlayScala, SbtAutoBuildPlugin, SbtDistributablesPlugin) .disablePlugins(JUnitXmlReportPlugin) //Required to prevent https://github.com/scalatest/scalatest/issues/1427 .settings(scoverageSettings: _*) .settings(majorVersion := 0) diff --git a/conf/application.conf b/conf/application.conf index 0954f40..5fbab6b 100644 --- a/conf/application.conf +++ b/conf/application.conf @@ -1,4 +1,4 @@ -# Copyright 2020 HM Revenue & Customs +# Copyright 2021 HM Revenue & Customs # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/it/controllers/PrepareUploadControllerISpec.scala b/it/controllers/PrepareUploadControllerISpec.scala index 0e63738..1c665f4 100644 --- a/it/controllers/PrepareUploadControllerISpec.scala +++ b/it/controllers/PrepareUploadControllerISpec.scala @@ -1,3 +1,19 @@ +/* + * Copyright 2021 HM Revenue & Customs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package controllers import org.scalatest.GivenWhenThen diff --git a/project/AppDependencies.scala b/project/AppDependencies.scala index 0c7914b..b0c29d9 100644 --- a/project/AppDependencies.scala +++ b/project/AppDependencies.scala @@ -1,22 +1,19 @@ import sbt._ object AppDependencies { - import play.core.PlayVersion + private val bootstrapVersion = "5.12.0" private val compile = Seq( - "uk.gov.hmrc" %% "bootstrap-backend-play-27" % "3.2.0", - "com.typesafe.play" %% "play-json" % "2.7.4", + "uk.gov.hmrc" %% "bootstrap-backend-play-28" % bootstrapVersion, "com.amazonaws" % "aws-java-sdk-s3" % "1.11.921", "org.apache.commons" % "commons-lang3" % "3.11" ) private val test = Seq( - "com.typesafe.play" %% "play-test" % PlayVersion.current % s"$Test,$IntegrationTest", - "org.scalatest" %% "scalatest" % "3.1.4" % s"$Test,$IntegrationTest", - "org.scalatestplus.play" %% "scalatestplus-play" % "4.0.3" % s"$Test,$IntegrationTest", - "com.vladsch.flexmark" % "flexmark-all" % "0.35.10" % s"$Test,$IntegrationTest", - "org.mockito" %% "mockito-scala-scalatest" % "1.15.1" % Test + "uk.gov.hmrc" %% "bootstrap-test-play-28" % bootstrapVersion % s"$Test,$IntegrationTest", + "com.vladsch.flexmark" % "flexmark-all" % "0.35.10" % s"$Test,$IntegrationTest", + "org.mockito" %% "mockito-scala-scalatest" % "1.16.25" % Test ) def apply(): Seq[ModuleID] = compile ++ test -} \ No newline at end of file +} diff --git a/project/build.properties b/project/build.properties index d7ec7ac..f17a59c 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1 +1 @@ -sbt.version=1.3.13 \ No newline at end of file +sbt.version=1.4.9 \ No newline at end of file diff --git a/project/plugins.sbt b/project/plugins.sbt index ad7abdf..1f054dd 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -1,17 +1,12 @@ import sbt._ -resolvers += Resolver.bintrayIvyRepo("hmrc", "sbt-plugin-releases") -resolvers += Resolver.bintrayRepo("hmrc", "releases") +resolvers += MavenRepository("HMRC-open-artefacts-maven2", "https://open.artefacts.tax.service.gov.uk/maven2") +resolvers += Resolver.url("HMRC-open-artefacts-ivy2", url("https://open.artefacts.tax.service.gov.uk/ivy2"))( + Resolver.ivyStylePatterns) -addSbtPlugin("com.github.gseitz" % "sbt-release" % "1.0.13") +addSbtPlugin("com.typesafe.play" % "sbt-plugin" % "2.8.7") -addSbtPlugin("com.typesafe.play" % "sbt-plugin" % "2.7.5") - -addSbtPlugin("uk.gov.hmrc" % "sbt-auto-build" % "2.12.0") - -addSbtPlugin("uk.gov.hmrc" % "sbt-settings" % "4.6.0") - -addSbtPlugin("uk.gov.hmrc" % "sbt-git-versioning" % "2.2.0") +addSbtPlugin("uk.gov.hmrc" % "sbt-auto-build" % "3.4.0") addSbtPlugin("uk.gov.hmrc" % "sbt-distributables" % "2.1.0") diff --git a/test/connectors/s3/PolicySignerSpec.scala b/test/connectors/s3/PolicySignerSpec.scala index 695a633..6114e86 100644 --- a/test/connectors/s3/PolicySignerSpec.scala +++ b/test/connectors/s3/PolicySignerSpec.scala @@ -1,5 +1,5 @@ /* - * Copyright 2020 HM Revenue & Customs + * Copyright 2021 HM Revenue & Customs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/test/connectors/s3/S3UploadFormGeneratorSpec.scala b/test/connectors/s3/S3UploadFormGeneratorSpec.scala index 3018890..486d98d 100644 --- a/test/connectors/s3/S3UploadFormGeneratorSpec.scala +++ b/test/connectors/s3/S3UploadFormGeneratorSpec.scala @@ -1,5 +1,5 @@ /* - * Copyright 2020 HM Revenue & Customs + * Copyright 2021 HM Revenue & Customs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/test/controllers/PrepareUploadControllerSpec.scala b/test/controllers/PrepareUploadControllerSpec.scala index 0780b8e..ab50d6d 100644 --- a/test/controllers/PrepareUploadControllerSpec.scala +++ b/test/controllers/PrepareUploadControllerSpec.scala @@ -1,5 +1,5 @@ /* - * Copyright 2020 HM Revenue & Customs + * Copyright 2021 HM Revenue & Customs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/test/controllers/model/PrepareUploadRequestV2Spec.scala b/test/controllers/model/PrepareUploadRequestV2Spec.scala index b0aa577..d0d3f37 100644 --- a/test/controllers/model/PrepareUploadRequestV2Spec.scala +++ b/test/controllers/model/PrepareUploadRequestV2Spec.scala @@ -1,5 +1,5 @@ /* - * Copyright 2020 HM Revenue & Customs + * Copyright 2021 HM Revenue & Customs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/test/model/PrepareUploadServiceSpec.scala b/test/model/PrepareUploadServiceSpec.scala index 50a35e4..7a40978 100644 --- a/test/model/PrepareUploadServiceSpec.scala +++ b/test/model/PrepareUploadServiceSpec.scala @@ -1,5 +1,5 @@ /* - * Copyright 2020 HM Revenue & Customs + * Copyright 2021 HM Revenue & Customs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/test/test/UnitSpec.scala b/test/test/UnitSpec.scala index 82d0e82..aee4825 100644 --- a/test/test/UnitSpec.scala +++ b/test/test/UnitSpec.scala @@ -1,5 +1,5 @@ /* - * Copyright 2020 HM Revenue & Customs + * Copyright 2021 HM Revenue & Customs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/test/utils/UserAgentFilterSpec.scala b/test/utils/UserAgentFilterSpec.scala index 7a29aff..af903bf 100644 --- a/test/utils/UserAgentFilterSpec.scala +++ b/test/utils/UserAgentFilterSpec.scala @@ -1,5 +1,5 @@ /* - * Copyright 2020 HM Revenue & Customs + * Copyright 2021 HM Revenue & Customs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. From cf572ea5e433a500d1c057679775cca711769440 Mon Sep 17 00:00:00 2001 From: Richard Sherman <23365258+RikSherman@users.noreply.github.com> Date: Mon, 10 Jan 2022 14:59:53 +0000 Subject: [PATCH 68/96] Copyright Update --- app/UpscanInitiateModule.scala | 2 +- app/config/ServiceConfiguration.scala | 2 +- app/connectors/model/AwsCredentials.scala | 2 +- app/connectors/model/ContentLengthRange.scala | 2 +- app/connectors/model/UploadFormGenerator.scala | 2 +- app/connectors/model/UploadParameters.scala | 2 +- app/connectors/s3/PolicySigner.scala | 2 +- app/connectors/s3/S3Module.scala | 2 +- app/connectors/s3/S3UploadFormGenerator.scala | 2 +- app/connectors/s3/S3UploadFormGeneratorProvider.scala | 2 +- app/controllers/PrepareUploadController.scala | 2 +- app/controllers/model/PrepareUpload.scala | 2 +- app/controllers/model/PrepareUploadRequestV1.scala | 2 +- app/controllers/model/PrepareUploadRequestV2.scala | 2 +- app/controllers/model/PreparedUploadResponse.scala | 2 +- app/controllers/model/Reference.scala | 2 +- app/controllers/model/UploadFormTemplate.scala | 2 +- app/services/PrepareUploadService.scala | 2 +- app/services/model/UploadSettings.scala | 2 +- app/utils/UserAgentFilter.scala | 2 +- conf/application.conf | 2 +- test/connectors/s3/PolicySignerSpec.scala | 2 +- test/connectors/s3/S3UploadFormGeneratorSpec.scala | 2 +- test/controllers/PrepareUploadControllerSpec.scala | 2 +- test/controllers/model/PrepareUploadRequestV2Spec.scala | 2 +- test/model/PrepareUploadServiceSpec.scala | 2 +- test/test/UnitSpec.scala | 2 +- test/utils/UserAgentFilterSpec.scala | 2 +- 28 files changed, 28 insertions(+), 28 deletions(-) diff --git a/app/UpscanInitiateModule.scala b/app/UpscanInitiateModule.scala index ec68e34..84bab1d 100644 --- a/app/UpscanInitiateModule.scala +++ b/app/UpscanInitiateModule.scala @@ -1,5 +1,5 @@ /* - * Copyright 2021 HM Revenue & Customs + * Copyright 2022 HM Revenue & Customs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/app/config/ServiceConfiguration.scala b/app/config/ServiceConfiguration.scala index 0591852..c7faba5 100644 --- a/app/config/ServiceConfiguration.scala +++ b/app/config/ServiceConfiguration.scala @@ -1,5 +1,5 @@ /* - * Copyright 2021 HM Revenue & Customs + * Copyright 2022 HM Revenue & Customs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/app/connectors/model/AwsCredentials.scala b/app/connectors/model/AwsCredentials.scala index e794034..4ea3807 100644 --- a/app/connectors/model/AwsCredentials.scala +++ b/app/connectors/model/AwsCredentials.scala @@ -1,5 +1,5 @@ /* - * Copyright 2021 HM Revenue & Customs + * Copyright 2022 HM Revenue & Customs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/app/connectors/model/ContentLengthRange.scala b/app/connectors/model/ContentLengthRange.scala index 360ff9b..571fb15 100644 --- a/app/connectors/model/ContentLengthRange.scala +++ b/app/connectors/model/ContentLengthRange.scala @@ -1,5 +1,5 @@ /* - * Copyright 2021 HM Revenue & Customs + * Copyright 2022 HM Revenue & Customs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/app/connectors/model/UploadFormGenerator.scala b/app/connectors/model/UploadFormGenerator.scala index 052499d..0104f17 100644 --- a/app/connectors/model/UploadFormGenerator.scala +++ b/app/connectors/model/UploadFormGenerator.scala @@ -1,5 +1,5 @@ /* - * Copyright 2021 HM Revenue & Customs + * Copyright 2022 HM Revenue & Customs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/app/connectors/model/UploadParameters.scala b/app/connectors/model/UploadParameters.scala index c721ad3..85c73cb 100644 --- a/app/connectors/model/UploadParameters.scala +++ b/app/connectors/model/UploadParameters.scala @@ -1,5 +1,5 @@ /* - * Copyright 2021 HM Revenue & Customs + * Copyright 2022 HM Revenue & Customs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/app/connectors/s3/PolicySigner.scala b/app/connectors/s3/PolicySigner.scala index 10d7378..c3d3cf9 100644 --- a/app/connectors/s3/PolicySigner.scala +++ b/app/connectors/s3/PolicySigner.scala @@ -1,5 +1,5 @@ /* - * Copyright 2021 HM Revenue & Customs + * Copyright 2022 HM Revenue & Customs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/app/connectors/s3/S3Module.scala b/app/connectors/s3/S3Module.scala index 2e3c290..4f37776 100644 --- a/app/connectors/s3/S3Module.scala +++ b/app/connectors/s3/S3Module.scala @@ -1,5 +1,5 @@ /* - * Copyright 2021 HM Revenue & Customs + * Copyright 2022 HM Revenue & Customs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/app/connectors/s3/S3UploadFormGenerator.scala b/app/connectors/s3/S3UploadFormGenerator.scala index 0767c91..1d440a8 100644 --- a/app/connectors/s3/S3UploadFormGenerator.scala +++ b/app/connectors/s3/S3UploadFormGenerator.scala @@ -1,5 +1,5 @@ /* - * Copyright 2021 HM Revenue & Customs + * Copyright 2022 HM Revenue & Customs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/app/connectors/s3/S3UploadFormGeneratorProvider.scala b/app/connectors/s3/S3UploadFormGeneratorProvider.scala index 9ecf5ea..17f4c86 100644 --- a/app/connectors/s3/S3UploadFormGeneratorProvider.scala +++ b/app/connectors/s3/S3UploadFormGeneratorProvider.scala @@ -1,5 +1,5 @@ /* - * Copyright 2021 HM Revenue & Customs + * Copyright 2022 HM Revenue & Customs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/app/controllers/PrepareUploadController.scala b/app/controllers/PrepareUploadController.scala index 265fd8a..a2d725c 100644 --- a/app/controllers/PrepareUploadController.scala +++ b/app/controllers/PrepareUploadController.scala @@ -1,5 +1,5 @@ /* - * Copyright 2021 HM Revenue & Customs + * Copyright 2022 HM Revenue & Customs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/app/controllers/model/PrepareUpload.scala b/app/controllers/model/PrepareUpload.scala index 9547cf2..bb5c079 100644 --- a/app/controllers/model/PrepareUpload.scala +++ b/app/controllers/model/PrepareUpload.scala @@ -1,5 +1,5 @@ /* - * Copyright 2021 HM Revenue & Customs + * Copyright 2022 HM Revenue & Customs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/app/controllers/model/PrepareUploadRequestV1.scala b/app/controllers/model/PrepareUploadRequestV1.scala index e0ef54e..1f59250 100644 --- a/app/controllers/model/PrepareUploadRequestV1.scala +++ b/app/controllers/model/PrepareUploadRequestV1.scala @@ -1,5 +1,5 @@ /* - * Copyright 2021 HM Revenue & Customs + * Copyright 2022 HM Revenue & Customs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/app/controllers/model/PrepareUploadRequestV2.scala b/app/controllers/model/PrepareUploadRequestV2.scala index 53ce8de..81a5f72 100644 --- a/app/controllers/model/PrepareUploadRequestV2.scala +++ b/app/controllers/model/PrepareUploadRequestV2.scala @@ -1,5 +1,5 @@ /* - * Copyright 2021 HM Revenue & Customs + * Copyright 2022 HM Revenue & Customs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/app/controllers/model/PreparedUploadResponse.scala b/app/controllers/model/PreparedUploadResponse.scala index c28d3f2..c576304 100644 --- a/app/controllers/model/PreparedUploadResponse.scala +++ b/app/controllers/model/PreparedUploadResponse.scala @@ -1,5 +1,5 @@ /* - * Copyright 2021 HM Revenue & Customs + * Copyright 2022 HM Revenue & Customs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/app/controllers/model/Reference.scala b/app/controllers/model/Reference.scala index c517fdf..f8476a3 100644 --- a/app/controllers/model/Reference.scala +++ b/app/controllers/model/Reference.scala @@ -1,5 +1,5 @@ /* - * Copyright 2021 HM Revenue & Customs + * Copyright 2022 HM Revenue & Customs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/app/controllers/model/UploadFormTemplate.scala b/app/controllers/model/UploadFormTemplate.scala index 42cdb70..00b9ee2 100644 --- a/app/controllers/model/UploadFormTemplate.scala +++ b/app/controllers/model/UploadFormTemplate.scala @@ -1,5 +1,5 @@ /* - * Copyright 2021 HM Revenue & Customs + * Copyright 2022 HM Revenue & Customs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/app/services/PrepareUploadService.scala b/app/services/PrepareUploadService.scala index fe0f66a..341d5f2 100644 --- a/app/services/PrepareUploadService.scala +++ b/app/services/PrepareUploadService.scala @@ -1,5 +1,5 @@ /* - * Copyright 2021 HM Revenue & Customs + * Copyright 2022 HM Revenue & Customs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/app/services/model/UploadSettings.scala b/app/services/model/UploadSettings.scala index 40258fa..1a40029 100644 --- a/app/services/model/UploadSettings.scala +++ b/app/services/model/UploadSettings.scala @@ -1,5 +1,5 @@ /* - * Copyright 2021 HM Revenue & Customs + * Copyright 2022 HM Revenue & Customs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/app/utils/UserAgentFilter.scala b/app/utils/UserAgentFilter.scala index c49e654..6ed2a4f 100644 --- a/app/utils/UserAgentFilter.scala +++ b/app/utils/UserAgentFilter.scala @@ -1,5 +1,5 @@ /* - * Copyright 2021 HM Revenue & Customs + * Copyright 2022 HM Revenue & Customs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/conf/application.conf b/conf/application.conf index 5fbab6b..254be8a 100644 --- a/conf/application.conf +++ b/conf/application.conf @@ -1,4 +1,4 @@ -# Copyright 2021 HM Revenue & Customs +# Copyright 2022 HM Revenue & Customs # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/test/connectors/s3/PolicySignerSpec.scala b/test/connectors/s3/PolicySignerSpec.scala index 6114e86..f69f994 100644 --- a/test/connectors/s3/PolicySignerSpec.scala +++ b/test/connectors/s3/PolicySignerSpec.scala @@ -1,5 +1,5 @@ /* - * Copyright 2021 HM Revenue & Customs + * Copyright 2022 HM Revenue & Customs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/test/connectors/s3/S3UploadFormGeneratorSpec.scala b/test/connectors/s3/S3UploadFormGeneratorSpec.scala index 486d98d..ac54ba8 100644 --- a/test/connectors/s3/S3UploadFormGeneratorSpec.scala +++ b/test/connectors/s3/S3UploadFormGeneratorSpec.scala @@ -1,5 +1,5 @@ /* - * Copyright 2021 HM Revenue & Customs + * Copyright 2022 HM Revenue & Customs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/test/controllers/PrepareUploadControllerSpec.scala b/test/controllers/PrepareUploadControllerSpec.scala index ab50d6d..0efb065 100644 --- a/test/controllers/PrepareUploadControllerSpec.scala +++ b/test/controllers/PrepareUploadControllerSpec.scala @@ -1,5 +1,5 @@ /* - * Copyright 2021 HM Revenue & Customs + * Copyright 2022 HM Revenue & Customs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/test/controllers/model/PrepareUploadRequestV2Spec.scala b/test/controllers/model/PrepareUploadRequestV2Spec.scala index d0d3f37..ade5844 100644 --- a/test/controllers/model/PrepareUploadRequestV2Spec.scala +++ b/test/controllers/model/PrepareUploadRequestV2Spec.scala @@ -1,5 +1,5 @@ /* - * Copyright 2021 HM Revenue & Customs + * Copyright 2022 HM Revenue & Customs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/test/model/PrepareUploadServiceSpec.scala b/test/model/PrepareUploadServiceSpec.scala index 7a40978..3ac0036 100644 --- a/test/model/PrepareUploadServiceSpec.scala +++ b/test/model/PrepareUploadServiceSpec.scala @@ -1,5 +1,5 @@ /* - * Copyright 2021 HM Revenue & Customs + * Copyright 2022 HM Revenue & Customs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/test/test/UnitSpec.scala b/test/test/UnitSpec.scala index aee4825..3fed5e2 100644 --- a/test/test/UnitSpec.scala +++ b/test/test/UnitSpec.scala @@ -1,5 +1,5 @@ /* - * Copyright 2021 HM Revenue & Customs + * Copyright 2022 HM Revenue & Customs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/test/utils/UserAgentFilterSpec.scala b/test/utils/UserAgentFilterSpec.scala index af903bf..644e3b5 100644 --- a/test/utils/UserAgentFilterSpec.scala +++ b/test/utils/UserAgentFilterSpec.scala @@ -1,5 +1,5 @@ /* - * Copyright 2021 HM Revenue & Customs + * Copyright 2022 HM Revenue & Customs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. From 88e13b1eb6608de98da178e4c7bcffb35ceaadef Mon Sep 17 00:00:00 2001 From: Richard Sherman <23365258+RikSherman@users.noreply.github.com> Date: Mon, 10 Jan 2022 15:47:22 +0000 Subject: [PATCH 69/96] BDOG-1394 remove unused expectedContentType field --- README.md | 4 -- app/connectors/model/UploadParameters.scala | 1 - app/connectors/s3/S3UploadFormGenerator.scala | 7 +- .../model/PrepareUploadRequestV1.scala | 3 - .../model/PrepareUploadRequestV2.scala | 7 +- app/services/PrepareUploadService.scala | 1 - app/services/model/UploadSettings.scala | 1 - .../PrepareUploadControllerISpec.scala | 64 ++++++++++++++++--- .../s3/S3UploadFormGeneratorSpec.scala | 4 -- .../PrepareUploadControllerSpec.scala | 6 -- .../model/PrepareUploadRequestV2Spec.scala | 3 +- test/model/PrepareUploadServiceSpec.scala | 13 ---- 12 files changed, 60 insertions(+), 54 deletions(-) diff --git a/README.md b/README.md index db99c62..c4f5fbd 100644 --- a/README.md +++ b/README.md @@ -123,7 +123,6 @@ Example `upscan/v2/initiate` request: |errorRedirect|Url to redirect to if error encountered during upload.|no| |minimumFileSize|Minimum file size (in Bytes). Default is 0.|no| |maximumFileSize|Maximum file size (in Bytes). Cannot be greater than 100MB. Default is 100MB.|no| -|expectedContentType|MIME type describing the upload contents.|no| Example response @@ -133,7 +132,6 @@ Example response "uploadRequest": { "href": "https://xxxx/upscan-upload-proxy/bucketName", "fields": { - "Content-Type": "application/xml", "acl": "private", "key": "11370e18-6e24-453e-b45a-76d3e32ea33d", "policy": "xxxxxxxx==", @@ -230,7 +228,6 @@ Example request: |minimumFileSize|Minimum file size (in Bytes). Default is 0.|no| |maximumFileSize|Maximum file size (in Bytes). Cannot be greater than 100MB. Default is 100MB.|no| |successRedirect|Url to redirect to after file has been successfully uploaded.|no| -|expectedContentType|MIME type describing the upload contents.|no| Example Response: @@ -241,7 +238,6 @@ Example Response: "uploadRequest": { "href": "https://bucketName.s3.eu-west-2.amazonaws.com", "fields": { - "Content-Type": "application/xml", "acl": "private", "key": "11370e18-6e24-453e-b45a-76d3e32ea33d", "policy": "xxxxxxxx==", diff --git a/app/connectors/model/UploadParameters.scala b/app/connectors/model/UploadParameters.scala index 85c73cb..d6ba352 100644 --- a/app/connectors/model/UploadParameters.scala +++ b/app/connectors/model/UploadParameters.scala @@ -25,7 +25,6 @@ case class UploadParameters( acl: String, additionalMetadata: Map[String, String], contentLengthRange: ContentLengthRange, - expectedContentType: Option[String], successRedirect: Option[String], errorRedirect: Option[String] ) diff --git a/app/connectors/s3/S3UploadFormGenerator.scala b/app/connectors/s3/S3UploadFormGenerator.scala index 1d440a8..7b6b4e5 100644 --- a/app/connectors/s3/S3UploadFormGenerator.scala +++ b/app/connectors/s3/S3UploadFormGenerator.scala @@ -86,14 +86,13 @@ class S3UploadFormGenerator( } val sessionCredentials = securityToken.map(v => Map("x-amz-security-token" -> v)).getOrElse(Map.empty) - val contentTypeField = uploadParameters.expectedContentType.map(v => Map("Content-Type" -> v)).getOrElse(Map.empty) val successRedirect = uploadParameters.successRedirect.map(v => Map("success_action_redirect" -> v)).getOrElse(Map.empty) val errorRedirect = uploadParameters.errorRedirect.map(v => Map("error_action_redirect" -> v)).getOrElse(Map.empty) - fields ++ metadataFields ++ sessionCredentials ++ contentTypeField ++ successRedirect ++ errorRedirect + fields ++ metadataFields ++ sessionCredentials ++ successRedirect ++ errorRedirect } private def buildPolicy( @@ -110,9 +109,6 @@ class S3UploadFormGenerator( Json.arr("starts-with", "$x-amz-meta-original-filename", "") :+ Json.arr("starts-with", "$x-amz-meta-upscan-initiate-response", "") - val contentTypeConstraintJson = - uploadParameters.expectedContentType.map(contentType => Json.obj("Content-Type" -> contentType)) - val successRedirectConstraint = uploadParameters.successRedirect.map(redirect => Json.obj("success_action_redirect" -> redirect)) @@ -135,7 +131,6 @@ class S3UploadFormGenerator( uploadParameters.contentLengthRange.max) ) ++ securityTokenJson ++ metadataJson - ++ contentTypeConstraintJson ++ successRedirectConstraint ++ errorRedirectConstraint ) diff --git a/app/controllers/model/PrepareUploadRequestV1.scala b/app/controllers/model/PrepareUploadRequestV1.scala index 1f59250..610ffc9 100644 --- a/app/controllers/model/PrepareUploadRequestV1.scala +++ b/app/controllers/model/PrepareUploadRequestV1.scala @@ -25,7 +25,6 @@ case class PrepareUploadRequestV1( callbackUrl: String, minimumFileSize: Option[Long], maximumFileSize: Option[Long], - expectedContentType: Option[String], successRedirect: Option[String]) extends PrepareUpload { @@ -34,7 +33,6 @@ case class PrepareUploadRequestV1( callbackUrl = callbackUrl, minimumFileSize = minimumFileSize, maximumFileSize = maximumFileSize, - expectedContentType = expectedContentType, successRedirect = successRedirect, errorRedirect = None ) @@ -47,7 +45,6 @@ object PrepareUploadRequestV1 { ((JsPath \ "callbackUrl").read[String] and (JsPath \ "minimumFileSize").readNullable[Long](min(0)) and (JsPath \ "maximumFileSize").readNullable[Long](min(0L) keepAnd max(maxFileSize)) and - (JsPath \ "expectedContentType").readNullable[String] and (JsPath \ "successRedirect").readNullable[String])(PrepareUploadRequestV1.apply _) .filter(JsonValidationError("Maximum file size must be equal or greater than minimum file size"))(request => request.minimumFileSize.getOrElse(0L) <= request.maximumFileSize.getOrElse(maxFileSize)) diff --git a/app/controllers/model/PrepareUploadRequestV2.scala b/app/controllers/model/PrepareUploadRequestV2.scala index 81a5f72..ee07285 100644 --- a/app/controllers/model/PrepareUploadRequestV2.scala +++ b/app/controllers/model/PrepareUploadRequestV2.scala @@ -26,8 +26,7 @@ case class PrepareUploadRequestV2( successRedirect: Option[String], errorRedirect: Option[String], minimumFileSize: Option[Long], - maximumFileSize: Option[Long], - expectedContentType: Option[String]) + maximumFileSize: Option[Long]) extends PrepareUpload { def toUploadSettings(uploadUrl: String): UploadSettings = UploadSettings( @@ -35,7 +34,6 @@ case class PrepareUploadRequestV2( callbackUrl = callbackUrl, minimumFileSize = minimumFileSize, maximumFileSize = maximumFileSize, - expectedContentType = expectedContentType, successRedirect = successRedirect, errorRedirect = errorRedirect ) @@ -48,8 +46,7 @@ object PrepareUploadRequestV2 { (JsPath \ "successRedirect").readNullable[String] and (JsPath \ "errorRedirect").readNullable[String] and (JsPath \ "minimumFileSize").readNullable[Long](min(0)) and - (JsPath \ "maximumFileSize").readNullable[Long](min(0L) keepAnd max(maxFileSize)) and - (JsPath \ "expectedContentType").readNullable[String])(PrepareUploadRequestV2.apply _) + (JsPath \ "maximumFileSize").readNullable[Long](min(0L) keepAnd max(maxFileSize)))(PrepareUploadRequestV2.apply _) .filter(JsonValidationError("Maximum file size must be equal or greater than minimum file size"))(request => request.minimumFileSize.getOrElse(0L) <= request.maximumFileSize.getOrElse(maxFileSize)) } diff --git a/app/services/PrepareUploadService.scala b/app/services/PrepareUploadService.scala index 341d5f2..b19c80e 100644 --- a/app/services/PrepareUploadService.scala +++ b/app/services/PrepareUploadService.scala @@ -96,7 +96,6 @@ class PrepareUploadService @Inject()( "upscan-initiate-received" -> receivedAt.toString ), contentLengthRange = ContentLengthRange(minFileSize, maxFileSize), - expectedContentType = settings.expectedContentType, successRedirect = settings.successRedirect, errorRedirect = settings.errorRedirect ) diff --git a/app/services/model/UploadSettings.scala b/app/services/model/UploadSettings.scala index 1a40029..6bf0aa2 100644 --- a/app/services/model/UploadSettings.scala +++ b/app/services/model/UploadSettings.scala @@ -21,6 +21,5 @@ case class UploadSettings( callbackUrl: String, minimumFileSize: Option[Long], maximumFileSize: Option[Long], - expectedContentType: Option[String], successRedirect: Option[String], errorRedirect: Option[String]) diff --git a/it/controllers/PrepareUploadControllerISpec.scala b/it/controllers/PrepareUploadControllerISpec.scala index 1c665f4..29cd70b 100644 --- a/it/controllers/PrepareUploadControllerISpec.scala +++ b/it/controllers/PrepareUploadControllerISpec.scala @@ -1,5 +1,5 @@ /* - * Copyright 2021 HM Revenue & Customs + * Copyright 2022 HM Revenue & Customs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -45,7 +45,6 @@ class PrepareUploadControllerISpec extends AnyWordSpecLike with should.Matchers | "callbackUrl": "https://some-url/callback", | "minimumFileSize" : 0, | "maximumFileSize" : 1024, - | "expectedContentType": "application/xml", | "successRedirect": "https://some-url/success" |}""".stripMargin) @@ -66,7 +65,6 @@ class PrepareUploadControllerISpec extends AnyWordSpecLike with should.Matchers And("the response should contain the requested upload fields") val fields = (responseJson \ "uploadRequest" \ "fields").as[Map[String, String]] fields.get("x-amz-meta-callback-url") should contain ("https://some-url/callback") - fields.get("Content-Type") should contain ("application/xml") fields.get("success_action_redirect") should contain ("https://some-url/success") } @@ -92,7 +90,6 @@ class PrepareUploadControllerISpec extends AnyWordSpecLike with should.Matchers And("the response should contain the requested upload fields") val fields = (responseJson \ "uploadRequest" \ "fields").as[Map[String, String]] fields.get("x-amz-meta-callback-url") should contain ("https://some-url/callback") - fields.get("Content-Type") shouldBe empty fields.get("success_action_redirect") shouldBe empty } @@ -103,8 +100,7 @@ class PrepareUploadControllerISpec extends AnyWordSpecLike with should.Matchers | "successRedirect": "https://some-url/success", | "errorRedirect": "https://some-url/error", | "minimumFileSize" : 0, - | "maximumFileSize" : 1024, - | "expectedContentType": "application/xml" + | "maximumFileSize" : 1024 |} """.stripMargin) @@ -127,7 +123,6 @@ class PrepareUploadControllerISpec extends AnyWordSpecLike with should.Matchers fields.get("x-amz-meta-callback-url") should contain ("https://some-url/callback") fields.get("success_action_redirect") should contain ("https://some-url/success") fields.get("error_action_redirect") should contain ("https://some-url/error") - fields.get("Content-Type") should contain ("application/xml") } "PrepareUploadController prepareUploadV2 with only mandatory request values" in { @@ -154,7 +149,6 @@ class PrepareUploadControllerISpec extends AnyWordSpecLike with should.Matchers fields.get("x-amz-meta-callback-url") should contain ("https://some-url/callback") fields.get("error_action_redirect") shouldBe empty fields.get("success_action_redirect") shouldBe empty - fields.get("Content-Type") shouldBe empty } "Upscan V1" should { @@ -251,6 +245,60 @@ class PrepareUploadControllerISpec extends AnyWordSpecLike with should.Matchers fields should contain key "policy" } } + + "PrepareUploadController prepareUploadV1 with expectedContentType" in { + val postBodyJson = Json.parse("""|{ + | "callbackUrl": "https://some-url/callback", + | "minimumFileSize" : 0, + | "maximumFileSize" : 1024, + | "expectedContentType": "application/xml", + | "successRedirect": "https://some-url/success" + |}""".stripMargin) + + Given("a request containing a User-Agent header") + val headers = FakeHeaders(Seq((USER_AGENT, SomeConsumingService))) + val initiateRequest = FakeRequest(POST, uri = "/upscan/initiate", headers, postBodyJson) + + When("a request is posted to the /initiate endpoint") + val initiateResponse = route(app, initiateRequest).get + + Then("the response should indicate success") + status(initiateResponse) shouldBe OK + + And("the response should not contain Content-Type") + val responseJson = contentAsJson(initiateResponse) + val fields = (responseJson \ "uploadRequest" \ "fields").as[Map[String, String]] + fields.get("Content-Type") shouldBe empty + } + + "PrepareUploadController prepareUploadV2 with expectedContentType" in { + val postBodyJson = Json.parse(""" + |{ + | "callbackUrl": "https://some-url/callback", + | "successRedirect": "https://some-url/success", + | "errorRedirect": "https://some-url/error", + | "minimumFileSize" : 0, + | "expectedContentType": "application/xml", + | "maximumFileSize" : 1024 + |} + """.stripMargin) + + Given("a request containing a User-Agent header") + val headers = FakeHeaders(Seq((USER_AGENT, SomeConsumingService))) + val initiateRequest = FakeRequest(POST, uri = "/upscan/v2/initiate", headers, postBodyJson) + + When("a request is posted to the /initiate endpoint") + val initiateResponse = route(app, initiateRequest).get + + Then("the response should indicate success") + status(initiateResponse) shouldBe OK + + And("the response should contain the requested upload fields") + val responseJson = contentAsJson(initiateResponse) + val fields = (responseJson \ "uploadRequest" \ "fields").as[Map[String, String]] + fields.get("Content-Type") shouldBe empty + } + } private object PrepareUploadControllerISpec { diff --git a/test/connectors/s3/S3UploadFormGeneratorSpec.scala b/test/connectors/s3/S3UploadFormGeneratorSpec.scala index ac54ba8..d6d35de 100644 --- a/test/connectors/s3/S3UploadFormGeneratorSpec.scala +++ b/test/connectors/s3/S3UploadFormGeneratorSpec.scala @@ -51,7 +51,6 @@ class S3UploadFormGeneratorSpec extends UnitSpec with GivenWhenThen { acl = "private", additionalMetadata = Map("key1" -> "value1"), contentLengthRange = ContentLengthRange(0, 1024), - expectedContentType = Some("application/xml"), successRedirect = Some("http://test.com/abc"), errorRedirect = Some("http://test.com/error") ) @@ -73,7 +72,6 @@ class S3UploadFormGeneratorSpec extends UnitSpec with GivenWhenThen { ((policy \ "conditions").get \\ "x-amz-date").head.as[String] shouldBe "19970716T192030Z" ((policy \ "conditions").get \\ "x-amz-security-token").head.as[String] shouldBe "session-token" ((policy \ "conditions").get \\ "x-amz-meta-key1").head.as[String] shouldBe "value1" - ((policy \ "conditions").get \\ "Content-Type").head.as[String] shouldBe "application/xml" ((policy \ "conditions").get \\ "success_action_redirect").head.as[String] shouldBe "http://test.com/abc" ((policy \ "conditions").get \\ "error_action_redirect").head.as[String] shouldBe "http://test.com/error" @@ -128,7 +126,6 @@ class S3UploadFormGeneratorSpec extends UnitSpec with GivenWhenThen { acl = "private", additionalMetadata = Map("key1" -> "value1"), contentLengthRange = ContentLengthRange(0, 1024), - expectedContentType = Some("application/xml"), successRedirect = Some("http://test.server/success"), errorRedirect = None ) @@ -161,7 +158,6 @@ class S3UploadFormGeneratorSpec extends UnitSpec with GivenWhenThen { acl = "private", additionalMetadata = Map("key1" -> "value1"), contentLengthRange = ContentLengthRange(0, 1024), - expectedContentType = Some("application/xml"), successRedirect = None, errorRedirect = Some("http://test.server/error") ) diff --git a/test/controllers/PrepareUploadControllerSpec.scala b/test/controllers/PrepareUploadControllerSpec.scala index 0efb065..0a415b4 100644 --- a/test/controllers/PrepareUploadControllerSpec.scala +++ b/test/controllers/PrepareUploadControllerSpec.scala @@ -81,7 +81,6 @@ class PrepareUploadControllerSpec extends UnitSpec with StubControllerComponents callbackUrl = "https://www.example.com", minimumFileSize = None, maximumFileSize = None, - expectedContentType = None, successRedirect = None, errorRedirect = None ) @@ -97,7 +96,6 @@ class PrepareUploadControllerSpec extends UnitSpec with StubControllerComponents "callbackUrl" -> "https://www.example.com", "minimumFileSize" -> 1, "maximumFileSize" -> 1024, - "expectedContentType" -> "application/pdf", "successRedirect" -> "https://www.example.com/success" ) val expectedUploadSettings = UploadSettings( @@ -105,7 +103,6 @@ class PrepareUploadControllerSpec extends UnitSpec with StubControllerComponents callbackUrl = "https://www.example.com", minimumFileSize = Some(1), maximumFileSize = Some(1024), - expectedContentType = Some("application/pdf"), successRedirect = Some("https://www.example.com/success"), errorRedirect = None ) @@ -123,7 +120,6 @@ class PrepareUploadControllerSpec extends UnitSpec with StubControllerComponents callbackUrl = "https://www.example.com", minimumFileSize = None, maximumFileSize = None, - expectedContentType = None, successRedirect = None, errorRedirect = None ) @@ -139,7 +135,6 @@ class PrepareUploadControllerSpec extends UnitSpec with StubControllerComponents "callbackUrl" -> "https://www.example.com", "minimumFileSize" -> 1, "maximumFileSize" -> 1024, - "expectedContentType" -> "application/pdf", "successRedirect" -> "https://www.example.com/success", "errorRedirect" -> "https://www.example.com/error" ) @@ -148,7 +143,6 @@ class PrepareUploadControllerSpec extends UnitSpec with StubControllerComponents callbackUrl = "https://www.example.com", minimumFileSize = Some(1), maximumFileSize = Some(1024), - expectedContentType = Some("application/pdf"), successRedirect = Some("https://www.example.com/success"), errorRedirect = Some("https://www.example.com/error") ) diff --git a/test/controllers/model/PrepareUploadRequestV2Spec.scala b/test/controllers/model/PrepareUploadRequestV2Spec.scala index ade5844..d2085b4 100644 --- a/test/controllers/model/PrepareUploadRequestV2Spec.scala +++ b/test/controllers/model/PrepareUploadRequestV2Spec.scala @@ -141,8 +141,7 @@ private object PrepareUploadRequestV2Spec { successRedirect = None, errorRedirect = None, minimumFileSize = None, - maximumFileSize = None, - expectedContentType = None) + maximumFileSize = None) def aV2RequestWithSuccessRedirectUrlOf(url: Option[String]): PrepareUploadRequestV2 = template.copy(successRedirect = url) diff --git a/test/model/PrepareUploadServiceSpec.scala b/test/model/PrepareUploadServiceSpec.scala index 3ac0036..5ad4a8a 100644 --- a/test/model/PrepareUploadServiceSpec.scala +++ b/test/model/PrepareUploadServiceSpec.scala @@ -67,9 +67,6 @@ class PrepareUploadServiceSpec extends UnitSpec with GivenWhenThen { "maxSize" -> uploadParameters.contentLengthRange.max.toString ) ++ uploadParameters.additionalMetadata.map { case (k, v) => s"x-amz-meta-$k" -> v } ++ - uploadParameters.expectedContentType.map { contentType => - "Content-Type" -> contentType - } ++ uploadParameters.successRedirect.map { "success_redirect_url" -> _ } ++ uploadParameters.errorRedirect.map { "error_redirect_url" -> _ } } @@ -94,7 +91,6 @@ class PrepareUploadServiceSpec extends UnitSpec with GivenWhenThen { callbackUrl = callbackUrl, minimumFileSize = None, maximumFileSize = None, - expectedContentType = Some("application/xml"), successRedirect = None, errorRedirect = None ) @@ -116,7 +112,6 @@ class PrepareUploadServiceSpec extends UnitSpec with GivenWhenThen { "x-amz-meta-request-id" -> "some-request-id", "minSize" -> "0", "maxSize" -> "1024", - "Content-Type" -> "application/xml", "x-amz-meta-upscan-initiate-received" -> receivedAt.toString ) @@ -139,7 +134,6 @@ class PrepareUploadServiceSpec extends UnitSpec with GivenWhenThen { callbackUrl = callbackUrl, minimumFileSize = Some(100), maximumFileSize = Some(200), - expectedContentType = None, successRedirect = None, errorRedirect = None ) @@ -169,7 +163,6 @@ class PrepareUploadServiceSpec extends UnitSpec with GivenWhenThen { callbackUrl = callbackUrl, minimumFileSize = Some(-1), maximumFileSize = Some(1024), - expectedContentType = None, successRedirect = None, errorRedirect = None ) @@ -199,7 +192,6 @@ class PrepareUploadServiceSpec extends UnitSpec with GivenWhenThen { callbackUrl = callbackUrl, minimumFileSize = Some(0), maximumFileSize = Some(1025), - expectedContentType = None, successRedirect = None, errorRedirect = None ) @@ -228,7 +220,6 @@ class PrepareUploadServiceSpec extends UnitSpec with GivenWhenThen { callbackUrl = callbackUrl, minimumFileSize = Some(1024), maximumFileSize = Some(0), - expectedContentType = None, successRedirect = None, errorRedirect = None ) @@ -257,7 +248,6 @@ class PrepareUploadServiceSpec extends UnitSpec with GivenWhenThen { callbackUrl = callbackUrl, minimumFileSize = None, maximumFileSize = None, - expectedContentType = Some("application/xml"), successRedirect = Some("https://new.service/page1"), errorRedirect = None ) @@ -279,7 +269,6 @@ class PrepareUploadServiceSpec extends UnitSpec with GivenWhenThen { "x-amz-meta-request-id" -> "some-request-id", "minSize" -> "0", "maxSize" -> "1024", - "Content-Type" -> "application/xml", "x-amz-meta-upscan-initiate-received" -> receivedAt.toString, "success_redirect_url" -> "https://new.service/page1" ) @@ -303,7 +292,6 @@ class PrepareUploadServiceSpec extends UnitSpec with GivenWhenThen { callbackUrl = callbackUrl, minimumFileSize = None, maximumFileSize = None, - expectedContentType = Some("application/xml"), successRedirect = None, errorRedirect = Some("https://new.service/error") ) @@ -325,7 +313,6 @@ class PrepareUploadServiceSpec extends UnitSpec with GivenWhenThen { "x-amz-meta-request-id" -> "some-request-id", "minSize" -> "0", "maxSize" -> "1024", - "Content-Type" -> "application/xml", "x-amz-meta-upscan-initiate-received" -> receivedAt.toString, "error_redirect_url" -> "https://new.service/error" ) From 1d7a56cf407a173a93305437cb0da9f3f47ebf3c Mon Sep 17 00:00:00 2001 From: Tom Wadeson <3607811+tomwadeson@users.noreply.github.com> Date: Fri, 29 Jul 2022 13:53:14 +0100 Subject: [PATCH 70/96] BDOG-2016 Support explicit `"consumingService"` identification --- README.md | 16 ++++ app/controllers/PrepareUploadController.scala | 5 +- app/controllers/model/PrepareUpload.scala | 1 + .../model/PrepareUploadRequestV1.scala | 6 +- .../model/PrepareUploadRequestV2.scala | 6 +- .../PrepareUploadControllerISpec.scala | 33 +++++---- .../PrepareUploadControllerSpec.scala | 73 +++++++++++++++---- .../model/PrepareUploadRequestV2Spec.scala | 16 +++- 8 files changed, 120 insertions(+), 36 deletions(-) diff --git a/README.md b/README.md index c4f5fbd..a17b981 100644 --- a/README.md +++ b/README.md @@ -74,6 +74,9 @@ The consuming service makes a POST request to `/upscan/initiate` or `upscan/v2/i This request must contain a `User-Agent` header that can be used to identify the service, but an allow list of authorised services is no longer cross-checked. The service must also provide a callbackUrl for asynchronous notification of the outcome of an upload. The callback will be made from inside the MDTP environment. Hence, the callback URL should comprise the MDTP internal callback address and not the public domain address. +**Note:** Although a `User-Agent` header is always required, consuming services may wish to explicitly identify themselves by including a `"consumingService"` field in the request body. +This functionality may be useful to you if the actual service that initiates the request is different to your preferred logical service. + **Note:** `callbackUrl` must use the `https` protocol. (Although this rule is relaxed when testing locally with [upscan-stub](https://github.com/hmrc/upscan-stub) rather than [upscan-initiate](https://github.com/hmrc/upscan-initiate). In this stubbed scenario a `callbackUrl` referring to localhost may still specify `http` as the protocol.) @@ -106,6 +109,19 @@ Example `upscan/v2/initiate` request: } ``` +Another example `upscan/v2/initiate` requests that explicitly identifies the consuming service: + +```json +{ + "callbackUrl": "https://myservice.com/callback", + "successRedirect": "https://myservice.com/nextPage", + "errorRedirect": "https://myservice.com/errorPage", + "minimumFileSize" : 0, + "maximumFileSize" : 1024, + "consumingService": "some-consuming-service" +} +``` + #### HTTP Headers: | Header name|Description|Required| diff --git a/app/controllers/PrepareUploadController.scala b/app/controllers/PrepareUploadController.scala index a2d725c..bdaca94 100644 --- a/app/controllers/PrepareUploadController.scala +++ b/app/controllers/PrepareUploadController.scala @@ -60,12 +60,13 @@ class PrepareUploadController @Inject()( Action.async(parse.json) { implicit request => val receivedAt = Instant.now(clock) - requireUserAgent[JsValue] { (_, consumingService) => + requireUserAgent[JsValue] { (_, userAgent) => withJsonBody[T] { prepareUploadRequest: T => withAllowedCallbackProtocol(prepareUploadRequest.callbackUrl) { val sessionId = hc(request).sessionId.map(_.value).getOrElse("n/a") val requestId = hc(request).requestId.map(_.value).getOrElse("n/a") - logger.debug(s"Processing request: [$prepareUploadRequest] with requestId: [$requestId] sessionId: [$sessionId] from: [$consumingService].") + logger.debug(s"Processing request: [$prepareUploadRequest] with requestId: [$requestId] sessionId: [$sessionId] from: [$userAgent].") + val consumingService = prepareUploadRequest.consumingService.getOrElse(userAgent) val result: PreparedUploadResponse = prepareUploadService .prepareUpload( diff --git a/app/controllers/model/PrepareUpload.scala b/app/controllers/model/PrepareUpload.scala index bb5c079..2953b71 100644 --- a/app/controllers/model/PrepareUpload.scala +++ b/app/controllers/model/PrepareUpload.scala @@ -20,5 +20,6 @@ import services.model.UploadSettings trait PrepareUpload { def callbackUrl: String + def consumingService: Option[String] def toUploadSettings(uploadUrl: String): UploadSettings } diff --git a/app/controllers/model/PrepareUploadRequestV1.scala b/app/controllers/model/PrepareUploadRequestV1.scala index 610ffc9..3376f6b 100644 --- a/app/controllers/model/PrepareUploadRequestV1.scala +++ b/app/controllers/model/PrepareUploadRequestV1.scala @@ -25,7 +25,8 @@ case class PrepareUploadRequestV1( callbackUrl: String, minimumFileSize: Option[Long], maximumFileSize: Option[Long], - successRedirect: Option[String]) + successRedirect: Option[String], + consumingService: Option[String]) extends PrepareUpload { def toUploadSettings(uploadUrl: String): UploadSettings = UploadSettings( @@ -45,7 +46,8 @@ object PrepareUploadRequestV1 { ((JsPath \ "callbackUrl").read[String] and (JsPath \ "minimumFileSize").readNullable[Long](min(0)) and (JsPath \ "maximumFileSize").readNullable[Long](min(0L) keepAnd max(maxFileSize)) and - (JsPath \ "successRedirect").readNullable[String])(PrepareUploadRequestV1.apply _) + (JsPath \ "successRedirect").readNullable[String] and + (JsPath \ "consumingService").readNullable[String])(PrepareUploadRequestV1.apply _) .filter(JsonValidationError("Maximum file size must be equal or greater than minimum file size"))(request => request.minimumFileSize.getOrElse(0L) <= request.maximumFileSize.getOrElse(maxFileSize)) diff --git a/app/controllers/model/PrepareUploadRequestV2.scala b/app/controllers/model/PrepareUploadRequestV2.scala index ee07285..1481a44 100644 --- a/app/controllers/model/PrepareUploadRequestV2.scala +++ b/app/controllers/model/PrepareUploadRequestV2.scala @@ -26,7 +26,8 @@ case class PrepareUploadRequestV2( successRedirect: Option[String], errorRedirect: Option[String], minimumFileSize: Option[Long], - maximumFileSize: Option[Long]) + maximumFileSize: Option[Long], + consumingService: Option[String]) extends PrepareUpload { def toUploadSettings(uploadUrl: String): UploadSettings = UploadSettings( @@ -46,7 +47,8 @@ object PrepareUploadRequestV2 { (JsPath \ "successRedirect").readNullable[String] and (JsPath \ "errorRedirect").readNullable[String] and (JsPath \ "minimumFileSize").readNullable[Long](min(0)) and - (JsPath \ "maximumFileSize").readNullable[Long](min(0L) keepAnd max(maxFileSize)))(PrepareUploadRequestV2.apply _) + (JsPath \ "maximumFileSize").readNullable[Long](min(0L) keepAnd max(maxFileSize)) and + (JsPath \ "consumingService").readNullable[String])(PrepareUploadRequestV2.apply _) .filter(JsonValidationError("Maximum file size must be equal or greater than minimum file size"))(request => request.minimumFileSize.getOrElse(0L) <= request.maximumFileSize.getOrElse(maxFileSize)) } diff --git a/it/controllers/PrepareUploadControllerISpec.scala b/it/controllers/PrepareUploadControllerISpec.scala index 29cd70b..26b5c9d 100644 --- a/it/controllers/PrepareUploadControllerISpec.scala +++ b/it/controllers/PrepareUploadControllerISpec.scala @@ -41,12 +41,13 @@ class PrepareUploadControllerISpec extends AnyWordSpecLike with should.Matchers .build() "PrepareUploadController prepareUploadV1 with all request values" in { - val postBodyJson = Json.parse("""|{ - | "callbackUrl": "https://some-url/callback", - | "minimumFileSize" : 0, - | "maximumFileSize" : 1024, - | "successRedirect": "https://some-url/success" - |}""".stripMargin) + val postBodyJson = Json.parse(s"""|{ + | "callbackUrl": "https://some-url/callback", + | "minimumFileSize" : 0, + | "maximumFileSize" : 1024, + | "successRedirect": "https://some-url/success", + | "consumingService": "$SomeOtherConsumingService" + |}""".stripMargin) Given("a request containing a User-Agent header") val headers = FakeHeaders(Seq((USER_AGENT, SomeConsumingService))) @@ -66,6 +67,7 @@ class PrepareUploadControllerISpec extends AnyWordSpecLike with should.Matchers val fields = (responseJson \ "uploadRequest" \ "fields").as[Map[String, String]] fields.get("x-amz-meta-callback-url") should contain ("https://some-url/callback") fields.get("success_action_redirect") should contain ("https://some-url/success") + fields.get("x-amz-meta-consuming-service") should contain (SomeOtherConsumingService) } "PrepareUploadController prepareUploadV1 with only mandatory request values" in { @@ -91,16 +93,18 @@ class PrepareUploadControllerISpec extends AnyWordSpecLike with should.Matchers val fields = (responseJson \ "uploadRequest" \ "fields").as[Map[String, String]] fields.get("x-amz-meta-callback-url") should contain ("https://some-url/callback") fields.get("success_action_redirect") shouldBe empty + fields.get("x-amz-meta-consuming-service") should contain (SomeConsumingService) } "PrepareUploadController prepareUploadV2 with all request values" in { - val postBodyJson = Json.parse(""" + val postBodyJson = Json.parse(s""" |{ - | "callbackUrl": "https://some-url/callback", - | "successRedirect": "https://some-url/success", - | "errorRedirect": "https://some-url/error", - | "minimumFileSize" : 0, - | "maximumFileSize" : 1024 + | "callbackUrl": "https://some-url/callback", + | "successRedirect": "https://some-url/success", + | "errorRedirect": "https://some-url/error", + | "minimumFileSize" : 0, + | "maximumFileSize" : 1024, + | "consumingService": "$SomeOtherConsumingService" |} """.stripMargin) @@ -123,6 +127,7 @@ class PrepareUploadControllerISpec extends AnyWordSpecLike with should.Matchers fields.get("x-amz-meta-callback-url") should contain ("https://some-url/callback") fields.get("success_action_redirect") should contain ("https://some-url/success") fields.get("error_action_redirect") should contain ("https://some-url/error") + fields.get("x-amz-meta-consuming-service") should contain (SomeOtherConsumingService) } "PrepareUploadController prepareUploadV2 with only mandatory request values" in { @@ -149,6 +154,7 @@ class PrepareUploadControllerISpec extends AnyWordSpecLike with should.Matchers fields.get("x-amz-meta-callback-url") should contain ("https://some-url/callback") fields.get("error_action_redirect") shouldBe empty fields.get("success_action_redirect") shouldBe empty + fields.get("x-amz-meta-consuming-service") should contain (SomeConsumingService) } "Upscan V1" should { @@ -303,4 +309,5 @@ class PrepareUploadControllerISpec extends AnyWordSpecLike with should.Matchers private object PrepareUploadControllerISpec { val SomeConsumingService = "PrepareUploadControllerISpec" -} \ No newline at end of file + val SomeOtherConsumingService = "some-other-consuming-service" +} diff --git a/test/controllers/PrepareUploadControllerSpec.scala b/test/controllers/PrepareUploadControllerSpec.scala index 0a415b4..588ae9a 100644 --- a/test/controllers/PrepareUploadControllerSpec.scala +++ b/test/controllers/PrepareUploadControllerSpec.scala @@ -17,7 +17,6 @@ package controllers import java.time.Clock - import akka.actor.ActorSystem import akka.stream.ActorMaterializer import config.ServiceConfiguration @@ -34,6 +33,8 @@ import services.PrepareUploadService import services.model.UploadSettings import test.UnitSpec import Helpers.contentAsJson +import controllers.PrepareUploadControllerSpec.{ConsumingService, UserAgent} + import scala.concurrent.Future import scala.concurrent.duration._ import scala.language.postfixOps @@ -86,8 +87,16 @@ class PrepareUploadControllerSpec extends UnitSpec with StubControllerComponents ) val _ = new WithV1SuccessFixture { - behave like successfulInitiate(config, _.prepareUploadV1(), minimalRequestBody, requestId = None, sessionId = None, - expectedUploadSettings) + behave like + successfulInitiate( + config = config, + actionUnderTest = _.prepareUploadV1(), + requestBody = minimalRequestBody, + requestId = None, + sessionId = None, + expectedUploadSettings = expectedUploadSettings, + expectedConsumingService = UserAgent + ) } } @@ -96,7 +105,8 @@ class PrepareUploadControllerSpec extends UnitSpec with StubControllerComponents "callbackUrl" -> "https://www.example.com", "minimumFileSize" -> 1, "maximumFileSize" -> 1024, - "successRedirect" -> "https://www.example.com/success" + "successRedirect" -> "https://www.example.com/success", + "consumingService" -> ConsumingService ) val expectedUploadSettings = UploadSettings( uploadUrl = "https://inbound-bucket.s3.amazonaws.com", @@ -108,8 +118,16 @@ class PrepareUploadControllerSpec extends UnitSpec with StubControllerComponents ) val _ = new WithV1SuccessFixture { - behave like successfulInitiate(config, _.prepareUploadV1(), maximalRequestBody, requestId = Some("a-request-id"), - sessionId = Some("a-session-id"), expectedUploadSettings) + behave like + successfulInitiate( + config = config, + actionUnderTest = _.prepareUploadV1(), + requestBody = maximalRequestBody, + requestId = Some("a-request-id"), + sessionId = Some("a-session-id"), + expectedUploadSettings = expectedUploadSettings, + expectedConsumingService = ConsumingService + ) } } @@ -125,8 +143,16 @@ class PrepareUploadControllerSpec extends UnitSpec with StubControllerComponents ) val _ = new WithV2SuccessFixture { - behave like successfulInitiate(config, _.prepareUploadV2(), minimalRequestBody, requestId = None, sessionId = None, - expectedUploadSettings) + behave like + successfulInitiate( + config = config, + actionUnderTest = _.prepareUploadV2(), + requestBody = minimalRequestBody, + requestId = None, + sessionId = None, + expectedUploadSettings = expectedUploadSettings, + expectedConsumingService = UserAgent + ) } } @@ -136,7 +162,8 @@ class PrepareUploadControllerSpec extends UnitSpec with StubControllerComponents "minimumFileSize" -> 1, "maximumFileSize" -> 1024, "successRedirect" -> "https://www.example.com/success", - "errorRedirect" -> "https://www.example.com/error" + "errorRedirect" -> "https://www.example.com/error", + "consumingService" -> ConsumingService ) val expectedUploadSettings = UploadSettings( uploadUrl = "https://upload-proxy.com/v1/uploads/inbound-bucket", @@ -148,8 +175,16 @@ class PrepareUploadControllerSpec extends UnitSpec with StubControllerComponents ) val _ = new WithV2SuccessFixture { - behave like successfulInitiate(config, _.prepareUploadV2(), maximalRequestBody, requestId = Some("a-request-id"), - sessionId = Some("a-session-id"), expectedUploadSettings) + behave like + successfulInitiate( + config = config, + actionUnderTest = _.prepareUploadV2(), + requestBody = maximalRequestBody, + requestId = Some("a-request-id"), + sessionId = Some("a-session-id"), + expectedUploadSettings = expectedUploadSettings, + expectedConsumingService = ConsumingService + ) } } @@ -159,13 +194,14 @@ class PrepareUploadControllerSpec extends UnitSpec with StubControllerComponents requestBody: JsValue, requestId: Option[String], sessionId: Option[String], - expectedUploadSettings: UploadSettings): Unit = { + expectedUploadSettings: UploadSettings, + expectedConsumingService: String): Unit = { "prepare upload request" in new WithGlobalFileSizeLimitFixture { val controller = new PrepareUploadController(prepareUploadService, config, clock, stubControllerComponents()) Given("a valid initiate request") - val headers = (USER_AGENT, "SOME-USER-AGENT") +: Seq( + val headers = (USER_AGENT, UserAgent) +: Seq( requestId.map(Tuple2("x-request-id", _)), sessionId.map(Tuple2("x-session-id", _)) ).flatten @@ -181,7 +217,7 @@ class PrepareUploadControllerSpec extends UnitSpec with StubControllerComponents when(prepareUploadService.prepareUpload( expectedUploadSettings, - "SOME-USER-AGENT", + expectedConsumingService, requestId.getOrElse("n/a"), sessionId.getOrElse("n/a"), clock.instant) @@ -229,7 +265,7 @@ class PrepareUploadControllerSpec extends UnitSpec with StubControllerComponents Given("there is an invalid upload request") val request = FakeRequest().withHeaders( - (USER_AGENT, "SOME-USER-AGENT"), + (USER_AGENT, UserAgent), ("x-session-id", "some-session-id"), ("x-request-id", "some-request-id") ).withBody(Json.obj("invalid" -> "body")) @@ -246,7 +282,7 @@ class PrepareUploadControllerSpec extends UnitSpec with StubControllerComponents Given("there is an invalid upload request") val request = FakeRequest().withHeaders( - (USER_AGENT, "SOME-USER-AGENT"), + (USER_AGENT, UserAgent), ("x-session-id", "some-session-id") ).withBody(Json.obj( "callbackUrl" -> "https://www.example.com", @@ -314,3 +350,8 @@ class PrepareUploadControllerSpec extends UnitSpec with StubControllerComponents } } } + +object PrepareUploadControllerSpec { + val UserAgent = "user-agent" + val ConsumingService = "some-consuming-service" +} diff --git a/test/controllers/model/PrepareUploadRequestV2Spec.scala b/test/controllers/model/PrepareUploadRequestV2Spec.scala index d2085b4..f21e6c8 100644 --- a/test/controllers/model/PrepareUploadRequestV2Spec.scala +++ b/test/controllers/model/PrepareUploadRequestV2Spec.scala @@ -126,6 +126,18 @@ class PrepareUploadRequestV2Spec extends UnitSpec with EitherValues { parseResult.isError shouldBe true } } + + "specifying an explicit consuming service" should { + "deserialise accordingly" in { + val request1 = s"""{"callbackUrl":"$CallbackUrl", "consumingService": "$ConsumingService"}""" + val prepareUploadRequest1 = Json.parse(request1).validate(PrepareUploadRequestV2.reads(MaxFileSize)).get + prepareUploadRequest1.consumingService shouldBe Some(ConsumingService) + + val request2 = s"""{"callbackUrl":"$CallbackUrl"}""" + val prepareUploadRequest2 = Json.parse(request2).validate(PrepareUploadRequestV2.reads(MaxFileSize)).get + prepareUploadRequest2.consumingService shouldBe None + } + } } } @@ -135,13 +147,15 @@ private object PrepareUploadRequestV2Spec { val SuccessRedirectUrl = "https://myservice.com/nextPage" val ErrorRedirectUrl = "https://myservice.com/errorPage" val MaxFileSize = 512 + val ConsumingService = "some-consuming-service" private val template = PrepareUploadRequestV2( callbackUrl = CallbackUrl, successRedirect = None, errorRedirect = None, minimumFileSize = None, - maximumFileSize = None) + maximumFileSize = None, + consumingService = None) def aV2RequestWithSuccessRedirectUrlOf(url: Option[String]): PrepareUploadRequestV2 = template.copy(successRedirect = url) From 44d77250d7a2ad83a054b9d942e2f4788107c030 Mon Sep 17 00:00:00 2001 From: Tom Wadeson <3607811+tomwadeson@users.noreply.github.com> Date: Fri, 29 Jul 2022 14:07:05 +0100 Subject: [PATCH 71/96] BDOG-2016 Update .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 5e36931..0515f20 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +.bsp/ .cache /.classpath db From db60d934ccf49b21c9052511cd9cbaefc276d309 Mon Sep 17 00:00:00 2001 From: Tom Wadeson <3607811+tomwadeson@users.noreply.github.com> Date: Fri, 29 Jul 2022 14:07:29 +0100 Subject: [PATCH 72/96] BDOG-2016 Apply PR-Commenter suggestions See: https://github.com/hmrc/upscan-initiate/pull/81#issuecomment-1199259379 --- conf/application.conf | 8 -------- project/build.properties | 2 +- project/plugins.sbt | 2 +- 3 files changed, 2 insertions(+), 10 deletions(-) diff --git a/conf/application.conf b/conf/application.conf index 254be8a..a01a415 100644 --- a/conf/application.conf +++ b/conf/application.conf @@ -40,17 +40,9 @@ play.modules.enabled += "uk.gov.hmrc.play.bootstrap.backend.BackendModule" play.modules.enabled += "UpscanInitiateModule" play.modules.enabled += "connectors.s3.S3Module" -play.http.filters = "uk.gov.hmrc.play.bootstrap.backend.filters.BackendFilters" - # Json error handler play.http.errorHandler = "uk.gov.hmrc.play.bootstrap.backend.http.JsonErrorHandler" -# Secret key -# ~~~~~ -# The secret key is used to secure cryptographics functions. -# If you deploy your application to several instances be sure to use the same key! -play.http.secret.key="mKW4mMngaPz1UVOrEkgkJrZT9kloW9Neva5wUFDUq84n9xlz42lQMGrtlrQ2bMF4" - # The application languages # ~~~~~ play.i18n.langs = ["en"] diff --git a/project/build.properties b/project/build.properties index f17a59c..88dc997 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1 +1 @@ -sbt.version=1.4.9 \ No newline at end of file +sbt.version=1.5.8 \ No newline at end of file diff --git a/project/plugins.sbt b/project/plugins.sbt index 1f054dd..5bcce57 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -6,7 +6,7 @@ resolvers += Resolver.url("HMRC-open-artefacts-ivy2", url("https://open.artefact addSbtPlugin("com.typesafe.play" % "sbt-plugin" % "2.8.7") -addSbtPlugin("uk.gov.hmrc" % "sbt-auto-build" % "3.4.0") +addSbtPlugin("uk.gov.hmrc" % "sbt-auto-build" % "3.7.0") addSbtPlugin("uk.gov.hmrc" % "sbt-distributables" % "2.1.0") From 494b3629bcc77cd76084b27383d55edda2291884 Mon Sep 17 00:00:00 2001 From: Tom Wadeson <3607811+tomwadeson@users.noreply.github.com> Date: Mon, 1 Aug 2022 15:30:21 +0100 Subject: [PATCH 73/96] BDOG-2016 Consolidate `PrepareUploadRequestV*` models --- app/controllers/PrepareUploadController.scala | 50 +++-- app/controllers/model/PrepareUpload.scala | 25 --- ...estV1.scala => PrepareUploadRequest.scala} | 47 ++--- .../model/PrepareUploadRequestV2.scala | 54 ----- app/services/PrepareUploadService.scala | 40 ++-- app/services/model/UploadSettings.scala | 16 +- .../PrepareUploadControllerSpec.scala | 147 +++++++------ .../model/PrepareUploadRequestSpec.scala | 151 ++++++++++++++ .../model/PrepareUploadRequestV2Spec.scala | 165 --------------- test/model/PrepareUploadServiceSpec.scala | 197 +++++++++--------- 10 files changed, 418 insertions(+), 474 deletions(-) delete mode 100644 app/controllers/model/PrepareUpload.scala rename app/controllers/model/{PrepareUploadRequestV1.scala => PrepareUploadRequest.scala} (52%) delete mode 100644 app/controllers/model/PrepareUploadRequestV2.scala create mode 100644 test/controllers/model/PrepareUploadRequestSpec.scala delete mode 100644 test/controllers/model/PrepareUploadRequestV2Spec.scala diff --git a/app/controllers/PrepareUploadController.scala b/app/controllers/PrepareUploadController.scala index bdaca94..f18124c 100644 --- a/app/controllers/PrepareUploadController.scala +++ b/app/controllers/PrepareUploadController.scala @@ -18,14 +18,15 @@ package controllers import java.net.URL import java.time.{Clock, Instant} - import config.ServiceConfiguration -import controllers.model.{PrepareUpload, PrepareUploadRequestV1, PrepareUploadRequestV2, PreparedUploadResponse} +import controllers.model.{PrepareUploadRequest, PreparedUploadResponse} + import javax.inject.{Inject, Singleton} import play.api.Logging import play.api.libs.json._ import play.api.mvc.{Action, ControllerComponents, Result} import services.PrepareUploadService +import services.model.UploadSettings import uk.gov.hmrc.play.bootstrap.backend.controller.BackendController import utils.UserAgentFilter @@ -39,42 +40,55 @@ class PrepareUploadController @Inject()( clock: Clock, controllerComponents: ControllerComponents) extends BackendController(controllerComponents) with UserAgentFilter with Logging { - implicit val prepareUploadRequestReads: Reads[PrepareUploadRequestV1] = - PrepareUploadRequestV1.reads(prepareUploadService.globalFileSizeLimit) + val prepareUploadRequestReadsV1: Reads[PrepareUploadRequest] = + PrepareUploadRequest.readsV1(prepareUploadService.globalFileSizeLimit) - implicit val prepareUploadRequestV2Reads: Reads[PrepareUploadRequestV2] = - PrepareUploadRequestV2.reads(prepareUploadService.globalFileSizeLimit) + val prepareUploadRequestReadsV2: Reads[PrepareUploadRequest] = + PrepareUploadRequest.readsV2(prepareUploadService.globalFileSizeLimit) + /** + * V1 of the API supports direct upload to an S3 bucket and *does not support* error redirects in the event of failure + */ def prepareUploadV1(): Action[JsValue] = { val uploadUrl = s"https://${configuration.inboundBucketName}.s3.amazonaws.com" - prepareUpload[PrepareUploadRequestV1](uploadUrl) + prepareUpload(uploadUrl)(prepareUploadRequestReadsV1) } + /** + * V2 of the API supports upload to an S3 bucket via a proxy that additionally supports error redirects in the event of failure + */ def prepareUploadV2(): Action[JsValue] = { val uploadUrl = s"${configuration.uploadProxyUrl}/v1/uploads/${configuration.inboundBucketName}" - prepareUpload[PrepareUploadRequestV2](uploadUrl) + prepareUpload(uploadUrl)(prepareUploadRequestReadsV2) } - private def prepareUpload[T <: PrepareUpload](uploadUrl: String) - (implicit reads: Reads[T], manifest: Manifest[T]): Action[JsValue] = + private def prepareUpload(uploadUrl: String)( + implicit reads: Reads[PrepareUploadRequest]): Action[JsValue] = Action.async(parse.json) { implicit request => val receivedAt = Instant.now(clock) requireUserAgent[JsValue] { (_, userAgent) => - withJsonBody[T] { prepareUploadRequest: T => + withJsonBody[PrepareUploadRequest] { prepareUploadRequest => withAllowedCallbackProtocol(prepareUploadRequest.callbackUrl) { + val sessionId = hc(request).sessionId.map(_.value).getOrElse("n/a") val requestId = hc(request).requestId.map(_.value).getOrElse("n/a") logger.debug(s"Processing request: [$prepareUploadRequest] with requestId: [$requestId] sessionId: [$sessionId] from: [$userAgent].") - val consumingService = prepareUploadRequest.consumingService.getOrElse(userAgent) - val result: PreparedUploadResponse = + + val settings = + UploadSettings( + uploadUrl = uploadUrl, + userAgent = userAgent, + prepareUploadRequest = prepareUploadRequest + ) + + val result = prepareUploadService .prepareUpload( - settings = prepareUploadRequest.toUploadSettings(uploadUrl), - consumingService = consumingService, - requestId = requestId, - sessionId = sessionId, - receivedAt = receivedAt + settings = settings, + requestId = requestId, + sessionId = sessionId, + receivedAt = receivedAt ) Future.successful(Ok(Json.toJson(result)(PreparedUploadResponse.writes))) diff --git a/app/controllers/model/PrepareUpload.scala b/app/controllers/model/PrepareUpload.scala deleted file mode 100644 index 2953b71..0000000 --- a/app/controllers/model/PrepareUpload.scala +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright 2022 HM Revenue & Customs - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package controllers.model - -import services.model.UploadSettings - -trait PrepareUpload { - def callbackUrl: String - def consumingService: Option[String] - def toUploadSettings(uploadUrl: String): UploadSettings -} diff --git a/app/controllers/model/PrepareUploadRequestV1.scala b/app/controllers/model/PrepareUploadRequest.scala similarity index 52% rename from app/controllers/model/PrepareUploadRequestV1.scala rename to app/controllers/model/PrepareUploadRequest.scala index 3376f6b..ee288e7 100644 --- a/app/controllers/model/PrepareUploadRequestV1.scala +++ b/app/controllers/model/PrepareUploadRequest.scala @@ -16,39 +16,34 @@ package controllers.model +import play.api.libs.json.Reads import play.api.libs.functional.syntax._ +import play.api.libs.json._ import play.api.libs.json.Reads.{max, min} -import play.api.libs.json.{JsPath, JsonValidationError, Reads} -import services.model.UploadSettings -case class PrepareUploadRequestV1( +final case class PrepareUploadRequest( callbackUrl: String, minimumFileSize: Option[Long], maximumFileSize: Option[Long], successRedirect: Option[String], - consumingService: Option[String]) - extends PrepareUpload { - - def toUploadSettings(uploadUrl: String): UploadSettings = UploadSettings( - uploadUrl = uploadUrl, - callbackUrl = callbackUrl, - minimumFileSize = minimumFileSize, - maximumFileSize = maximumFileSize, - successRedirect = successRedirect, - errorRedirect = None - ) - -} - -object PrepareUploadRequestV1 { - - def reads(maxFileSize: Long): Reads[PrepareUploadRequestV1] = - ((JsPath \ "callbackUrl").read[String] and - (JsPath \ "minimumFileSize").readNullable[Long](min(0)) and - (JsPath \ "maximumFileSize").readNullable[Long](min(0L) keepAnd max(maxFileSize)) and - (JsPath \ "successRedirect").readNullable[String] and - (JsPath \ "consumingService").readNullable[String])(PrepareUploadRequestV1.apply _) + errorRedirect: Option[String], + consumingService: Option[String] +) + +object PrepareUploadRequest { + + def readsV1(maxFileSize: Long): Reads[PrepareUploadRequest] = + readsV2(maxFileSize) + .map(_.copy(errorRedirect = None)) + + def readsV2(maxFileSize: Long): Reads[PrepareUploadRequest] = + ( (__ \ "callbackUrl" ).read[String] + ~ (__ \ "minimumFileSize" ).readNullable[Long](min(0L)) + ~ (__ \ "maximumFileSize" ).readNullable[Long](min(0L) keepAnd max(maxFileSize)) + ~ (__ \ "successRedirect" ).readNullable[String] + ~ (__ \ "errorRedirect" ).readNullable[String] + ~ (__ \ "consumingService").readNullable[String] + )(PrepareUploadRequest.apply _) .filter(JsonValidationError("Maximum file size must be equal or greater than minimum file size"))(request => request.minimumFileSize.getOrElse(0L) <= request.maximumFileSize.getOrElse(maxFileSize)) - } diff --git a/app/controllers/model/PrepareUploadRequestV2.scala b/app/controllers/model/PrepareUploadRequestV2.scala deleted file mode 100644 index 1481a44..0000000 --- a/app/controllers/model/PrepareUploadRequestV2.scala +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright 2022 HM Revenue & Customs - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package controllers.model - -import play.api.libs.functional.syntax._ -import play.api.libs.json.Reads.{max, min} -import play.api.libs.json.{JsPath, JsonValidationError, Reads} -import services.model.UploadSettings - -case class PrepareUploadRequestV2( - callbackUrl: String, - successRedirect: Option[String], - errorRedirect: Option[String], - minimumFileSize: Option[Long], - maximumFileSize: Option[Long], - consumingService: Option[String]) - extends PrepareUpload { - - def toUploadSettings(uploadUrl: String): UploadSettings = UploadSettings( - uploadUrl = uploadUrl, - callbackUrl = callbackUrl, - minimumFileSize = minimumFileSize, - maximumFileSize = maximumFileSize, - successRedirect = successRedirect, - errorRedirect = errorRedirect - ) -} - -object PrepareUploadRequestV2 { - - def reads(maxFileSize: Long): Reads[PrepareUploadRequestV2] = - ((JsPath \ "callbackUrl").read[String] and - (JsPath \ "successRedirect").readNullable[String] and - (JsPath \ "errorRedirect").readNullable[String] and - (JsPath \ "minimumFileSize").readNullable[Long](min(0)) and - (JsPath \ "maximumFileSize").readNullable[Long](min(0L) keepAnd max(maxFileSize)) and - (JsPath \ "consumingService").readNullable[String])(PrepareUploadRequestV2.apply _) - .filter(JsonValidationError("Maximum file size must be equal or greater than minimum file size"))(request => - request.minimumFileSize.getOrElse(0L) <= request.maximumFileSize.getOrElse(maxFileSize)) -} diff --git a/app/services/PrepareUploadService.scala b/app/services/PrepareUploadService.scala index b19c80e..b4299d8 100644 --- a/app/services/PrepareUploadService.scala +++ b/app/services/PrepareUploadService.scala @@ -17,11 +17,11 @@ package services import java.time.Instant - import com.kenshoo.play.metrics.Metrics import config.ServiceConfiguration import connectors.model.{ContentLengthRange, UploadFormGenerator, UploadParameters} import controllers.model.{PreparedUploadResponse, Reference, UploadFormTemplate} + import javax.inject.{Inject, Singleton} import org.slf4j.MDC import play.api.Logging @@ -35,7 +35,6 @@ class PrepareUploadService @Inject()( def prepareUpload( settings: UploadSettings, - consumingService: String, requestId: String, sessionId: String, receivedAt: Instant): PreparedUploadResponse = { @@ -44,20 +43,21 @@ class PrepareUploadService @Inject()( val result = PreparedUploadResponse( - reference = reference, - uploadRequest = generatePost( - key = reference, - expiration = expiration, - settings = settings, - consumingService = consumingService, - requestId = requestId, - sessionId = sessionId, - receivedAt = receivedAt) + reference = reference, + uploadRequest = + generatePost( + key = reference, + expiration = expiration, + settings = settings, + requestId = requestId, + sessionId = sessionId, + receivedAt = receivedAt + ) ) try { MDC.put("file-reference", reference.toString) - logger.info(s"Allocated key=[${reference.value}] to uploadRequest with requestId=[$requestId] sessionId=[$sessionId] from [$consumingService].") + logger.info(s"Allocated key=[${reference.value}] to uploadRequest with requestId=[$requestId] sessionId=[$sessionId] from [${settings.consumingService}].") logger.debug(s"Prepared upload response [$result].") metrics.defaultRegistry.counter("uploadInitiated").inc() @@ -71,13 +71,12 @@ class PrepareUploadService @Inject()( key: Reference, expiration: Instant, settings: UploadSettings, - consumingService: String, requestId: String, sessionId: String, receivedAt: Instant): UploadFormTemplate = { - val minFileSize = settings.minimumFileSize.getOrElse(0L) - val maxFileSize = settings.maximumFileSize.getOrElse(globalFileSizeLimit) + val minFileSize = settings.prepareUploadRequest.minimumFileSize.getOrElse(0L) + val maxFileSize = settings.prepareUploadRequest.maximumFileSize.getOrElse(globalFileSizeLimit) require(minFileSize >= 0, "Minimum file size is less than 0") require(maxFileSize <= globalFileSizeLimit, "Maximum file size is greater than global maximum file size") @@ -89,21 +88,20 @@ class PrepareUploadService @Inject()( objectKey = key.value, acl = "private", additionalMetadata = Map( - "callback-url" -> settings.callbackUrl, - "consuming-service" -> consumingService, + "callback-url" -> settings.prepareUploadRequest.callbackUrl, + "consuming-service" -> settings.consumingService, "session-id" -> sessionId, "request-id" -> requestId, "upscan-initiate-received" -> receivedAt.toString ), contentLengthRange = ContentLengthRange(minFileSize, maxFileSize), - successRedirect = settings.successRedirect, - errorRedirect = settings.errorRedirect + successRedirect = settings.prepareUploadRequest.successRedirect, + errorRedirect = settings.prepareUploadRequest.errorRedirect ) val form = postSigner.generateFormFields(uploadParameters) - val endpoint = settings.uploadUrl - UploadFormTemplate(endpoint, form) + UploadFormTemplate(settings.uploadUrl, form) } def globalFileSizeLimit: Long = configuration.globalFileSizeLimit diff --git a/app/services/model/UploadSettings.scala b/app/services/model/UploadSettings.scala index 6bf0aa2..ba237ec 100644 --- a/app/services/model/UploadSettings.scala +++ b/app/services/model/UploadSettings.scala @@ -16,10 +16,14 @@ package services.model -case class UploadSettings( +import controllers.model.PrepareUploadRequest + +final case class UploadSettings( uploadUrl: String, - callbackUrl: String, - minimumFileSize: Option[Long], - maximumFileSize: Option[Long], - successRedirect: Option[String], - errorRedirect: Option[String]) + userAgent: String, + prepareUploadRequest: PrepareUploadRequest +) { + + lazy val consumingService: String = + prepareUploadRequest.consumingService.getOrElse(userAgent) +} diff --git a/test/controllers/PrepareUploadControllerSpec.scala b/test/controllers/PrepareUploadControllerSpec.scala index 588ae9a..414e677 100644 --- a/test/controllers/PrepareUploadControllerSpec.scala +++ b/test/controllers/PrepareUploadControllerSpec.scala @@ -20,7 +20,7 @@ import java.time.Clock import akka.actor.ActorSystem import akka.stream.ActorMaterializer import config.ServiceConfiguration -import controllers.model.{PreparedUploadResponse, Reference, UploadFormTemplate} +import controllers.model.{PrepareUploadRequest, PreparedUploadResponse, Reference, UploadFormTemplate} import org.scalatest.GivenWhenThen import play.api.http.HeaderNames.USER_AGENT import play.api.http.Status.{BAD_REQUEST, OK} @@ -33,7 +33,7 @@ import services.PrepareUploadService import services.model.UploadSettings import test.UnitSpec import Helpers.contentAsJson -import controllers.PrepareUploadControllerSpec.{ConsumingService, UserAgent} +import controllers.PrepareUploadControllerSpec.{ConsumingService, UserAgent, requestTemplate, settingsTemplate} import scala.concurrent.Future import scala.concurrent.duration._ @@ -76,15 +76,14 @@ class PrepareUploadControllerSpec extends UnitSpec with StubControllerComponents private trait WithV2BadRequestFixture extends WithV1BadRequestFixture with WithUploadProxyUrl "V1 initiate with minimal request settings" should { - val minimalRequestBody = Json.obj("callbackUrl" -> "https://www.example.com") - val expectedUploadSettings = UploadSettings( - uploadUrl = "https://inbound-bucket.s3.amazonaws.com", - callbackUrl = "https://www.example.com", - minimumFileSize = None, - maximumFileSize = None, - successRedirect = None, - errorRedirect = None - ) + val minimalRequestBody = + Json.obj("callbackUrl" -> "https://www.example.com") + + val expectedUploadSettings = + settingsTemplate + .copy( + uploadUrl = "https://inbound-bucket.s3.amazonaws.com" + ) val _ = new WithV1SuccessFixture { behave like @@ -94,28 +93,33 @@ class PrepareUploadControllerSpec extends UnitSpec with StubControllerComponents requestBody = minimalRequestBody, requestId = None, sessionId = None, - expectedUploadSettings = expectedUploadSettings, - expectedConsumingService = UserAgent + expectedUploadSettings = expectedUploadSettings ) } } "V1 initiate with all request settings" should { - val maximalRequestBody = Json.obj( - "callbackUrl" -> "https://www.example.com", - "minimumFileSize" -> 1, - "maximumFileSize" -> 1024, - "successRedirect" -> "https://www.example.com/success", - "consumingService" -> ConsumingService - ) - val expectedUploadSettings = UploadSettings( - uploadUrl = "https://inbound-bucket.s3.amazonaws.com", - callbackUrl = "https://www.example.com", - minimumFileSize = Some(1), - maximumFileSize = Some(1024), - successRedirect = Some("https://www.example.com/success"), - errorRedirect = None - ) + val maximalRequestBody = + Json.obj( + "callbackUrl" -> "https://www.example.com", + "minimumFileSize" -> 1, + "maximumFileSize" -> 1024, + "successRedirect" -> "https://www.example.com/success", + "consumingService" -> ConsumingService + ) + + val expectedUploadSettings = + settingsTemplate + .copy( + uploadUrl = "https://inbound-bucket.s3.amazonaws.com", + prepareUploadRequest = + requestTemplate.copy( + minimumFileSize = Some(1), + maximumFileSize = Some(1024), + successRedirect = Some("https://www.example.com/success"), + consumingService = Some(ConsumingService) + ) + ) val _ = new WithV1SuccessFixture { behave like @@ -125,22 +129,20 @@ class PrepareUploadControllerSpec extends UnitSpec with StubControllerComponents requestBody = maximalRequestBody, requestId = Some("a-request-id"), sessionId = Some("a-session-id"), - expectedUploadSettings = expectedUploadSettings, - expectedConsumingService = ConsumingService + expectedUploadSettings = expectedUploadSettings ) } } "V2 initiate with minimal request settings" should { - val minimalRequestBody = Json.obj("callbackUrl" -> "https://www.example.com") - val expectedUploadSettings = UploadSettings( - uploadUrl = "https://upload-proxy.com/v1/uploads/inbound-bucket", - callbackUrl = "https://www.example.com", - minimumFileSize = None, - maximumFileSize = None, - successRedirect = None, - errorRedirect = None - ) + val minimalRequestBody = + Json.obj("callbackUrl" -> "https://www.example.com") + + val expectedUploadSettings = + settingsTemplate + .copy( + uploadUrl = "https://upload-proxy.com/v1/uploads/inbound-bucket" + ) val _ = new WithV2SuccessFixture { behave like @@ -150,29 +152,35 @@ class PrepareUploadControllerSpec extends UnitSpec with StubControllerComponents requestBody = minimalRequestBody, requestId = None, sessionId = None, - expectedUploadSettings = expectedUploadSettings, - expectedConsumingService = UserAgent + expectedUploadSettings = expectedUploadSettings ) } } "V2 initiate with all request settings" should { - val maximalRequestBody = Json.obj( - "callbackUrl" -> "https://www.example.com", - "minimumFileSize" -> 1, - "maximumFileSize" -> 1024, - "successRedirect" -> "https://www.example.com/success", - "errorRedirect" -> "https://www.example.com/error", - "consumingService" -> ConsumingService - ) - val expectedUploadSettings = UploadSettings( - uploadUrl = "https://upload-proxy.com/v1/uploads/inbound-bucket", - callbackUrl = "https://www.example.com", - minimumFileSize = Some(1), - maximumFileSize = Some(1024), - successRedirect = Some("https://www.example.com/success"), - errorRedirect = Some("https://www.example.com/error") - ) + val maximalRequestBody = + Json.obj( + "callbackUrl" -> "https://www.example.com", + "minimumFileSize" -> 1, + "maximumFileSize" -> 1024, + "successRedirect" -> "https://www.example.com/success", + "errorRedirect" -> "https://www.example.com/error", + "consumingService" -> ConsumingService + ) + + val expectedUploadSettings = + settingsTemplate + .copy( + uploadUrl = "https://upload-proxy.com/v1/uploads/inbound-bucket", + prepareUploadRequest = + requestTemplate.copy( + minimumFileSize = Some(1), + maximumFileSize = Some(1024), + successRedirect = Some("https://www.example.com/success"), + errorRedirect = Some("https://www.example.com/error"), + consumingService = Some(ConsumingService) + ) + ) val _ = new WithV2SuccessFixture { behave like @@ -182,8 +190,7 @@ class PrepareUploadControllerSpec extends UnitSpec with StubControllerComponents requestBody = maximalRequestBody, requestId = Some("a-request-id"), sessionId = Some("a-session-id"), - expectedUploadSettings = expectedUploadSettings, - expectedConsumingService = ConsumingService + expectedUploadSettings = expectedUploadSettings ) } } @@ -194,8 +201,7 @@ class PrepareUploadControllerSpec extends UnitSpec with StubControllerComponents requestBody: JsValue, requestId: Option[String], sessionId: Option[String], - expectedUploadSettings: UploadSettings, - expectedConsumingService: String): Unit = { + expectedUploadSettings: UploadSettings): Unit = { "prepare upload request" in new WithGlobalFileSizeLimitFixture { val controller = new PrepareUploadController(prepareUploadService, config, clock, stubControllerComponents()) @@ -217,7 +223,6 @@ class PrepareUploadControllerSpec extends UnitSpec with StubControllerComponents when(prepareUploadService.prepareUpload( expectedUploadSettings, - expectedConsumingService, requestId.getOrElse("n/a"), sessionId.getOrElse("n/a"), clock.instant) @@ -352,6 +357,24 @@ class PrepareUploadControllerSpec extends UnitSpec with StubControllerComponents } object PrepareUploadControllerSpec { + val UserAgent = "user-agent" val ConsumingService = "some-consuming-service" + + val requestTemplate: PrepareUploadRequest = + PrepareUploadRequest( + callbackUrl = "https://www.example.com", + minimumFileSize = None, + maximumFileSize = None, + successRedirect = None, + errorRedirect = None, + consumingService = None + ) + + val settingsTemplate: UploadSettings = + UploadSettings( + uploadUrl = s"http://upload-proxy.com", + userAgent = UserAgent, + prepareUploadRequest = requestTemplate + ) } diff --git a/test/controllers/model/PrepareUploadRequestSpec.scala b/test/controllers/model/PrepareUploadRequestSpec.scala new file mode 100644 index 0000000..e90904e --- /dev/null +++ b/test/controllers/model/PrepareUploadRequestSpec.scala @@ -0,0 +1,151 @@ +/* + * Copyright 2022 HM Revenue & Customs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package controllers.model + +import controllers.model.PrepareUploadRequestSpec._ +import play.api.libs.json.{JsSuccess, Json, Reads} +import test.UnitSpec + +class PrepareUploadRequestSpec extends UnitSpec { + + "V1 deserialisation" should { + behave like + testDeserialisation( + reads = PrepareUploadRequest.readsV1(maxFileSize), + expectedErrorRedirect = None + ) + } + + "V2 deserialisation" should { + behave like + testDeserialisation( + reads = PrepareUploadRequest.readsV2(maxFileSize), + expectedErrorRedirect = Some("https://www.example.com/error") + ) + } + + //noinspection ScalaStyle + private def testDeserialisation( + reads: Reads[PrepareUploadRequest], + expectedErrorRedirect: Option[String] + ) = { + + "deserialise from required fields" in { + val parse = + Json.parse( + """ + |{ + | "callbackUrl": "https://www.example.com/" + |} + |""".stripMargin + ).validate(reads) + + val expectedPrepareUploadRequest = + requestTemplate + + parse shouldBe JsSuccess(expectedPrepareUploadRequest) + } + + "deserialise from all fields" in { + val parse = + Json.parse( + """ + |{ + | "callbackUrl" : "https://www.example.com/", + | "minimumFileSize" : 0, + | "maximumFileSize" : 10, + | "successRedirect" : "https://www.example.com/success", + | "errorRedirect" : "https://www.example.com/error", + | "consumingService": "some-consuming-service" + |} + |""".stripMargin + ).validate(reads) + + val expectedPrepareUploadRequest = + requestTemplate + .copy( + minimumFileSize = Some(0), + maximumFileSize = Some(10), + successRedirect = Some("https://www.example.com/success"), + errorRedirect = expectedErrorRedirect, + consumingService = Some("some-consuming-service") + ) + + parse shouldBe JsSuccess(expectedPrepareUploadRequest) + } + + "reject out-of-bounds `minimumFileSize` values" in { + val parse = + Json.parse( + """ + |{ + | "callbackUrl" : "https://www.example.com/", + | "minimumFileSize": -1 + |} + |""".stripMargin + ).validate(reads) + + parse.isError shouldBe true + } + + "reject out-of-bounds `maximumFileSize` values" in { + def parse(withMaximumFileSizeValue: Long) = + Json.parse( + s""" + |{ + | "callbackUrl" : "https://www.example.com/", + | "maximumFileSize": $withMaximumFileSizeValue + |} + |""".stripMargin + ).validate(reads) + + parse(withMaximumFileSizeValue = -1).isError shouldBe true + parse(withMaximumFileSizeValue = maxFileSize + 1).isError shouldBe true + } + + "reject erroneous `minimumFileSize` & `maximumFileSize` combinations" in { + val parse = + Json.parse( + """ + |{ + | "callbackUrl" : "https://www.example.com/", + | "minimumFileSize": 100, + | "maximumFileSize": 99 + |} + |""".stripMargin + ).validate(reads) + + parse.isError shouldBe true + } + } +} + +object PrepareUploadRequestSpec { + + val maxFileSize: Long = + 100 + + val requestTemplate: PrepareUploadRequest = + PrepareUploadRequest( + callbackUrl = "https://www.example.com/", + minimumFileSize = None, + maximumFileSize = None, + successRedirect = None, + errorRedirect = None, + consumingService = None + ) +} diff --git a/test/controllers/model/PrepareUploadRequestV2Spec.scala b/test/controllers/model/PrepareUploadRequestV2Spec.scala deleted file mode 100644 index f21e6c8..0000000 --- a/test/controllers/model/PrepareUploadRequestV2Spec.scala +++ /dev/null @@ -1,165 +0,0 @@ -/* - * Copyright 2022 HM Revenue & Customs - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package controllers.model - -import org.scalatest.EitherValues -import play.api.libs.json.Json -import test.UnitSpec - - -class PrepareUploadRequestV2Spec extends UnitSpec with EitherValues { - - import PrepareUploadRequestV2Spec._ - - "A v2 upload request" when { - "specifying an upload success redirect URL" should { - "parse as a valid request containing a success redirect URL" in { - val request = s"""|{"callbackUrl":"$CallbackUrl", - | "successRedirect":"$SuccessRedirectUrl"}""".stripMargin - val parseResult = Json.parse(request).validate(PrepareUploadRequestV2.reads(MaxFileSize)) - - parseResult.asEither.map(_.successRedirect).right.value should contain (SuccessRedirectUrl) - } - - "generate upload settings that contain the success redirect URL" in { - val uploadSettings = aV2RequestWithSuccessRedirectUrlOf(Some(SuccessRedirectUrl)).toUploadSettings(UploadUrl) - - uploadSettings.successRedirect should contain (SuccessRedirectUrl) - } - } - - "omitting an upload success redirect URL" should { - "parse as a valid request without a success redirect URL" in { - val request = s"""{"callbackUrl":"$CallbackUrl"}""" - val parseResult = Json.parse(request).validate(PrepareUploadRequestV2.reads(MaxFileSize)) - - parseResult.asEither.map(_.successRedirect).right.value shouldBe empty - } - - "generate upload settings without a success redirect URL" in { - val uploadSettings = aV2RequestWithSuccessRedirectUrlOf(None).toUploadSettings(UploadUrl) - - uploadSettings.successRedirect shouldBe empty - } - } - - "specifying an error redirect URL" should { - "parse as a valid request containing an error redirect URL" in { - val request = s"""|{"callbackUrl":"$CallbackUrl", - | "errorRedirect":"$ErrorRedirectUrl"}""".stripMargin - val parseResult = Json.parse(request).validate(PrepareUploadRequestV2.reads(MaxFileSize)) - - parseResult.asEither.map(_.errorRedirect).right.value should contain (ErrorRedirectUrl) - } - - "generate upload settings that contain the error redirect URL" in { - val uploadSettings = aV2RequestWithErrorRedirectUrlOf(Some(ErrorRedirectUrl)).toUploadSettings(UploadUrl) - - uploadSettings.errorRedirect should contain (ErrorRedirectUrl) - } - } - - "omitting an error redirect URL" should { - "parse as a valid request without an error redirect URL" in { - val request = s"""{"callbackUrl":"$CallbackUrl"}""" - val parseResult = Json.parse(request).validate(PrepareUploadRequestV2.reads(MaxFileSize)) - - parseResult.asEither.map(_.errorRedirect).right.value shouldBe empty - } - - "generate upload settings without an error redirect URL" in { - val uploadSettings = aV2RequestWithErrorRedirectUrlOf(None).toUploadSettings(UploadUrl) - - uploadSettings.errorRedirect shouldBe empty - } - } - - "specifying a minimum file size" should { - "be rejected when negative" in { - val request = s"""{"callbackUrl":"$CallbackUrl", "minimumFileSize": -1}""" - val parseResult = Json.parse(request).validate(PrepareUploadRequestV2.reads(MaxFileSize)) - - parseResult.isError shouldBe true - } - - "be accepted when zero" in { - val request = s"""{"callbackUrl":"$CallbackUrl", "minimumFileSize": 0}""" - val parseResult = Json.parse(request).validate(PrepareUploadRequestV2.reads(MaxFileSize)) - - parseResult.asEither.map(_.minimumFileSize).right.value should contain (0) - } - } - - "specifying a maximum file size" should { - "be accepted when equal to the global maximum" in { - val request = s"""{"callbackUrl":"$CallbackUrl", "maximumFileSize": $MaxFileSize}""" - val parseResult = Json.parse(request).validate(PrepareUploadRequestV2.reads(MaxFileSize)) - - parseResult.asEither.map(_.maximumFileSize).right.value should contain (MaxFileSize) - } - - "be rejected when greater than the global maximum" in { - val request = s"""{"callbackUrl":"$CallbackUrl", "maximumFileSize": ${MaxFileSize + 1}}""" - val parseResult = Json.parse(request).validate(PrepareUploadRequestV2.reads(MaxFileSize)) - - parseResult.isError shouldBe true - } - - "be rejected when less than the specified minimumFileSize" in { - val request = s"""{"callbackUrl":"$CallbackUrl", "minimumFileSize": $MaxFileSize, "maximumFileSize": ${MaxFileSize - 1}}""" - val parseResult = Json.parse(request).validate(PrepareUploadRequestV2.reads(MaxFileSize)) - - parseResult.isError shouldBe true - } - } - - "specifying an explicit consuming service" should { - "deserialise accordingly" in { - val request1 = s"""{"callbackUrl":"$CallbackUrl", "consumingService": "$ConsumingService"}""" - val prepareUploadRequest1 = Json.parse(request1).validate(PrepareUploadRequestV2.reads(MaxFileSize)).get - prepareUploadRequest1.consumingService shouldBe Some(ConsumingService) - - val request2 = s"""{"callbackUrl":"$CallbackUrl"}""" - val prepareUploadRequest2 = Json.parse(request2).validate(PrepareUploadRequestV2.reads(MaxFileSize)).get - prepareUploadRequest2.consumingService shouldBe None - } - } - } -} - -private object PrepareUploadRequestV2Spec { - val UploadUrl = "https://xxxx/upscan-upload-proxy/bucketName" - val CallbackUrl = "https://myservice.com/callback" - val SuccessRedirectUrl = "https://myservice.com/nextPage" - val ErrorRedirectUrl = "https://myservice.com/errorPage" - val MaxFileSize = 512 - val ConsumingService = "some-consuming-service" - - private val template = PrepareUploadRequestV2( - callbackUrl = CallbackUrl, - successRedirect = None, - errorRedirect = None, - minimumFileSize = None, - maximumFileSize = None, - consumingService = None) - - def aV2RequestWithSuccessRedirectUrlOf(url: Option[String]): PrepareUploadRequestV2 = - template.copy(successRedirect = url) - - def aV2RequestWithErrorRedirectUrlOf(url: Option[String]): PrepareUploadRequestV2 = - template.copy(errorRedirect = url) -} \ No newline at end of file diff --git a/test/model/PrepareUploadServiceSpec.scala b/test/model/PrepareUploadServiceSpec.scala index 5ad4a8a..368c887 100644 --- a/test/model/PrepareUploadServiceSpec.scala +++ b/test/model/PrepareUploadServiceSpec.scala @@ -18,11 +18,12 @@ package model import java.time import java.time.Instant - import com.codahale.metrics.MetricRegistry import com.kenshoo.play.metrics.Metrics import config.ServiceConfiguration import connectors.model.{UploadFormGenerator, UploadParameters} +import controllers.model.PrepareUploadRequest +import model.PrepareUploadServiceSpec.{requestTemplate, settingsTemplate} import org.scalatest.GivenWhenThen import services.PrepareUploadService import services.model.UploadSettings @@ -83,31 +84,28 @@ class PrepareUploadServiceSpec extends UnitSpec with GivenWhenThen { Given("there are valid upload settings") - val uploadUrl = s"http://upload-proxy.com" - val callbackUrl = "http://www.callback.com" - - val uploadSettings = UploadSettings( - uploadUrl = uploadUrl, - callbackUrl = callbackUrl, - minimumFileSize = None, - maximumFileSize = None, - successRedirect = None, - errorRedirect = None - ) + val settings = + settingsTemplate When("we setup the upload") - val result = service(metrics) - .prepareUpload(uploadSettings, "PrepareUploadServiceSpec", "some-request-id", "some-session-id", receivedAt) + val result = + service(metrics) + .prepareUpload( + settings, + "some-request-id", + "some-session-id", + receivedAt + ) Then("proper upload request form definition should be returned") - result.uploadRequest.href shouldBe uploadUrl + result.uploadRequest.href shouldBe settings.uploadUrl result.uploadRequest.fields shouldBe Map( "bucket" -> serviceConfiguration.inboundBucketName, "key" -> result.reference.value, - "x-amz-meta-callback-url" -> callbackUrl, - "x-amz-meta-consuming-service" -> "PrepareUploadServiceSpec", + "x-amz-meta-callback-url" -> settings.prepareUploadRequest.callbackUrl, + "x-amz-meta-consuming-service" -> settings.consumingService, "x-amz-meta-session-id" -> "some-session-id", "x-amz-meta-request-id" -> "some-request-id", "minSize" -> "0", @@ -126,22 +124,20 @@ class PrepareUploadServiceSpec extends UnitSpec with GivenWhenThen { Given("there are valid upload settings with size limits") - val uploadUrl = s"http://upload-proxy.com" - val callbackUrl = "http://www.callback.com" - - val uploadSettings = UploadSettings( - uploadUrl = uploadUrl, - callbackUrl = callbackUrl, - minimumFileSize = Some(100), - maximumFileSize = Some(200), - successRedirect = None, - errorRedirect = None - ) + val settings = + settingsTemplate + .copy( + prepareUploadRequest = + requestTemplate.copy( + minimumFileSize = Some(100), + maximumFileSize = Some(200), + ) + ) When("we setup the upload") val result = service(metrics) - .prepareUpload(uploadSettings, "PrepareUploadServiceSpec", "some-request-id", "some-session-id", receivedAt) + .prepareUpload(settings, "some-request-id", "some-session-id", receivedAt) Then("upload request should contain requested min/max size") @@ -155,23 +151,21 @@ class PrepareUploadServiceSpec extends UnitSpec with GivenWhenThen { val metrics = metricsStub() - val uploadUrl = s"http://upload-proxy.com" - val callbackUrl = "http://www.callback.com" - - val uploadSettings = UploadSettings( - uploadUrl = uploadUrl, - callbackUrl = callbackUrl, - minimumFileSize = Some(-1), - maximumFileSize = Some(1024), - successRedirect = None, - errorRedirect = None - ) + val settings = + settingsTemplate + .copy( + prepareUploadRequest = + requestTemplate.copy( + minimumFileSize = Some(-1), + maximumFileSize = Some(1024), + ) + ) When("we setup the upload") Then("an exception should be thrown") val thrown = the[IllegalArgumentException] thrownBy service(metrics) - .prepareUpload(uploadSettings, "PrepareUploadServiceSpec", "some-request-id", "some-session-id", receivedAt) + .prepareUpload(settings, "some-request-id", "some-session-id", receivedAt) thrown.getMessage should include("Minimum file size is less than 0") metrics.defaultRegistry.counter("uploadInitiated").getCount shouldBe 0 @@ -184,23 +178,21 @@ class PrepareUploadServiceSpec extends UnitSpec with GivenWhenThen { val metrics = metricsStub() - val uploadUrl = s"http://upload-proxy.com" - val callbackUrl = "http://www.callback.com" - - val uploadSettings = UploadSettings( - uploadUrl = uploadUrl, - callbackUrl = callbackUrl, - minimumFileSize = Some(0), - maximumFileSize = Some(1025), - successRedirect = None, - errorRedirect = None - ) + val settings = + settingsTemplate + .copy( + prepareUploadRequest = + requestTemplate.copy( + minimumFileSize = Some(0), + maximumFileSize = Some(1025), + ) + ) When("we setup the upload") Then("an exception should be thrown") val thrown = the[IllegalArgumentException] thrownBy service(metrics) - .prepareUpload(uploadSettings, "PrepareUploadServiceSpec", "some-request-id", "some-session-id", receivedAt) + .prepareUpload(settings, "some-request-id", "some-session-id", receivedAt) thrown.getMessage should include("Maximum file size is greater than global maximum file size") metrics.defaultRegistry.counter("uploadInitiated").getCount shouldBe 0 @@ -212,23 +204,21 @@ class PrepareUploadServiceSpec extends UnitSpec with GivenWhenThen { val metrics = metricsStub() - val uploadUrl = s"http://upload-proxy.com" - val callbackUrl = "http://www.callback.com" - - val uploadSettings = UploadSettings( - uploadUrl = uploadUrl, - callbackUrl = callbackUrl, - minimumFileSize = Some(1024), - maximumFileSize = Some(0), - successRedirect = None, - errorRedirect = None - ) + val settings = + settingsTemplate + .copy( + prepareUploadRequest = + requestTemplate.copy( + minimumFileSize = Some(1024), + maximumFileSize = Some(0), + ) + ) When("we setup the upload") Then("an exception should be thrown") val thrown = the[IllegalArgumentException] thrownBy service(metrics) - .prepareUpload(uploadSettings, "PrepareUploadServiceSpec", "some-request-id", "some-session-id", receivedAt) + .prepareUpload(settings, "some-request-id", "some-session-id", receivedAt) thrown.getMessage should include("Minimum file size is greater than maximum file size") metrics.defaultRegistry.counter("uploadInitiated").getCount shouldBe 0 @@ -240,37 +230,34 @@ class PrepareUploadServiceSpec extends UnitSpec with GivenWhenThen { Given("there are valid upload settings") - val uploadUrl = s"http://upload-proxy.com" - val callbackUrl = "http://www.callback.com" - - val uploadSettings = UploadSettings( - uploadUrl = uploadUrl, - callbackUrl = callbackUrl, - minimumFileSize = None, - maximumFileSize = None, - successRedirect = Some("https://new.service/page1"), - errorRedirect = None - ) + val settings = + settingsTemplate + .copy( + prepareUploadRequest = + requestTemplate.copy( + successRedirect = Some("https://new.service/page1") + ) + ) When("we setup the upload") val result = service(metrics) - .prepareUpload(uploadSettings, "PrepareUploadServiceSpec", "some-request-id", "some-session-id", receivedAt) + .prepareUpload(settings, "some-request-id", "some-session-id", receivedAt) Then("proper upload request form definition should be returned") - result.uploadRequest.href shouldBe uploadUrl + result.uploadRequest.href shouldBe settings.uploadUrl result.uploadRequest.fields shouldBe Map( "bucket" -> serviceConfiguration.inboundBucketName, "key" -> result.reference.value, - "x-amz-meta-callback-url" -> callbackUrl, - "x-amz-meta-consuming-service" -> "PrepareUploadServiceSpec", + "x-amz-meta-callback-url" -> settings.prepareUploadRequest.callbackUrl, + "x-amz-meta-consuming-service" -> settings.consumingService, "x-amz-meta-session-id" -> "some-session-id", "x-amz-meta-request-id" -> "some-request-id", "minSize" -> "0", "maxSize" -> "1024", "x-amz-meta-upscan-initiate-received" -> receivedAt.toString, - "success_redirect_url" -> "https://new.service/page1" + "success_redirect_url" -> settings.prepareUploadRequest.successRedirect.get ) And("uploadInitiated counter has been incremented") @@ -284,37 +271,34 @@ class PrepareUploadServiceSpec extends UnitSpec with GivenWhenThen { Given("there are valid upload settings") - val uploadUrl = s"http://upload-proxy.com" - val callbackUrl = "http://www.callback.com" - - val uploadSettings = UploadSettings( - uploadUrl = uploadUrl, - callbackUrl = callbackUrl, - minimumFileSize = None, - maximumFileSize = None, - successRedirect = None, - errorRedirect = Some("https://new.service/error") - ) + val settings = + settingsTemplate + .copy( + prepareUploadRequest = + requestTemplate.copy( + errorRedirect = Some("https://new.service/error") + ) + ) When("we setup the upload") val result = service(metrics) - .prepareUpload(uploadSettings, "PrepareUploadServiceSpec", "some-request-id", "some-session-id", receivedAt) + .prepareUpload(settings, "some-request-id", "some-session-id", receivedAt) Then("proper upload request form definition should be returned") - result.uploadRequest.href shouldBe uploadUrl + result.uploadRequest.href shouldBe settings.uploadUrl result.uploadRequest.fields shouldBe Map( "bucket" -> serviceConfiguration.inboundBucketName, "key" -> result.reference.value, - "x-amz-meta-callback-url" -> callbackUrl, - "x-amz-meta-consuming-service" -> "PrepareUploadServiceSpec", + "x-amz-meta-callback-url" -> settings.prepareUploadRequest.callbackUrl, + "x-amz-meta-consuming-service" -> settings.consumingService, "x-amz-meta-session-id" -> "some-session-id", "x-amz-meta-request-id" -> "some-request-id", "minSize" -> "0", "maxSize" -> "1024", "x-amz-meta-upscan-initiate-received" -> receivedAt.toString, - "error_redirect_url" -> "https://new.service/error" + "error_redirect_url" -> settings.prepareUploadRequest.errorRedirect.get ) And("uploadInitiated counter has been incremented") @@ -323,5 +307,24 @@ class PrepareUploadServiceSpec extends UnitSpec with GivenWhenThen { } } +} +object PrepareUploadServiceSpec { + + val requestTemplate: PrepareUploadRequest = + PrepareUploadRequest( + callbackUrl = "http://www.callback.com", + minimumFileSize = None, + maximumFileSize = None, + successRedirect = None, + errorRedirect = None, + consumingService = None + ) + + val settingsTemplate: UploadSettings = + UploadSettings( + uploadUrl = s"http://upload-proxy.com", + userAgent = "PrepareUploadServiceSpec", + prepareUploadRequest = requestTemplate + ) } From b8e1ada9e760bb56a6c2e2ae7660f96e589b3b2b Mon Sep 17 00:00:00 2001 From: Tom Wadeson <3607811+tomwadeson@users.noreply.github.com> Date: Tue, 2 Aug 2022 13:52:37 +0100 Subject: [PATCH 74/96] BDOG-2016 Remove unused `expectedContentType` --- .../PrepareUploadControllerISpec.scala | 54 ------------------- 1 file changed, 54 deletions(-) diff --git a/it/controllers/PrepareUploadControllerISpec.scala b/it/controllers/PrepareUploadControllerISpec.scala index 26b5c9d..6ae0a19 100644 --- a/it/controllers/PrepareUploadControllerISpec.scala +++ b/it/controllers/PrepareUploadControllerISpec.scala @@ -251,60 +251,6 @@ class PrepareUploadControllerISpec extends AnyWordSpecLike with should.Matchers fields should contain key "policy" } } - - "PrepareUploadController prepareUploadV1 with expectedContentType" in { - val postBodyJson = Json.parse("""|{ - | "callbackUrl": "https://some-url/callback", - | "minimumFileSize" : 0, - | "maximumFileSize" : 1024, - | "expectedContentType": "application/xml", - | "successRedirect": "https://some-url/success" - |}""".stripMargin) - - Given("a request containing a User-Agent header") - val headers = FakeHeaders(Seq((USER_AGENT, SomeConsumingService))) - val initiateRequest = FakeRequest(POST, uri = "/upscan/initiate", headers, postBodyJson) - - When("a request is posted to the /initiate endpoint") - val initiateResponse = route(app, initiateRequest).get - - Then("the response should indicate success") - status(initiateResponse) shouldBe OK - - And("the response should not contain Content-Type") - val responseJson = contentAsJson(initiateResponse) - val fields = (responseJson \ "uploadRequest" \ "fields").as[Map[String, String]] - fields.get("Content-Type") shouldBe empty - } - - "PrepareUploadController prepareUploadV2 with expectedContentType" in { - val postBodyJson = Json.parse(""" - |{ - | "callbackUrl": "https://some-url/callback", - | "successRedirect": "https://some-url/success", - | "errorRedirect": "https://some-url/error", - | "minimumFileSize" : 0, - | "expectedContentType": "application/xml", - | "maximumFileSize" : 1024 - |} - """.stripMargin) - - Given("a request containing a User-Agent header") - val headers = FakeHeaders(Seq((USER_AGENT, SomeConsumingService))) - val initiateRequest = FakeRequest(POST, uri = "/upscan/v2/initiate", headers, postBodyJson) - - When("a request is posted to the /initiate endpoint") - val initiateResponse = route(app, initiateRequest).get - - Then("the response should indicate success") - status(initiateResponse) shouldBe OK - - And("the response should contain the requested upload fields") - val responseJson = contentAsJson(initiateResponse) - val fields = (responseJson \ "uploadRequest" \ "fields").as[Map[String, String]] - fields.get("Content-Type") shouldBe empty - } - } private object PrepareUploadControllerISpec { From b584cf510f4462b746edfd1a7516d998a7412108 Mon Sep 17 00:00:00 2001 From: Tom Wadeson <3607811+tomwadeson@users.noreply.github.com> Date: Tue, 16 Aug 2022 09:42:48 +0100 Subject: [PATCH 75/96] BDOG-2016 Update docs with more info on consuming services --- README.md | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index a17b981..491f0cc 100644 --- a/README.md +++ b/README.md @@ -17,10 +17,11 @@ This service is not for transfer of files from one HMRC service to another. Plea 1. [Introduction](#introduction) 2. [File upload workflow](#workflow) 3. [Service usage](#service) - a. [Requesting a URL to upload to](#service__request) - b. [The file upload](#service__upload) - c. [File upload outcome](#service__uoutcome) - d. [File processing outcome](#service__poutcome) + a. [Consuming services](#service__consuming-services) + b. [Requesting a URL to upload to](#service__request) + c. [The file upload](#service__upload) + d. [File upload outcome](#service__uoutcome) + e. [File processing outcome](#service__poutcome) i. [Success](#service__poutcome__success) ii. [Failure](#service__poutcome__failure) 4. [Error handling](#error) @@ -68,15 +69,22 @@ Please view the [Upscan Service & Flow Overview in Confluence](https://confluenc ## Service usage +### Consuming Services + +A _consuming service_ is the intermediary between the end-user and Upscan, responsible for initiating upload requests and configuring the parameters of the upload itself (minimum and maximum file sizes, permitted MIME types and redirect URLs, etc.) + +Consuming services must identify themselves to `upscan-initiate` in one of two ways: + + 1. Via the `User-Agent` header, which must always be provided + 2. By including a `"consumingService"` field in the request body, which overrides the value provided in the `User-Agent` header and may be useful if your use-case involves proxies or requires _logical_ consuming services that differ from the actual service that initiates the request + +Consuming services may configure permitted MIME-types in [upscan-app-config](https://github.com/hmrc/upscan-app-config) (a default set is used in the case that no consuming service-specific configuration is provided.) + ### Requesting a URL to upload to The consuming service makes a POST request to `/upscan/initiate` or `upscan/v2/initiate`. -This request must contain a `User-Agent` header that can be used to identify the service, but an allow list of authorised services is no longer cross-checked. The service must also provide a callbackUrl for asynchronous notification of the outcome of an upload. The callback will be made from inside the MDTP environment. Hence, the callback URL should comprise the MDTP internal callback address and not the public domain address. -**Note:** Although a `User-Agent` header is always required, consuming services may wish to explicitly identify themselves by including a `"consumingService"` field in the request body. -This functionality may be useful to you if the actual service that initiates the request is different to your preferred logical service. - **Note:** `callbackUrl` must use the `https` protocol. (Although this rule is relaxed when testing locally with [upscan-stub](https://github.com/hmrc/upscan-stub) rather than [upscan-initiate](https://github.com/hmrc/upscan-initiate). In this stubbed scenario a `callbackUrl` referring to localhost may still specify `http` as the protocol.) From 3a5e6b573e61c54d0de97413762dcd3b600575d0 Mon Sep 17 00:00:00 2001 From: Tom Wadeson <3607811+tomwadeson@users.noreply.github.com> Date: Fri, 11 Nov 2022 09:03:10 +0000 Subject: [PATCH 76/96] BDOG-2224 Provide guidance on additional form validation --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 491f0cc..73d998c 100644 --- a/README.md +++ b/README.md @@ -285,11 +285,13 @@ In order to upload the file, the following form is sent as the body of a POST re
... all the fields returned in "fields" map in the response above ... - <- form field representing the file to upload + <- form field representing the file to upload
``` +**Note: You should consider implementing validation on the frontend to prevent submitting the form if a file has not been selected (such as marking the file input `required`.)** + - You must make this request client side. Making this request server side defeats the objective of Upscan, which is to virus scan files before they are allowed on MDTP. - You must use multipart encoding (`multipart/form-data`) NOT `application/x-www-form-urlencoded`. If you use` application/x-www-form-urlencoded`, AWS will return a response from which this error is not clear. - The 'file' field must be the last field in the submitted form. From 7f81dc6ce4f2a01db7cbf128d13e3f6f401485b3 Mon Sep 17 00:00:00 2001 From: peteanning <22272449+peteanning@users.noreply.github.com> Date: Thu, 4 Jan 2024 12:59:07 +0000 Subject: [PATCH 77/96] BDOG-2800: upgrade to Play 3.0 --- app/UpscanInitiateModule.scala | 2 +- app/config/ServiceConfiguration.scala | 2 +- app/connectors/model/AwsCredentials.scala | 2 +- app/connectors/model/ContentLengthRange.scala | 2 +- app/connectors/model/UploadFormGenerator.scala | 2 +- app/connectors/model/UploadParameters.scala | 2 +- app/connectors/s3/PolicySigner.scala | 2 +- app/connectors/s3/S3Module.scala | 2 +- app/connectors/s3/S3UploadFormGenerator.scala | 2 +- .../s3/S3UploadFormGeneratorProvider.scala | 2 +- app/controllers/PrepareUploadController.scala | 6 +++--- app/controllers/model/PrepareUploadRequest.scala | 2 +- .../model/PreparedUploadResponse.scala | 2 +- app/controllers/model/Reference.scala | 2 +- app/controllers/model/UploadFormTemplate.scala | 2 +- app/services/PrepareUploadService.scala | 4 ++-- app/services/model/UploadSettings.scala | 2 +- app/utils/UserAgentFilter.scala | 2 +- build.sbt | 14 +++++++++----- conf/application.conf | 2 +- conf/prod.routes | 1 - project/AppDependencies.scala | 13 ++++++------- project/build.properties | 2 +- project/plugins.sbt | 13 +++++++------ test/connectors/s3/PolicySignerSpec.scala | 2 +- .../s3/S3UploadFormGeneratorSpec.scala | 4 ++-- .../PrepareUploadControllerSpec.scala | 16 +++++++--------- .../model/PrepareUploadRequestSpec.scala | 2 +- test/model/PrepareUploadServiceSpec.scala | 5 ++--- test/test/UnitSpec.scala | 2 +- test/utils/UserAgentFilterSpec.scala | 6 +++--- 31 files changed, 62 insertions(+), 62 deletions(-) diff --git a/app/UpscanInitiateModule.scala b/app/UpscanInitiateModule.scala index 84bab1d..3452628 100644 --- a/app/UpscanInitiateModule.scala +++ b/app/UpscanInitiateModule.scala @@ -1,5 +1,5 @@ /* - * Copyright 2022 HM Revenue & Customs + * Copyright 2024 HM Revenue & Customs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/app/config/ServiceConfiguration.scala b/app/config/ServiceConfiguration.scala index c7faba5..61d4c2d 100644 --- a/app/config/ServiceConfiguration.scala +++ b/app/config/ServiceConfiguration.scala @@ -1,5 +1,5 @@ /* - * Copyright 2022 HM Revenue & Customs + * Copyright 2024 HM Revenue & Customs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/app/connectors/model/AwsCredentials.scala b/app/connectors/model/AwsCredentials.scala index 4ea3807..b34895f 100644 --- a/app/connectors/model/AwsCredentials.scala +++ b/app/connectors/model/AwsCredentials.scala @@ -1,5 +1,5 @@ /* - * Copyright 2022 HM Revenue & Customs + * Copyright 2024 HM Revenue & Customs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/app/connectors/model/ContentLengthRange.scala b/app/connectors/model/ContentLengthRange.scala index 571fb15..126160d 100644 --- a/app/connectors/model/ContentLengthRange.scala +++ b/app/connectors/model/ContentLengthRange.scala @@ -1,5 +1,5 @@ /* - * Copyright 2022 HM Revenue & Customs + * Copyright 2024 HM Revenue & Customs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/app/connectors/model/UploadFormGenerator.scala b/app/connectors/model/UploadFormGenerator.scala index 0104f17..42c2a6c 100644 --- a/app/connectors/model/UploadFormGenerator.scala +++ b/app/connectors/model/UploadFormGenerator.scala @@ -1,5 +1,5 @@ /* - * Copyright 2022 HM Revenue & Customs + * Copyright 2024 HM Revenue & Customs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/app/connectors/model/UploadParameters.scala b/app/connectors/model/UploadParameters.scala index d6ba352..acac60e 100644 --- a/app/connectors/model/UploadParameters.scala +++ b/app/connectors/model/UploadParameters.scala @@ -1,5 +1,5 @@ /* - * Copyright 2022 HM Revenue & Customs + * Copyright 2024 HM Revenue & Customs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/app/connectors/s3/PolicySigner.scala b/app/connectors/s3/PolicySigner.scala index c3d3cf9..9d3911f 100644 --- a/app/connectors/s3/PolicySigner.scala +++ b/app/connectors/s3/PolicySigner.scala @@ -1,5 +1,5 @@ /* - * Copyright 2022 HM Revenue & Customs + * Copyright 2024 HM Revenue & Customs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/app/connectors/s3/S3Module.scala b/app/connectors/s3/S3Module.scala index 4f37776..3a4a9a1 100644 --- a/app/connectors/s3/S3Module.scala +++ b/app/connectors/s3/S3Module.scala @@ -1,5 +1,5 @@ /* - * Copyright 2022 HM Revenue & Customs + * Copyright 2024 HM Revenue & Customs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/app/connectors/s3/S3UploadFormGenerator.scala b/app/connectors/s3/S3UploadFormGenerator.scala index 7b6b4e5..8197186 100644 --- a/app/connectors/s3/S3UploadFormGenerator.scala +++ b/app/connectors/s3/S3UploadFormGenerator.scala @@ -1,5 +1,5 @@ /* - * Copyright 2022 HM Revenue & Customs + * Copyright 2024 HM Revenue & Customs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/app/connectors/s3/S3UploadFormGeneratorProvider.scala b/app/connectors/s3/S3UploadFormGeneratorProvider.scala index 17f4c86..abaeb47 100644 --- a/app/connectors/s3/S3UploadFormGeneratorProvider.scala +++ b/app/connectors/s3/S3UploadFormGeneratorProvider.scala @@ -1,5 +1,5 @@ /* - * Copyright 2022 HM Revenue & Customs + * Copyright 2024 HM Revenue & Customs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/app/controllers/PrepareUploadController.scala b/app/controllers/PrepareUploadController.scala index f18124c..091a057 100644 --- a/app/controllers/PrepareUploadController.scala +++ b/app/controllers/PrepareUploadController.scala @@ -1,5 +1,5 @@ /* - * Copyright 2022 HM Revenue & Customs + * Copyright 2024 HM Revenue & Customs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -49,7 +49,7 @@ class PrepareUploadController @Inject()( /** * V1 of the API supports direct upload to an S3 bucket and *does not support* error redirects in the event of failure */ - def prepareUploadV1(): Action[JsValue] = { + def prepareUploadV1: Action[JsValue] = { val uploadUrl = s"https://${configuration.inboundBucketName}.s3.amazonaws.com" prepareUpload(uploadUrl)(prepareUploadRequestReadsV1) } @@ -57,7 +57,7 @@ class PrepareUploadController @Inject()( /** * V2 of the API supports upload to an S3 bucket via a proxy that additionally supports error redirects in the event of failure */ - def prepareUploadV2(): Action[JsValue] = { + def prepareUploadV2: Action[JsValue] = { val uploadUrl = s"${configuration.uploadProxyUrl}/v1/uploads/${configuration.inboundBucketName}" prepareUpload(uploadUrl)(prepareUploadRequestReadsV2) } diff --git a/app/controllers/model/PrepareUploadRequest.scala b/app/controllers/model/PrepareUploadRequest.scala index ee288e7..c828edd 100644 --- a/app/controllers/model/PrepareUploadRequest.scala +++ b/app/controllers/model/PrepareUploadRequest.scala @@ -1,5 +1,5 @@ /* - * Copyright 2022 HM Revenue & Customs + * Copyright 2024 HM Revenue & Customs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/app/controllers/model/PreparedUploadResponse.scala b/app/controllers/model/PreparedUploadResponse.scala index c576304..d754d89 100644 --- a/app/controllers/model/PreparedUploadResponse.scala +++ b/app/controllers/model/PreparedUploadResponse.scala @@ -1,5 +1,5 @@ /* - * Copyright 2022 HM Revenue & Customs + * Copyright 2024 HM Revenue & Customs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/app/controllers/model/Reference.scala b/app/controllers/model/Reference.scala index f8476a3..9ef577f 100644 --- a/app/controllers/model/Reference.scala +++ b/app/controllers/model/Reference.scala @@ -1,5 +1,5 @@ /* - * Copyright 2022 HM Revenue & Customs + * Copyright 2024 HM Revenue & Customs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/app/controllers/model/UploadFormTemplate.scala b/app/controllers/model/UploadFormTemplate.scala index 00b9ee2..fe8f453 100644 --- a/app/controllers/model/UploadFormTemplate.scala +++ b/app/controllers/model/UploadFormTemplate.scala @@ -1,5 +1,5 @@ /* - * Copyright 2022 HM Revenue & Customs + * Copyright 2024 HM Revenue & Customs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/app/services/PrepareUploadService.scala b/app/services/PrepareUploadService.scala index b4299d8..65835a8 100644 --- a/app/services/PrepareUploadService.scala +++ b/app/services/PrepareUploadService.scala @@ -1,5 +1,5 @@ /* - * Copyright 2022 HM Revenue & Customs + * Copyright 2024 HM Revenue & Customs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,7 +17,6 @@ package services import java.time.Instant -import com.kenshoo.play.metrics.Metrics import config.ServiceConfiguration import connectors.model.{ContentLengthRange, UploadFormGenerator, UploadParameters} import controllers.model.{PreparedUploadResponse, Reference, UploadFormTemplate} @@ -26,6 +25,7 @@ import javax.inject.{Inject, Singleton} import org.slf4j.MDC import play.api.Logging import services.model.UploadSettings +import uk.gov.hmrc.play.bootstrap.metrics.Metrics @Singleton class PrepareUploadService @Inject()( diff --git a/app/services/model/UploadSettings.scala b/app/services/model/UploadSettings.scala index ba237ec..597b616 100644 --- a/app/services/model/UploadSettings.scala +++ b/app/services/model/UploadSettings.scala @@ -1,5 +1,5 @@ /* - * Copyright 2022 HM Revenue & Customs + * Copyright 2024 HM Revenue & Customs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/app/utils/UserAgentFilter.scala b/app/utils/UserAgentFilter.scala index 6ed2a4f..b6376a1 100644 --- a/app/utils/UserAgentFilter.scala +++ b/app/utils/UserAgentFilter.scala @@ -1,5 +1,5 @@ /* - * Copyright 2022 HM Revenue & Customs + * Copyright 2024 HM Revenue & Customs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/build.sbt b/build.sbt index 914cc16..262b03c 100644 --- a/build.sbt +++ b/build.sbt @@ -8,6 +8,9 @@ import uk.gov.hmrc.sbtdistributables.SbtDistributablesPlugin val appName = "upscan-initiate" +ThisBuild / majorVersion := 0 +ThisBuild / scalaVersion := "2.13.12" + lazy val scoverageSettings = Seq( // Semicolon-separated list of regexs matching classes to exclude ScoverageKeys.coverageExcludedPackages := ";Reverse.*;.*AuthService.*;models/.data/..*;view.*", @@ -23,13 +26,14 @@ lazy val microservice = Project(appName, file(".")) .enablePlugins(play.sbt.PlayScala, SbtAutoBuildPlugin, SbtDistributablesPlugin) .disablePlugins(JUnitXmlReportPlugin) //Required to prevent https://github.com/scalatest/scalatest/issues/1427 .settings(scoverageSettings: _*) - .settings(majorVersion := 0) - .settings(SbtDistributablesPlugin.publishingSettings: _*) + .settings(scalacOptions ++= Seq( + "-Wconf:cat=unused-imports&src=.*routes.*:s" //silence import warnings in routes generated by comments + , "-Wconf:cat=unused&src=.*routes.*:s" //silence private val defaultPrefix in class Routes is never used + ) + ) .settings(playDefaultPort := 9571) .settings(libraryDependencies ++= AppDependencies()) .settings(resolvers += Resolver.jcenterRepo) - .settings(scalacOptions += "-target:jvm-1.8") - .settings(scalaVersion := "2.12.12") .settings(Test / parallelExecution := false) .configs(IntegrationTest) - .settings(integrationTestSettings(): _*) \ No newline at end of file + .settings(integrationTestSettings(): _*) diff --git a/conf/application.conf b/conf/application.conf index a01a415..a1758de 100644 --- a/conf/application.conf +++ b/conf/application.conf @@ -1,4 +1,4 @@ -# Copyright 2022 HM Revenue & Customs +# Copyright 2024 HM Revenue & Customs # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/conf/prod.routes b/conf/prod.routes index 3337678..1d6e922 100644 --- a/conf/prod.routes +++ b/conf/prod.routes @@ -1,4 +1,3 @@ # Add all the application routes to the app.routes file -> /upscan app.Routes -> / health.Routes -GET /admin/metrics com.kenshoo.play.metrics.MetricsController.metrics \ No newline at end of file diff --git a/project/AppDependencies.scala b/project/AppDependencies.scala index b0c29d9..bec9ce8 100644 --- a/project/AppDependencies.scala +++ b/project/AppDependencies.scala @@ -1,18 +1,17 @@ import sbt._ object AppDependencies { - private val bootstrapVersion = "5.12.0" + private val bootstrapVersion = "8.4.0" private val compile = Seq( - "uk.gov.hmrc" %% "bootstrap-backend-play-28" % bootstrapVersion, - "com.amazonaws" % "aws-java-sdk-s3" % "1.11.921", - "org.apache.commons" % "commons-lang3" % "3.11" + "uk.gov.hmrc" %% "bootstrap-backend-play-30" % bootstrapVersion, + "com.amazonaws" % "aws-java-sdk-s3" % "1.12.606", + "org.apache.commons" % "commons-lang3" % "3.12.0" ) private val test = Seq( - "uk.gov.hmrc" %% "bootstrap-test-play-28" % bootstrapVersion % s"$Test,$IntegrationTest", - "com.vladsch.flexmark" % "flexmark-all" % "0.35.10" % s"$Test,$IntegrationTest", - "org.mockito" %% "mockito-scala-scalatest" % "1.16.25" % Test + "uk.gov.hmrc" %% "bootstrap-test-play-30" % bootstrapVersion % s"$Test,$IntegrationTest", + "org.mockito" %% "mockito-scala-scalatest" % "1.17.29" % Test ) def apply(): Seq[ModuleID] = compile ++ test diff --git a/project/build.properties b/project/build.properties index 88dc997..e8a1e24 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1 +1 @@ -sbt.version=1.5.8 \ No newline at end of file +sbt.version=1.9.7 diff --git a/project/plugins.sbt b/project/plugins.sbt index 5bcce57..5cf3726 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -4,12 +4,13 @@ resolvers += MavenRepository("HMRC-open-artefacts-maven2", "https://open.artefac resolvers += Resolver.url("HMRC-open-artefacts-ivy2", url("https://open.artefacts.tax.service.gov.uk/ivy2"))( Resolver.ivyStylePatterns) -addSbtPlugin("com.typesafe.play" % "sbt-plugin" % "2.8.7") +addSbtPlugin("org.playframework" % "sbt-plugin" % "3.0.0") -addSbtPlugin("uk.gov.hmrc" % "sbt-auto-build" % "3.7.0") +addSbtPlugin("uk.gov.hmrc" % "sbt-auto-build" % "3.18.0") -addSbtPlugin("uk.gov.hmrc" % "sbt-distributables" % "2.1.0") +addSbtPlugin("uk.gov.hmrc" % "sbt-distributables" % "2.4.0") -addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.6.1") - -addSbtPlugin("org.scalastyle" %% "scalastyle-sbt-plugin" % "1.0.0") +addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.6.1" + exclude("org.scala-lang.modules", "scala-xml_2.12")) +addSbtPlugin("org.scalastyle" %% "scalastyle-sbt-plugin" % "1.0.0" + exclude("org.scala-lang.modules", "scala-xml_2.12")) diff --git a/test/connectors/s3/PolicySignerSpec.scala b/test/connectors/s3/PolicySignerSpec.scala index f69f994..207237c 100644 --- a/test/connectors/s3/PolicySignerSpec.scala +++ b/test/connectors/s3/PolicySignerSpec.scala @@ -1,5 +1,5 @@ /* - * Copyright 2022 HM Revenue & Customs + * Copyright 2024 HM Revenue & Customs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/test/connectors/s3/S3UploadFormGeneratorSpec.scala b/test/connectors/s3/S3UploadFormGeneratorSpec.scala index d6d35de..a6984ce 100644 --- a/test/connectors/s3/S3UploadFormGeneratorSpec.scala +++ b/test/connectors/s3/S3UploadFormGeneratorSpec.scala @@ -1,5 +1,5 @@ /* - * Copyright 2022 HM Revenue & Customs + * Copyright 2024 HM Revenue & Customs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -76,7 +76,7 @@ class S3UploadFormGeneratorSpec extends UnitSpec with GivenWhenThen { ((policy \ "conditions").get \\ "error_action_redirect").head.as[String] shouldBe "http://test.com/error" val conditions = (policy \ "conditions").as[JsArray].value - val arrayConditions: Seq[Seq[JsValue]] = conditions.flatMap(_.asOpt[JsArray].map(_.value)) + val arrayConditions = conditions.flatMap(_.asOpt[JsArray].map(_.value)) And("policy contains proper size constraints") val fileSizeCondition = arrayConditions.find(_.toIndexedSeq(0).as[String] == "content-length-range") diff --git a/test/controllers/PrepareUploadControllerSpec.scala b/test/controllers/PrepareUploadControllerSpec.scala index 414e677..74b0d74 100644 --- a/test/controllers/PrepareUploadControllerSpec.scala +++ b/test/controllers/PrepareUploadControllerSpec.scala @@ -1,5 +1,5 @@ /* - * Copyright 2022 HM Revenue & Customs + * Copyright 2024 HM Revenue & Customs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,25 +16,23 @@ package controllers -import java.time.Clock -import akka.actor.ActorSystem -import akka.stream.ActorMaterializer import config.ServiceConfiguration +import controllers.PrepareUploadControllerSpec.{ConsumingService, UserAgent, requestTemplate, settingsTemplate} import controllers.model.{PrepareUploadRequest, PreparedUploadResponse, Reference, UploadFormTemplate} +import org.apache.pekko.actor.ActorSystem import org.scalatest.GivenWhenThen import play.api.http.HeaderNames.USER_AGENT import play.api.http.Status.{BAD_REQUEST, OK} import play.api.libs.json.{JsValue, Json} import play.api.mvc.Action import play.api.mvc.Results.Ok -import play.api.test.Helpers.{contentAsString, status} +import play.api.test.Helpers.{contentAsJson, contentAsString, status} import play.api.test.{FakeRequest, Helpers, StubControllerComponentsFactory} import services.PrepareUploadService import services.model.UploadSettings import test.UnitSpec -import Helpers.contentAsJson -import controllers.PrepareUploadControllerSpec.{ConsumingService, UserAgent, requestTemplate, settingsTemplate} +import java.time.Clock import scala.concurrent.Future import scala.concurrent.duration._ import scala.language.postfixOps @@ -42,8 +40,8 @@ import scala.language.postfixOps class PrepareUploadControllerSpec extends UnitSpec with StubControllerComponentsFactory with GivenWhenThen { private implicit val actorSystem: ActorSystem = ActorSystem() - private implicit val materializer: ActorMaterializer = ActorMaterializer() - private implicit val timeout: akka.util.Timeout = 10 seconds + + private implicit val timeout: org.apache.pekko.util.Timeout = 10 seconds private val clock = Clock.fixed(Clock.systemDefaultZone().instant(), Clock.systemDefaultZone().getZone) diff --git a/test/controllers/model/PrepareUploadRequestSpec.scala b/test/controllers/model/PrepareUploadRequestSpec.scala index e90904e..a075425 100644 --- a/test/controllers/model/PrepareUploadRequestSpec.scala +++ b/test/controllers/model/PrepareUploadRequestSpec.scala @@ -1,5 +1,5 @@ /* - * Copyright 2022 HM Revenue & Customs + * Copyright 2024 HM Revenue & Customs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/test/model/PrepareUploadServiceSpec.scala b/test/model/PrepareUploadServiceSpec.scala index 368c887..bb2c5e3 100644 --- a/test/model/PrepareUploadServiceSpec.scala +++ b/test/model/PrepareUploadServiceSpec.scala @@ -1,5 +1,5 @@ /* - * Copyright 2022 HM Revenue & Customs + * Copyright 2024 HM Revenue & Customs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,7 +19,6 @@ package model import java.time import java.time.Instant import com.codahale.metrics.MetricRegistry -import com.kenshoo.play.metrics.Metrics import config.ServiceConfiguration import connectors.model.{UploadFormGenerator, UploadParameters} import controllers.model.PrepareUploadRequest @@ -28,6 +27,7 @@ import org.scalatest.GivenWhenThen import services.PrepareUploadService import services.model.UploadSettings import test.UnitSpec +import uk.gov.hmrc.play.bootstrap.metrics.Metrics class PrepareUploadServiceSpec extends UnitSpec with GivenWhenThen { @@ -56,7 +56,6 @@ class PrepareUploadServiceSpec extends UnitSpec with GivenWhenThen { override val defaultRegistry: MetricRegistry = new MetricRegistry - override def toJson: String = ??? } private val s3PostSigner = new UploadFormGenerator { diff --git a/test/test/UnitSpec.scala b/test/test/UnitSpec.scala index 3fed5e2..2c8d814 100644 --- a/test/test/UnitSpec.scala +++ b/test/test/UnitSpec.scala @@ -1,5 +1,5 @@ /* - * Copyright 2022 HM Revenue & Customs + * Copyright 2024 HM Revenue & Customs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/test/utils/UserAgentFilterSpec.scala b/test/utils/UserAgentFilterSpec.scala index 644e3b5..7ee88e3 100644 --- a/test/utils/UserAgentFilterSpec.scala +++ b/test/utils/UserAgentFilterSpec.scala @@ -1,5 +1,5 @@ /* - * Copyright 2022 HM Revenue & Customs + * Copyright 2024 HM Revenue & Customs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,7 +16,7 @@ package utils -import akka.util.Timeout +import org.apache.pekko.util.Timeout import org.scalatest.GivenWhenThen import play.api.Logging import play.api.http.Status.{BAD_REQUEST, OK} @@ -69,4 +69,4 @@ private object UserAgentFilterSpec { val block: (Request[_], String) => Future[Result] = (_, userAgent) => Future.successful(Ok(userAgent)) object UserAgentFilter extends Logging with UserAgentFilter -} \ No newline at end of file +} From 855e13e09a24635a0f7fb6fde093892e2fccb71c Mon Sep 17 00:00:00 2001 From: peteanning <22272449+peteanning@users.noreply.github.com> Date: Thu, 4 Jan 2024 13:41:44 +0000 Subject: [PATCH 78/96] BDOG-2800: convert it:test to it/test --- build.sbt | 10 +++++--- conf/application.conf | 24 +------------------ .../PrepareUploadControllerISpec.scala | 0 project/AppDependencies.scala | 2 +- .../PrepareUploadControllerSpec.scala | 2 +- 5 files changed, 10 insertions(+), 28 deletions(-) rename it/{ => test}/controllers/PrepareUploadControllerISpec.scala (100%) diff --git a/build.sbt b/build.sbt index 262b03c..790a34b 100644 --- a/build.sbt +++ b/build.sbt @@ -2,7 +2,7 @@ import play.sbt.PlayImport.PlayKeys.playDefaultPort import sbt.Keys._ import sbt._ import scoverage.ScoverageKeys -import uk.gov.hmrc.DefaultBuildSettings.integrationTestSettings +import uk.gov.hmrc.DefaultBuildSettings import uk.gov.hmrc.sbtdistributables.SbtDistributablesPlugin @@ -35,5 +35,9 @@ lazy val microservice = Project(appName, file(".")) .settings(libraryDependencies ++= AppDependencies()) .settings(resolvers += Resolver.jcenterRepo) .settings(Test / parallelExecution := false) - .configs(IntegrationTest) - .settings(integrationTestSettings(): _*) + +lazy val it = project + .enablePlugins(PlayScala) + .dependsOn(microservice % "test->test") // the "test->test" allows reusing test code and test dependencies + .settings(DefaultBuildSettings.itSettings()) + .settings(libraryDependencies ++= AppDependencies()) diff --git a/conf/application.conf b/conf/application.conf index a1758de..cb6dc82 100644 --- a/conf/application.conf +++ b/conf/application.conf @@ -18,31 +18,9 @@ include "backend.conf" appName=upscan-initiate -play.modules.enabled += "com.kenshoo.play.metrics.PlayModule" - -# An ApplicationLoader that uses Guice to bootstrap the application. -play.application.loader = "uk.gov.hmrc.play.bootstrap.ApplicationLoader" - -# Primary entry point for all HTTP requests on Play applications -play.http.requestHandler = "uk.gov.hmrc.play.bootstrap.http.RequestHandler" - -# Provides an implementation of AuditConnector. Use `uk.gov.hmrc.play.bootstrap.AuditModule` or create your own. -# An audit connector must be provided. -play.modules.enabled += "uk.gov.hmrc.play.bootstrap.AuditModule" - -# Provides an implementation of MetricsFilter. Use `uk.gov.hmrc.play.graphite.GraphiteMetricsModule` or create your own. -# A metric filter must be provided -play.modules.enabled += "uk.gov.hmrc.play.bootstrap.graphite.GraphiteMetricsModule" - -# Provides an implementation and configures all filters required by a Platform frontend microservice. -play.modules.enabled += "uk.gov.hmrc.play.bootstrap.backend.BackendModule" - play.modules.enabled += "UpscanInitiateModule" play.modules.enabled += "connectors.s3.S3Module" -# Json error handler -play.http.errorHandler = "uk.gov.hmrc.play.bootstrap.backend.http.JsonErrorHandler" - # The application languages # ~~~~~ play.i18n.langs = ["en"] @@ -96,4 +74,4 @@ auditing { port = 8100 } } -} \ No newline at end of file +} diff --git a/it/controllers/PrepareUploadControllerISpec.scala b/it/test/controllers/PrepareUploadControllerISpec.scala similarity index 100% rename from it/controllers/PrepareUploadControllerISpec.scala rename to it/test/controllers/PrepareUploadControllerISpec.scala diff --git a/project/AppDependencies.scala b/project/AppDependencies.scala index bec9ce8..714c249 100644 --- a/project/AppDependencies.scala +++ b/project/AppDependencies.scala @@ -10,7 +10,7 @@ object AppDependencies { ) private val test = Seq( - "uk.gov.hmrc" %% "bootstrap-test-play-30" % bootstrapVersion % s"$Test,$IntegrationTest", + "uk.gov.hmrc" %% "bootstrap-test-play-30" % bootstrapVersion % Test, "org.mockito" %% "mockito-scala-scalatest" % "1.17.29" % Test ) diff --git a/test/controllers/PrepareUploadControllerSpec.scala b/test/controllers/PrepareUploadControllerSpec.scala index 74b0d74..fac03d3 100644 --- a/test/controllers/PrepareUploadControllerSpec.scala +++ b/test/controllers/PrepareUploadControllerSpec.scala @@ -261,7 +261,7 @@ class PrepareUploadControllerSpec extends UnitSpec with StubControllerComponents private def badRequestInitiate(// scalastyle:ignore config: ServiceConfiguration, - prepareUploadAction: PrepareUploadController => Action[JsValue]) { + prepareUploadAction: PrepareUploadController => Action[JsValue]): Unit = { "return a bad request error when the request has the wrong structure" in new WithGlobalFileSizeLimitFixture { val controller = new PrepareUploadController(prepareUploadService, config, clock, stubControllerComponents()) From aaf223d63b3b9b5f6c2ff8f35581374e15aeb26b Mon Sep 17 00:00:00 2001 From: peteanning <22272449+peteanning@users.noreply.github.com> Date: Thu, 4 Jan 2024 13:54:04 +0000 Subject: [PATCH 79/96] BDOG-2800: pr commenter fixes --- conf/application-json-logger.xml | 6 +++--- conf/application.conf | 11 ----------- project/plugins.sbt | 2 +- 3 files changed, 4 insertions(+), 15 deletions(-) diff --git a/conf/application-json-logger.xml b/conf/application-json-logger.xml index 332df82..3a22bb0 100644 --- a/conf/application-json-logger.xml +++ b/conf/application-json-logger.xml @@ -5,11 +5,11 @@ - + - + - + diff --git a/conf/application.conf b/conf/application.conf index cb6dc82..2120997 100644 --- a/conf/application.conf +++ b/conf/application.conf @@ -64,14 +64,3 @@ aws { callbackValidation.allowedProtocols = "https" global.file.size.limit = 104857600 - -auditing { - enabled = false - traceRequests = true - consumer { - baseUri { - host = localhost - port = 8100 - } - } -} diff --git a/project/plugins.sbt b/project/plugins.sbt index 5cf3726..3b07ea3 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -4,7 +4,7 @@ resolvers += MavenRepository("HMRC-open-artefacts-maven2", "https://open.artefac resolvers += Resolver.url("HMRC-open-artefacts-ivy2", url("https://open.artefacts.tax.service.gov.uk/ivy2"))( Resolver.ivyStylePatterns) -addSbtPlugin("org.playframework" % "sbt-plugin" % "3.0.0") +addSbtPlugin("org.playframework" % "sbt-plugin" % "3.0.1") addSbtPlugin("uk.gov.hmrc" % "sbt-auto-build" % "3.18.0") From 9763fa2d02e8dde1afe57d3554beb0d14c85acc4 Mon Sep 17 00:00:00 2001 From: colin-lamed <9568290+colin-lamed@users.noreply.github.com> Date: Wed, 1 May 2024 10:22:54 +0100 Subject: [PATCH 80/96] BDOG-3087 Update sbt-scoverage --- build.sbt | 23 +++++++++++------------ conf/application.conf | 5 ----- conf/logback.xml | 10 ---------- project/AppDependencies.scala | 2 +- project/plugins.sbt | 15 +++++---------- 5 files changed, 17 insertions(+), 38 deletions(-) diff --git a/build.sbt b/build.sbt index 790a34b..8b0d250 100644 --- a/build.sbt +++ b/build.sbt @@ -12,14 +12,14 @@ ThisBuild / majorVersion := 0 ThisBuild / scalaVersion := "2.13.12" lazy val scoverageSettings = Seq( - // Semicolon-separated list of regexs matching classes to exclude - ScoverageKeys.coverageExcludedPackages := ";Reverse.*;.*AuthService.*;models/.data/..*;view.*", - ScoverageKeys.coverageExcludedFiles := - ".*/frontendGlobal.*;.*/frontendAppConfig.*;.*/frontendWiring.*;.*/views/.*_template.*;.*/govuk_wrapper.*;.*/routes_routing.*;.*/BuildInfo.*", - // Minimum is deliberately low to avoid failures initially - please increase as we add more coverage - ScoverageKeys.coverageMinimum := 25, - ScoverageKeys.coverageFailOnMinimum := false, - ScoverageKeys.coverageHighlighting := true + // Semicolon-separated list of regexs matching classes to exclude + ScoverageKeys.coverageExcludedPackages := ";Reverse.*;.*AuthService.*;models/.data/..*;view.*", + ScoverageKeys.coverageExcludedFiles := + ".*/frontendGlobal.*;.*/frontendAppConfig.*;.*/frontendWiring.*;.*/views/.*_template.*;.*/govuk_wrapper.*;.*/routes_routing.*;.*/BuildInfo.*", + // Minimum is deliberately low to avoid failures initially - please increase as we add more coverage + ScoverageKeys.coverageMinimumStmtTotal := 25, + ScoverageKeys.coverageFailOnMinimum := false, + ScoverageKeys.coverageHighlighting := true ) lazy val microservice = Project(appName, file(".")) @@ -27,10 +27,9 @@ lazy val microservice = Project(appName, file(".")) .disablePlugins(JUnitXmlReportPlugin) //Required to prevent https://github.com/scalatest/scalatest/issues/1427 .settings(scoverageSettings: _*) .settings(scalacOptions ++= Seq( - "-Wconf:cat=unused-imports&src=.*routes.*:s" //silence import warnings in routes generated by comments - , "-Wconf:cat=unused&src=.*routes.*:s" //silence private val defaultPrefix in class Routes is never used - ) - ) + "-Wconf:cat=unused-imports&src=.*routes.*:s" //silence import warnings in routes generated by comments + , "-Wconf:cat=unused&src=.*routes.*:s" //silence private val defaultPrefix in class Routes is never used + )) .settings(playDefaultPort := 9571) .settings(libraryDependencies ++= AppDependencies()) .settings(resolvers += Resolver.jcenterRepo) diff --git a/conf/application.conf b/conf/application.conf index 2120997..62d8e77 100644 --- a/conf/application.conf +++ b/conf/application.conf @@ -21,10 +21,6 @@ appName=upscan-initiate play.modules.enabled += "UpscanInitiateModule" play.modules.enabled += "connectors.s3.S3Module" -# The application languages -# ~~~~~ -play.i18n.langs = ["en"] - # Router # ~~~~~ # Define the Router object to use for this application. @@ -40,7 +36,6 @@ play.http.router=prod.Routes # Metrics plugin settings - graphite reporting is configured on a per env basis metrics { - name = ${appName} enabled = true } diff --git a/conf/logback.xml b/conf/logback.xml index 5c4ccee..3e51242 100644 --- a/conf/logback.xml +++ b/conf/logback.xml @@ -14,12 +14,6 @@ - - - %date{ISO8601} level=[%level] logger=[%logger] thread=[%thread] rid=[not-available] user=[not-available] message=[%message] %replace(exception=[%xException]){'^exception=\[\]$',''}%n - - - logs/access.log @@ -39,10 +33,6 @@ - - - - diff --git a/project/AppDependencies.scala b/project/AppDependencies.scala index 714c249..dc4f219 100644 --- a/project/AppDependencies.scala +++ b/project/AppDependencies.scala @@ -1,7 +1,7 @@ import sbt._ object AppDependencies { - private val bootstrapVersion = "8.4.0" + private val bootstrapVersion = "8.5.0" private val compile = Seq( "uk.gov.hmrc" %% "bootstrap-backend-play-30" % bootstrapVersion, diff --git a/project/plugins.sbt b/project/plugins.sbt index 3b07ea3..cbc58ca 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -4,13 +4,8 @@ resolvers += MavenRepository("HMRC-open-artefacts-maven2", "https://open.artefac resolvers += Resolver.url("HMRC-open-artefacts-ivy2", url("https://open.artefacts.tax.service.gov.uk/ivy2"))( Resolver.ivyStylePatterns) -addSbtPlugin("org.playframework" % "sbt-plugin" % "3.0.1") - -addSbtPlugin("uk.gov.hmrc" % "sbt-auto-build" % "3.18.0") - -addSbtPlugin("uk.gov.hmrc" % "sbt-distributables" % "2.4.0") - -addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.6.1" - exclude("org.scala-lang.modules", "scala-xml_2.12")) -addSbtPlugin("org.scalastyle" %% "scalastyle-sbt-plugin" % "1.0.0" - exclude("org.scala-lang.modules", "scala-xml_2.12")) +addSbtPlugin("org.playframework" % "sbt-plugin" % "3.0.2") +addSbtPlugin("uk.gov.hmrc" % "sbt-auto-build" % "3.21.0") +addSbtPlugin("uk.gov.hmrc" % "sbt-distributables" % "2.5.0") +addSbtPlugin("org.scoverage" % "sbt-scoverage" % "2.0.11") +addSbtPlugin("org.scalastyle" %% "scalastyle-sbt-plugin" % "1.0.0" exclude("org.scala-lang.modules", "scala-xml_2.12")) From 995da03ae635411a030005d509ffad63a9d17acd Mon Sep 17 00:00:00 2001 From: colin-lamed <9568290+colin-lamed@users.noreply.github.com> Date: Wed, 23 Oct 2024 15:02:35 +0100 Subject: [PATCH 81/96] BDOG-3255 Use hmrc package namespace --- .gitignore | 3 +++ .../UpscanInitiateModule.scala | 6 ++++-- .../config/ServiceConfiguration.scala | 8 +++---- .../connector}/model/AwsCredentials.scala | 2 +- .../connector}/model/ContentLengthRange.scala | 2 +- .../model/UploadFormGenerator.scala | 2 +- .../connector}/model/UploadParameters.scala | 2 +- .../connector}/s3/PolicySigner.scala | 8 +++---- .../connector}/s3/S3Module.scala | 4 ++-- .../connector}/s3/S3UploadFormGenerator.scala | 10 ++++----- .../s3/S3UploadFormGeneratorProvider.scala | 8 +++---- .../controller}/PrepareUploadController.scala | 21 +++++++++---------- .../model/PrepareUploadRequest.scala | 2 +- .../model/PreparedUploadResponse.scala | 2 +- .../controller}/model/Reference.scala | 6 +++--- .../model/UploadFormTemplate.scala | 2 +- .../service}/PrepareUploadService.scala | 16 +++++++------- .../service}/model/UploadSettings.scala | 4 ++-- .../util}/UserAgentFilter.scala | 2 +- conf/app.routes | 4 ++-- conf/application.conf | 4 ++-- conf/logback.xml | 7 +++---- .../PrepareUploadControllerISpec.scala | 2 +- .../connector}/s3/PolicySignerSpec.scala | 6 +++--- .../s3/S3UploadFormGeneratorSpec.scala | 12 +++++------ .../PrepareUploadControllerSpec.scala | 14 ++++++------- .../model/PrepareUploadRequestSpec.scala | 6 +++--- .../model/PrepareUploadServiceSpec.scala | 21 ++++++++++--------- .../hmrc/upscaninitiate}/test/UnitSpec.scala | 2 +- .../util}/UserAgentFilterSpec.scala | 4 ++-- 30 files changed, 98 insertions(+), 94 deletions(-) rename app/{ => uk/gov/hmrc/upscaninitiate}/UpscanInitiateModule.scala (88%) rename app/{ => uk/gov/hmrc/upscaninitiate}/config/ServiceConfiguration.scala (98%) rename app/{connectors => uk/gov/hmrc/upscaninitiate/connector}/model/AwsCredentials.scala (93%) rename app/{connectors => uk/gov/hmrc/upscaninitiate/connector}/model/ContentLengthRange.scala (92%) rename app/{connectors => uk/gov/hmrc/upscaninitiate/connector}/model/UploadFormGenerator.scala (93%) rename app/{connectors => uk/gov/hmrc/upscaninitiate/connector}/model/UploadParameters.scala (94%) rename app/{connectors => uk/gov/hmrc/upscaninitiate/connector}/s3/PolicySigner.scala (94%) rename app/{connectors => uk/gov/hmrc/upscaninitiate/connector}/s3/S3Module.scala (88%) rename app/{connectors => uk/gov/hmrc/upscaninitiate/connector}/s3/S3UploadFormGenerator.scala (97%) rename app/{connectors => uk/gov/hmrc/upscaninitiate/connector}/s3/S3UploadFormGeneratorProvider.scala (84%) rename app/{controllers => uk/gov/hmrc/upscaninitiate/controller}/PrepareUploadController.scala (90%) rename app/{controllers => uk/gov/hmrc/upscaninitiate/controller}/model/PrepareUploadRequest.scala (97%) rename app/{controllers => uk/gov/hmrc/upscaninitiate/controller}/model/PreparedUploadResponse.scala (95%) rename app/{controllers => uk/gov/hmrc/upscaninitiate/controller}/model/Reference.scala (94%) rename app/{controllers => uk/gov/hmrc/upscaninitiate/controller}/model/UploadFormTemplate.scala (94%) rename app/{services => uk/gov/hmrc/upscaninitiate/service}/PrepareUploadService.scala (90%) rename app/{services => uk/gov/hmrc/upscaninitiate/service}/model/UploadSettings.scala (87%) rename app/{utils => uk/gov/hmrc/upscaninitiate/util}/UserAgentFilter.scala (97%) rename it/test/{controllers => uk/gov/hmrc/upscaninitiate/controller}/PrepareUploadControllerISpec.scala (99%) rename test/{connectors => uk/gov/hmrc/upscaninitiate/connector}/s3/PolicySignerSpec.scala (94%) rename test/{connectors => uk/gov/hmrc/upscaninitiate/connector}/s3/S3UploadFormGeneratorSpec.scala (97%) rename test/{controllers => uk/gov/hmrc/upscaninitiate/controller}/PrepareUploadControllerSpec.scala (96%) rename test/{controllers => uk/gov/hmrc/upscaninitiate/controller}/model/PrepareUploadRequestSpec.scala (95%) rename test/{ => uk/gov/hmrc/upscaninitiate}/model/PrepareUploadServiceSpec.scala (95%) rename test/{ => uk/gov/hmrc/upscaninitiate}/test/UnitSpec.scala (95%) rename test/{utils => uk/gov/hmrc/upscaninitiate/util}/UserAgentFilterSpec.scala (96%) diff --git a/.gitignore b/.gitignore index 0515f20..974d7fa 100644 --- a/.gitignore +++ b/.gitignore @@ -36,3 +36,6 @@ bin t nohup.out nohup +.metals/ +.vscode/ + diff --git a/app/UpscanInitiateModule.scala b/app/uk/gov/hmrc/upscaninitiate/UpscanInitiateModule.scala similarity index 88% rename from app/UpscanInitiateModule.scala rename to app/uk/gov/hmrc/upscaninitiate/UpscanInitiateModule.scala index 3452628..7311bfc 100644 --- a/app/UpscanInitiateModule.scala +++ b/app/uk/gov/hmrc/upscaninitiate/UpscanInitiateModule.scala @@ -14,11 +14,13 @@ * limitations under the License. */ -import java.time.Clock +package uk.gov.hmrc.upscaninitiate -import config.{PlayBasedServiceConfiguration, ServiceConfiguration} import play.api.inject.{Binding, Module} import play.api.{Configuration, Environment} +import uk.gov.hmrc.upscaninitiate.config.{PlayBasedServiceConfiguration, ServiceConfiguration} + +import java.time.Clock class UpscanInitiateModule extends Module { override def bindings(environment: Environment, configuration: Configuration): Seq[Binding[_]] = diff --git a/app/config/ServiceConfiguration.scala b/app/uk/gov/hmrc/upscaninitiate/config/ServiceConfiguration.scala similarity index 98% rename from app/config/ServiceConfiguration.scala rename to app/uk/gov/hmrc/upscaninitiate/config/ServiceConfiguration.scala index 61d4c2d..b09b7c6 100644 --- a/app/config/ServiceConfiguration.scala +++ b/app/uk/gov/hmrc/upscaninitiate/config/ServiceConfiguration.scala @@ -14,15 +14,15 @@ * limitations under the License. */ -package config - -import java.time.Duration +package uk.gov.hmrc.upscaninitiate.config import com.typesafe.config.ConfigException -import javax.inject.Inject import org.apache.commons.lang3.StringUtils.isNotBlank import play.api.Configuration +import java.time.Duration +import javax.inject.Inject + trait ServiceConfiguration { def region: String diff --git a/app/connectors/model/AwsCredentials.scala b/app/uk/gov/hmrc/upscaninitiate/connector/model/AwsCredentials.scala similarity index 93% rename from app/connectors/model/AwsCredentials.scala rename to app/uk/gov/hmrc/upscaninitiate/connector/model/AwsCredentials.scala index b34895f..ad3612a 100644 --- a/app/connectors/model/AwsCredentials.scala +++ b/app/uk/gov/hmrc/upscaninitiate/connector/model/AwsCredentials.scala @@ -14,6 +14,6 @@ * limitations under the License. */ -package connectors.model +package uk.gov.hmrc.upscaninitiate.connector.model final case class AwsCredentials(accessKeyId: String, secretKey: String, sessionToken: Option[String]) diff --git a/app/connectors/model/ContentLengthRange.scala b/app/uk/gov/hmrc/upscaninitiate/connector/model/ContentLengthRange.scala similarity index 92% rename from app/connectors/model/ContentLengthRange.scala rename to app/uk/gov/hmrc/upscaninitiate/connector/model/ContentLengthRange.scala index 126160d..8cc3012 100644 --- a/app/connectors/model/ContentLengthRange.scala +++ b/app/uk/gov/hmrc/upscaninitiate/connector/model/ContentLengthRange.scala @@ -14,6 +14,6 @@ * limitations under the License. */ -package connectors.model +package uk.gov.hmrc.upscaninitiate.connector.model case class ContentLengthRange(min: Long, max: Long) diff --git a/app/connectors/model/UploadFormGenerator.scala b/app/uk/gov/hmrc/upscaninitiate/connector/model/UploadFormGenerator.scala similarity index 93% rename from app/connectors/model/UploadFormGenerator.scala rename to app/uk/gov/hmrc/upscaninitiate/connector/model/UploadFormGenerator.scala index 42c2a6c..6fe9cd9 100644 --- a/app/connectors/model/UploadFormGenerator.scala +++ b/app/uk/gov/hmrc/upscaninitiate/connector/model/UploadFormGenerator.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package connectors.model +package uk.gov.hmrc.upscaninitiate.connector.model trait UploadFormGenerator { def generateFormFields(uploadParameters: UploadParameters): Map[String, String] diff --git a/app/connectors/model/UploadParameters.scala b/app/uk/gov/hmrc/upscaninitiate/connector/model/UploadParameters.scala similarity index 94% rename from app/connectors/model/UploadParameters.scala rename to app/uk/gov/hmrc/upscaninitiate/connector/model/UploadParameters.scala index acac60e..0ef4f71 100644 --- a/app/connectors/model/UploadParameters.scala +++ b/app/uk/gov/hmrc/upscaninitiate/connector/model/UploadParameters.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package connectors.model +package uk.gov.hmrc.upscaninitiate.connector.model import java.time.Instant diff --git a/app/connectors/s3/PolicySigner.scala b/app/uk/gov/hmrc/upscaninitiate/connector/s3/PolicySigner.scala similarity index 94% rename from app/connectors/s3/PolicySigner.scala rename to app/uk/gov/hmrc/upscaninitiate/connector/s3/PolicySigner.scala index 9d3911f..5c49d08 100644 --- a/app/connectors/s3/PolicySigner.scala +++ b/app/uk/gov/hmrc/upscaninitiate/connector/s3/PolicySigner.scala @@ -14,12 +14,12 @@ * limitations under the License. */ -package connectors.s3 - -import java.nio.charset.Charset +package uk.gov.hmrc.upscaninitiate.connector.s3 import com.amazonaws.util.{BinaryUtils, StringUtils} -import connectors.model.AwsCredentials +import uk.gov.hmrc.upscaninitiate.connector.model.AwsCredentials + +import java.nio.charset.Charset import javax.crypto.Mac import javax.crypto.spec.SecretKeySpec diff --git a/app/connectors/s3/S3Module.scala b/app/uk/gov/hmrc/upscaninitiate/connector/s3/S3Module.scala similarity index 88% rename from app/connectors/s3/S3Module.scala rename to app/uk/gov/hmrc/upscaninitiate/connector/s3/S3Module.scala index 3a4a9a1..0a49a6e 100644 --- a/app/connectors/s3/S3Module.scala +++ b/app/uk/gov/hmrc/upscaninitiate/connector/s3/S3Module.scala @@ -14,11 +14,11 @@ * limitations under the License. */ -package connectors.s3 +package uk.gov.hmrc.upscaninitiate.connector.s3 -import connectors.model.UploadFormGenerator import play.api.inject.{Binding, Module} import play.api.{Configuration, Environment} +import uk.gov.hmrc.upscaninitiate.connector.model.UploadFormGenerator class S3Module extends Module { override def bindings(environment: Environment, configuration: Configuration): Seq[Binding[_]] = diff --git a/app/connectors/s3/S3UploadFormGenerator.scala b/app/uk/gov/hmrc/upscaninitiate/connector/s3/S3UploadFormGenerator.scala similarity index 97% rename from app/connectors/s3/S3UploadFormGenerator.scala rename to app/uk/gov/hmrc/upscaninitiate/connector/s3/S3UploadFormGenerator.scala index 8197186..ad30b65 100644 --- a/app/connectors/s3/S3UploadFormGenerator.scala +++ b/app/uk/gov/hmrc/upscaninitiate/connector/s3/S3UploadFormGenerator.scala @@ -14,14 +14,14 @@ * limitations under the License. */ -package connectors.s3 +package uk.gov.hmrc.upscaninitiate.connector.s3 +import play.api.libs.json.{JsArray, JsValue, Json} +import uk.gov.hmrc.upscaninitiate.connector.model.{AwsCredentials, UploadFormGenerator, UploadParameters} + +import java.time.{Instant, ZoneOffset} import java.time.format.DateTimeFormatter import java.time.format.DateTimeFormatter.ISO_INSTANT -import java.time.{Instant, ZoneOffset} - -import connectors.model.{AwsCredentials, UploadFormGenerator, UploadParameters} -import play.api.libs.json.{JsArray, JsValue, Json} class S3UploadFormGenerator( credentials: AwsCredentials, diff --git a/app/connectors/s3/S3UploadFormGeneratorProvider.scala b/app/uk/gov/hmrc/upscaninitiate/connector/s3/S3UploadFormGeneratorProvider.scala similarity index 84% rename from app/connectors/s3/S3UploadFormGeneratorProvider.scala rename to app/uk/gov/hmrc/upscaninitiate/connector/s3/S3UploadFormGeneratorProvider.scala index abaeb47..1912c93 100644 --- a/app/connectors/s3/S3UploadFormGeneratorProvider.scala +++ b/app/uk/gov/hmrc/upscaninitiate/connector/s3/S3UploadFormGeneratorProvider.scala @@ -14,12 +14,12 @@ * limitations under the License. */ -package connectors.s3 +package uk.gov.hmrc.upscaninitiate.connector.s3 -import config.ServiceConfiguration -import java.time.{Clock, Instant} +import uk.gov.hmrc.upscaninitiate.config.ServiceConfiguration +import uk.gov.hmrc.upscaninitiate.connector.model.{AwsCredentials, UploadFormGenerator} -import connectors.model.{AwsCredentials, UploadFormGenerator} +import java.time.{Clock, Instant} import javax.inject.{Inject, Provider, Singleton} @Singleton diff --git a/app/controllers/PrepareUploadController.scala b/app/uk/gov/hmrc/upscaninitiate/controller/PrepareUploadController.scala similarity index 90% rename from app/controllers/PrepareUploadController.scala rename to app/uk/gov/hmrc/upscaninitiate/controller/PrepareUploadController.scala index 091a057..cd2a03a 100644 --- a/app/controllers/PrepareUploadController.scala +++ b/app/uk/gov/hmrc/upscaninitiate/controller/PrepareUploadController.scala @@ -14,22 +14,21 @@ * limitations under the License. */ -package controllers +package uk.gov.hmrc.upscaninitiate.controller -import java.net.URL -import java.time.{Clock, Instant} -import config.ServiceConfiguration -import controllers.model.{PrepareUploadRequest, PreparedUploadResponse} - -import javax.inject.{Inject, Singleton} import play.api.Logging import play.api.libs.json._ import play.api.mvc.{Action, ControllerComponents, Result} -import services.PrepareUploadService -import services.model.UploadSettings import uk.gov.hmrc.play.bootstrap.backend.controller.BackendController -import utils.UserAgentFilter +import uk.gov.hmrc.upscaninitiate.config.ServiceConfiguration +import uk.gov.hmrc.upscaninitiate.controller.model.{PrepareUploadRequest, PreparedUploadResponse} +import uk.gov.hmrc.upscaninitiate.service.PrepareUploadService +import uk.gov.hmrc.upscaninitiate.service.model.UploadSettings +import uk.gov.hmrc.upscaninitiate.util.UserAgentFilter +import java.net.URL +import java.time.{Clock, Instant} +import javax.inject.{Inject, Singleton} import scala.concurrent.Future import scala.util.{Failure, Success, Try} @@ -97,7 +96,7 @@ class PrepareUploadController @Inject()( } } - private[controllers] def withAllowedCallbackProtocol[A](callbackUrl: String)( + private[controller] def withAllowedCallbackProtocol[A](callbackUrl: String)( block: => Future[Result]): Future[Result] = { val allowedCallbackProtocols: Seq[String] = configuration.allowedCallbackProtocols diff --git a/app/controllers/model/PrepareUploadRequest.scala b/app/uk/gov/hmrc/upscaninitiate/controller/model/PrepareUploadRequest.scala similarity index 97% rename from app/controllers/model/PrepareUploadRequest.scala rename to app/uk/gov/hmrc/upscaninitiate/controller/model/PrepareUploadRequest.scala index c828edd..e7c1404 100644 --- a/app/controllers/model/PrepareUploadRequest.scala +++ b/app/uk/gov/hmrc/upscaninitiate/controller/model/PrepareUploadRequest.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package controllers.model +package uk.gov.hmrc.upscaninitiate.controller.model import play.api.libs.json.Reads import play.api.libs.functional.syntax._ diff --git a/app/controllers/model/PreparedUploadResponse.scala b/app/uk/gov/hmrc/upscaninitiate/controller/model/PreparedUploadResponse.scala similarity index 95% rename from app/controllers/model/PreparedUploadResponse.scala rename to app/uk/gov/hmrc/upscaninitiate/controller/model/PreparedUploadResponse.scala index d754d89..0c6c508 100644 --- a/app/controllers/model/PreparedUploadResponse.scala +++ b/app/uk/gov/hmrc/upscaninitiate/controller/model/PreparedUploadResponse.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package controllers.model +package uk.gov.hmrc.upscaninitiate.controller.model import play.api.libs.functional.syntax._ import play.api.libs.json.{OWrites, __} diff --git a/app/controllers/model/Reference.scala b/app/uk/gov/hmrc/upscaninitiate/controller/model/Reference.scala similarity index 94% rename from app/controllers/model/Reference.scala rename to app/uk/gov/hmrc/upscaninitiate/controller/model/Reference.scala index 9ef577f..bc7d23a 100644 --- a/app/controllers/model/Reference.scala +++ b/app/uk/gov/hmrc/upscaninitiate/controller/model/Reference.scala @@ -14,12 +14,12 @@ * limitations under the License. */ -package controllers.model - -import java.util.UUID.randomUUID +package uk.gov.hmrc.upscaninitiate.controller.model import play.api.libs.json.Writes +import java.util.UUID.randomUUID + case class Reference(value: String) extends AnyVal { override def toString: String = value.toString } diff --git a/app/controllers/model/UploadFormTemplate.scala b/app/uk/gov/hmrc/upscaninitiate/controller/model/UploadFormTemplate.scala similarity index 94% rename from app/controllers/model/UploadFormTemplate.scala rename to app/uk/gov/hmrc/upscaninitiate/controller/model/UploadFormTemplate.scala index fe8f453..ddc3e4f 100644 --- a/app/controllers/model/UploadFormTemplate.scala +++ b/app/uk/gov/hmrc/upscaninitiate/controller/model/UploadFormTemplate.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package controllers.model +package uk.gov.hmrc.upscaninitiate.controller.model import play.api.libs.functional.syntax._ import play.api.libs.json.{OWrites, __} diff --git a/app/services/PrepareUploadService.scala b/app/uk/gov/hmrc/upscaninitiate/service/PrepareUploadService.scala similarity index 90% rename from app/services/PrepareUploadService.scala rename to app/uk/gov/hmrc/upscaninitiate/service/PrepareUploadService.scala index 65835a8..eb4dbcb 100644 --- a/app/services/PrepareUploadService.scala +++ b/app/uk/gov/hmrc/upscaninitiate/service/PrepareUploadService.scala @@ -14,18 +14,18 @@ * limitations under the License. */ -package services +package uk.gov.hmrc.upscaninitiate.service -import java.time.Instant -import config.ServiceConfiguration -import connectors.model.{ContentLengthRange, UploadFormGenerator, UploadParameters} -import controllers.model.{PreparedUploadResponse, Reference, UploadFormTemplate} - -import javax.inject.{Inject, Singleton} import org.slf4j.MDC import play.api.Logging -import services.model.UploadSettings import uk.gov.hmrc.play.bootstrap.metrics.Metrics +import uk.gov.hmrc.upscaninitiate.config.ServiceConfiguration +import uk.gov.hmrc.upscaninitiate.connector.model.{ContentLengthRange, UploadFormGenerator, UploadParameters} +import uk.gov.hmrc.upscaninitiate.controller.model.{PreparedUploadResponse, Reference, UploadFormTemplate} +import uk.gov.hmrc.upscaninitiate.service.model.UploadSettings + +import java.time.Instant +import javax.inject.{Inject, Singleton} @Singleton class PrepareUploadService @Inject()( diff --git a/app/services/model/UploadSettings.scala b/app/uk/gov/hmrc/upscaninitiate/service/model/UploadSettings.scala similarity index 87% rename from app/services/model/UploadSettings.scala rename to app/uk/gov/hmrc/upscaninitiate/service/model/UploadSettings.scala index 597b616..8245225 100644 --- a/app/services/model/UploadSettings.scala +++ b/app/uk/gov/hmrc/upscaninitiate/service/model/UploadSettings.scala @@ -14,9 +14,9 @@ * limitations under the License. */ -package services.model +package uk.gov.hmrc.upscaninitiate.service.model -import controllers.model.PrepareUploadRequest +import uk.gov.hmrc.upscaninitiate.controller.model.PrepareUploadRequest final case class UploadSettings( uploadUrl: String, diff --git a/app/utils/UserAgentFilter.scala b/app/uk/gov/hmrc/upscaninitiate/util/UserAgentFilter.scala similarity index 97% rename from app/utils/UserAgentFilter.scala rename to app/uk/gov/hmrc/upscaninitiate/util/UserAgentFilter.scala index b6376a1..4a5eb61 100644 --- a/app/utils/UserAgentFilter.scala +++ b/app/uk/gov/hmrc/upscaninitiate/util/UserAgentFilter.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package utils +package uk.gov.hmrc.upscaninitiate.util import play.api.Logging import play.api.http.HeaderNames.USER_AGENT diff --git a/conf/app.routes b/conf/app.routes index deea2da..772c3a7 100644 --- a/conf/app.routes +++ b/conf/app.routes @@ -1,2 +1,2 @@ -POST /initiate controllers.PrepareUploadController.prepareUploadV1 -POST /v2/initiate controllers.PrepareUploadController.prepareUploadV2 +POST /initiate uk.gov.hmrc.upscaninitiate.controller.PrepareUploadController.prepareUploadV1 +POST /v2/initiate uk.gov.hmrc.upscaninitiate.controller.PrepareUploadController.prepareUploadV2 diff --git a/conf/application.conf b/conf/application.conf index 62d8e77..af00491 100644 --- a/conf/application.conf +++ b/conf/application.conf @@ -18,8 +18,8 @@ include "backend.conf" appName=upscan-initiate -play.modules.enabled += "UpscanInitiateModule" -play.modules.enabled += "connectors.s3.S3Module" +play.modules.enabled += "uk.gov.hmrc.upscaninitiate.UpscanInitiateModule" +play.modules.enabled += "uk.gov.hmrc.upscaninitiate.connector.s3.S3Module" # Router # ~~~~~ diff --git a/conf/logback.xml b/conf/logback.xml index 3e51242..ff38ea5 100644 --- a/conf/logback.xml +++ b/conf/logback.xml @@ -33,11 +33,10 @@ - - - - + + + diff --git a/it/test/controllers/PrepareUploadControllerISpec.scala b/it/test/uk/gov/hmrc/upscaninitiate/controller/PrepareUploadControllerISpec.scala similarity index 99% rename from it/test/controllers/PrepareUploadControllerISpec.scala rename to it/test/uk/gov/hmrc/upscaninitiate/controller/PrepareUploadControllerISpec.scala index 6ae0a19..b5b28bc 100644 --- a/it/test/controllers/PrepareUploadControllerISpec.scala +++ b/it/test/uk/gov/hmrc/upscaninitiate/controller/PrepareUploadControllerISpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package controllers +package uk.gov.hmrc.upscaninitiate.controller import org.scalatest.GivenWhenThen import org.scalatest.matchers.should diff --git a/test/connectors/s3/PolicySignerSpec.scala b/test/uk/gov/hmrc/upscaninitiate/connector/s3/PolicySignerSpec.scala similarity index 94% rename from test/connectors/s3/PolicySignerSpec.scala rename to test/uk/gov/hmrc/upscaninitiate/connector/s3/PolicySignerSpec.scala index 207237c..1567fa8 100644 --- a/test/connectors/s3/PolicySignerSpec.scala +++ b/test/uk/gov/hmrc/upscaninitiate/connector/s3/PolicySignerSpec.scala @@ -14,10 +14,10 @@ * limitations under the License. */ -package connectors.s3 +package uk.gov.hmrc.upscaninitiate.connector.s3 -import connectors.model.AwsCredentials -import test.UnitSpec +import uk.gov.hmrc.upscaninitiate.connector.model.AwsCredentials +import uk.gov.hmrc.upscaninitiate.test.UnitSpec class PolicySignerSpec extends UnitSpec { diff --git a/test/connectors/s3/S3UploadFormGeneratorSpec.scala b/test/uk/gov/hmrc/upscaninitiate/connector/s3/S3UploadFormGeneratorSpec.scala similarity index 97% rename from test/connectors/s3/S3UploadFormGeneratorSpec.scala rename to test/uk/gov/hmrc/upscaninitiate/connector/s3/S3UploadFormGeneratorSpec.scala index a6984ce..f1e962c 100644 --- a/test/connectors/s3/S3UploadFormGeneratorSpec.scala +++ b/test/uk/gov/hmrc/upscaninitiate/connector/s3/S3UploadFormGeneratorSpec.scala @@ -14,15 +14,15 @@ * limitations under the License. */ -package connectors.s3 +package uk.gov.hmrc.upscaninitiate.connector.s3 -import java.time.Instant -import java.util.Base64 - -import connectors.model.{AwsCredentials, ContentLengthRange, UploadParameters} import org.scalatest.GivenWhenThen import play.api.libs.json.{JsArray, JsValue, Json} -import test.UnitSpec +import uk.gov.hmrc.upscaninitiate.connector.model.{AwsCredentials, ContentLengthRange, UploadParameters} +import uk.gov.hmrc.upscaninitiate.test.UnitSpec + +import java.time.Instant +import java.util.Base64 class S3UploadFormGeneratorSpec extends UnitSpec with GivenWhenThen { diff --git a/test/controllers/PrepareUploadControllerSpec.scala b/test/uk/gov/hmrc/upscaninitiate/controller/PrepareUploadControllerSpec.scala similarity index 96% rename from test/controllers/PrepareUploadControllerSpec.scala rename to test/uk/gov/hmrc/upscaninitiate/controller/PrepareUploadControllerSpec.scala index fac03d3..cca30e9 100644 --- a/test/controllers/PrepareUploadControllerSpec.scala +++ b/test/uk/gov/hmrc/upscaninitiate/controller/PrepareUploadControllerSpec.scala @@ -14,11 +14,8 @@ * limitations under the License. */ -package controllers +package uk.gov.hmrc.upscaninitiate.controller -import config.ServiceConfiguration -import controllers.PrepareUploadControllerSpec.{ConsumingService, UserAgent, requestTemplate, settingsTemplate} -import controllers.model.{PrepareUploadRequest, PreparedUploadResponse, Reference, UploadFormTemplate} import org.apache.pekko.actor.ActorSystem import org.scalatest.GivenWhenThen import play.api.http.HeaderNames.USER_AGENT @@ -28,9 +25,12 @@ import play.api.mvc.Action import play.api.mvc.Results.Ok import play.api.test.Helpers.{contentAsJson, contentAsString, status} import play.api.test.{FakeRequest, Helpers, StubControllerComponentsFactory} -import services.PrepareUploadService -import services.model.UploadSettings -import test.UnitSpec +import uk.gov.hmrc.upscaninitiate.config.ServiceConfiguration +import uk.gov.hmrc.upscaninitiate.controller.PrepareUploadControllerSpec.{ConsumingService, UserAgent, requestTemplate, settingsTemplate} +import uk.gov.hmrc.upscaninitiate.controller.model.{PrepareUploadRequest, PreparedUploadResponse, Reference, UploadFormTemplate} +import uk.gov.hmrc.upscaninitiate.service.PrepareUploadService +import uk.gov.hmrc.upscaninitiate.service.model.UploadSettings +import uk.gov.hmrc.upscaninitiate.test.UnitSpec import java.time.Clock import scala.concurrent.Future diff --git a/test/controllers/model/PrepareUploadRequestSpec.scala b/test/uk/gov/hmrc/upscaninitiate/controller/model/PrepareUploadRequestSpec.scala similarity index 95% rename from test/controllers/model/PrepareUploadRequestSpec.scala rename to test/uk/gov/hmrc/upscaninitiate/controller/model/PrepareUploadRequestSpec.scala index a075425..545617d 100644 --- a/test/controllers/model/PrepareUploadRequestSpec.scala +++ b/test/uk/gov/hmrc/upscaninitiate/controller/model/PrepareUploadRequestSpec.scala @@ -14,11 +14,11 @@ * limitations under the License. */ -package controllers.model +package uk.gov.hmrc.upscaninitiate.controller.model -import controllers.model.PrepareUploadRequestSpec._ import play.api.libs.json.{JsSuccess, Json, Reads} -import test.UnitSpec +import uk.gov.hmrc.upscaninitiate.controller.model.PrepareUploadRequestSpec._ +import uk.gov.hmrc.upscaninitiate.test.UnitSpec class PrepareUploadRequestSpec extends UnitSpec { diff --git a/test/model/PrepareUploadServiceSpec.scala b/test/uk/gov/hmrc/upscaninitiate/model/PrepareUploadServiceSpec.scala similarity index 95% rename from test/model/PrepareUploadServiceSpec.scala rename to test/uk/gov/hmrc/upscaninitiate/model/PrepareUploadServiceSpec.scala index bb2c5e3..66879d5 100644 --- a/test/model/PrepareUploadServiceSpec.scala +++ b/test/uk/gov/hmrc/upscaninitiate/model/PrepareUploadServiceSpec.scala @@ -14,20 +14,21 @@ * limitations under the License. */ -package model +package uk.gov.hmrc.upscaninitiate.model -import java.time -import java.time.Instant import com.codahale.metrics.MetricRegistry -import config.ServiceConfiguration -import connectors.model.{UploadFormGenerator, UploadParameters} -import controllers.model.PrepareUploadRequest -import model.PrepareUploadServiceSpec.{requestTemplate, settingsTemplate} import org.scalatest.GivenWhenThen -import services.PrepareUploadService -import services.model.UploadSettings -import test.UnitSpec import uk.gov.hmrc.play.bootstrap.metrics.Metrics +import uk.gov.hmrc.upscaninitiate.config.ServiceConfiguration +import uk.gov.hmrc.upscaninitiate.connector.model.{UploadFormGenerator, UploadParameters} +import uk.gov.hmrc.upscaninitiate.controller.model.PrepareUploadRequest +import uk.gov.hmrc.upscaninitiate.model.PrepareUploadServiceSpec.{requestTemplate, settingsTemplate} +import uk.gov.hmrc.upscaninitiate.service.PrepareUploadService +import uk.gov.hmrc.upscaninitiate.service.model.UploadSettings +import uk.gov.hmrc.upscaninitiate.test.UnitSpec + +import java.time +import java.time.Instant class PrepareUploadServiceSpec extends UnitSpec with GivenWhenThen { diff --git a/test/test/UnitSpec.scala b/test/uk/gov/hmrc/upscaninitiate/test/UnitSpec.scala similarity index 95% rename from test/test/UnitSpec.scala rename to test/uk/gov/hmrc/upscaninitiate/test/UnitSpec.scala index 2c8d814..bbd8fa0 100644 --- a/test/test/UnitSpec.scala +++ b/test/uk/gov/hmrc/upscaninitiate/test/UnitSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package test +package uk.gov.hmrc.upscaninitiate.test import org.mockito.scalatest.MockitoSugar import org.scalatest.matchers.should diff --git a/test/utils/UserAgentFilterSpec.scala b/test/uk/gov/hmrc/upscaninitiate/util/UserAgentFilterSpec.scala similarity index 96% rename from test/utils/UserAgentFilterSpec.scala rename to test/uk/gov/hmrc/upscaninitiate/util/UserAgentFilterSpec.scala index 7ee88e3..4340bac 100644 --- a/test/utils/UserAgentFilterSpec.scala +++ b/test/uk/gov/hmrc/upscaninitiate/util/UserAgentFilterSpec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package utils +package uk.gov.hmrc.upscaninitiate.util import org.apache.pekko.util.Timeout import org.scalatest.GivenWhenThen @@ -25,7 +25,7 @@ import play.api.mvc.{Request, Result} import play.api.test.FakeRequest import play.api.test.Helpers.{contentAsString, status} import play.mvc.Http.HeaderNames.USER_AGENT -import test.UnitSpec +import uk.gov.hmrc.upscaninitiate.test.UnitSpec import scala.concurrent.Future import scala.concurrent.duration._ From 16387c56b3ec17ef4e8613111b3a831caa7137f4 Mon Sep 17 00:00:00 2001 From: colin-lamed <9568290+colin-lamed@users.noreply.github.com> Date: Wed, 23 Oct 2024 16:10:44 +0100 Subject: [PATCH 82/96] BDOG-3255 Scala 3 --- .../controller/model/PrepareUploadRequest.scala | 1 - .../controller/model/PreparedUploadResponse.scala | 6 +++--- .../controller/model/UploadFormTemplate.scala | 5 +++-- build.sbt | 8 +++----- project/AppDependencies.scala | 9 ++++----- project/build.properties | 2 +- project/plugins.sbt | 4 ++-- .../connector/s3/S3UploadFormGeneratorSpec.scala | 2 ++ .../controller/PrepareUploadControllerSpec.scala | 7 ++++--- test/uk/gov/hmrc/upscaninitiate/test/UnitSpec.scala | 2 +- 10 files changed, 23 insertions(+), 23 deletions(-) diff --git a/app/uk/gov/hmrc/upscaninitiate/controller/model/PrepareUploadRequest.scala b/app/uk/gov/hmrc/upscaninitiate/controller/model/PrepareUploadRequest.scala index e7c1404..f5224a7 100644 --- a/app/uk/gov/hmrc/upscaninitiate/controller/model/PrepareUploadRequest.scala +++ b/app/uk/gov/hmrc/upscaninitiate/controller/model/PrepareUploadRequest.scala @@ -16,7 +16,6 @@ package uk.gov.hmrc.upscaninitiate.controller.model -import play.api.libs.json.Reads import play.api.libs.functional.syntax._ import play.api.libs.json._ import play.api.libs.json.Reads.{max, min} diff --git a/app/uk/gov/hmrc/upscaninitiate/controller/model/PreparedUploadResponse.scala b/app/uk/gov/hmrc/upscaninitiate/controller/model/PreparedUploadResponse.scala index 0c6c508..a0e57db 100644 --- a/app/uk/gov/hmrc/upscaninitiate/controller/model/PreparedUploadResponse.scala +++ b/app/uk/gov/hmrc/upscaninitiate/controller/model/PreparedUploadResponse.scala @@ -24,7 +24,7 @@ case class PreparedUploadResponse(reference: Reference, uploadRequest: UploadFor object PreparedUploadResponse { val writes: OWrites[PreparedUploadResponse] = - ((__ \ "reference").write(Reference.writes) - ~ (__ \ "uploadRequest").write(UploadFormTemplate.writes))(unlift(PreparedUploadResponse.unapply)) - + ( (__ \ "reference" ).write(Reference.writes) + ~ (__ \ "uploadRequest").write(UploadFormTemplate.writes) + )(pur => Tuple.fromProductTyped(pur)) } diff --git a/app/uk/gov/hmrc/upscaninitiate/controller/model/UploadFormTemplate.scala b/app/uk/gov/hmrc/upscaninitiate/controller/model/UploadFormTemplate.scala index ddc3e4f..00276f7 100644 --- a/app/uk/gov/hmrc/upscaninitiate/controller/model/UploadFormTemplate.scala +++ b/app/uk/gov/hmrc/upscaninitiate/controller/model/UploadFormTemplate.scala @@ -24,7 +24,8 @@ case class UploadFormTemplate(href: String, fields: Map[String, String]) object UploadFormTemplate { val writes: OWrites[UploadFormTemplate] = - ((__ \ "href").write[String] - ~ (__ \ "fields").write[Map[String, String]])(unlift(UploadFormTemplate.unapply)) + ( (__ \ "href" ).write[String] + ~ (__ \ "fields").write[Map[String, String]] + )(uft => Tuple.fromProductTyped(uft)) } diff --git a/build.sbt b/build.sbt index 8b0d250..3d0f609 100644 --- a/build.sbt +++ b/build.sbt @@ -9,7 +9,8 @@ import uk.gov.hmrc.sbtdistributables.SbtDistributablesPlugin val appName = "upscan-initiate" ThisBuild / majorVersion := 0 -ThisBuild / scalaVersion := "2.13.12" +ThisBuild / scalaVersion := "3.3.4" +ThisBuild / scalacOptions += "-Wconf:msg=Flag.*repeatedly:s" lazy val scoverageSettings = Seq( // Semicolon-separated list of regexs matching classes to exclude @@ -26,10 +27,7 @@ lazy val microservice = Project(appName, file(".")) .enablePlugins(play.sbt.PlayScala, SbtAutoBuildPlugin, SbtDistributablesPlugin) .disablePlugins(JUnitXmlReportPlugin) //Required to prevent https://github.com/scalatest/scalatest/issues/1427 .settings(scoverageSettings: _*) - .settings(scalacOptions ++= Seq( - "-Wconf:cat=unused-imports&src=.*routes.*:s" //silence import warnings in routes generated by comments - , "-Wconf:cat=unused&src=.*routes.*:s" //silence private val defaultPrefix in class Routes is never used - )) + .settings(scalacOptions += "-Wconf:src=routes/.*:s") .settings(playDefaultPort := 9571) .settings(libraryDependencies ++= AppDependencies()) .settings(resolvers += Resolver.jcenterRepo) diff --git a/project/AppDependencies.scala b/project/AppDependencies.scala index dc4f219..16e430b 100644 --- a/project/AppDependencies.scala +++ b/project/AppDependencies.scala @@ -1,17 +1,16 @@ import sbt._ object AppDependencies { - private val bootstrapVersion = "8.5.0" + private val bootstrapVersion = "9.5.0" private val compile = Seq( "uk.gov.hmrc" %% "bootstrap-backend-play-30" % bootstrapVersion, - "com.amazonaws" % "aws-java-sdk-s3" % "1.12.606", - "org.apache.commons" % "commons-lang3" % "3.12.0" + "com.amazonaws" % "aws-java-sdk-s3" % "1.12.606", + "org.apache.commons" % "commons-lang3" % "3.12.0" ) private val test = Seq( - "uk.gov.hmrc" %% "bootstrap-test-play-30" % bootstrapVersion % Test, - "org.mockito" %% "mockito-scala-scalatest" % "1.17.29" % Test + "uk.gov.hmrc" %% "bootstrap-test-play-30" % bootstrapVersion % Test ) def apply(): Seq[ModuleID] = compile ++ test diff --git a/project/build.properties b/project/build.properties index e8a1e24..04267b1 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1 +1 @@ -sbt.version=1.9.7 +sbt.version=1.9.9 diff --git a/project/plugins.sbt b/project/plugins.sbt index cbc58ca..d0f92be 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -4,8 +4,8 @@ resolvers += MavenRepository("HMRC-open-artefacts-maven2", "https://open.artefac resolvers += Resolver.url("HMRC-open-artefacts-ivy2", url("https://open.artefacts.tax.service.gov.uk/ivy2"))( Resolver.ivyStylePatterns) -addSbtPlugin("org.playframework" % "sbt-plugin" % "3.0.2") -addSbtPlugin("uk.gov.hmrc" % "sbt-auto-build" % "3.21.0") +addSbtPlugin("org.playframework" % "sbt-plugin" % "3.0.5") +addSbtPlugin("uk.gov.hmrc" % "sbt-auto-build" % "3.22.0") addSbtPlugin("uk.gov.hmrc" % "sbt-distributables" % "2.5.0") addSbtPlugin("org.scoverage" % "sbt-scoverage" % "2.0.11") addSbtPlugin("org.scalastyle" %% "scalastyle-sbt-plugin" % "1.0.0" exclude("org.scala-lang.modules", "scala-xml_2.12")) diff --git a/test/uk/gov/hmrc/upscaninitiate/connector/s3/S3UploadFormGeneratorSpec.scala b/test/uk/gov/hmrc/upscaninitiate/connector/s3/S3UploadFormGeneratorSpec.scala index f1e962c..e7989c8 100644 --- a/test/uk/gov/hmrc/upscaninitiate/connector/s3/S3UploadFormGeneratorSpec.scala +++ b/test/uk/gov/hmrc/upscaninitiate/connector/s3/S3UploadFormGeneratorSpec.scala @@ -16,6 +16,8 @@ package uk.gov.hmrc.upscaninitiate.connector.s3 +import org.mockito.ArgumentMatchers.any +import org.mockito.Mockito.{verify, when} import org.scalatest.GivenWhenThen import play.api.libs.json.{JsArray, JsValue, Json} import uk.gov.hmrc.upscaninitiate.connector.model.{AwsCredentials, ContentLengthRange, UploadParameters} diff --git a/test/uk/gov/hmrc/upscaninitiate/controller/PrepareUploadControllerSpec.scala b/test/uk/gov/hmrc/upscaninitiate/controller/PrepareUploadControllerSpec.scala index cca30e9..48a14bd 100644 --- a/test/uk/gov/hmrc/upscaninitiate/controller/PrepareUploadControllerSpec.scala +++ b/test/uk/gov/hmrc/upscaninitiate/controller/PrepareUploadControllerSpec.scala @@ -17,6 +17,7 @@ package uk.gov.hmrc.upscaninitiate.controller import org.apache.pekko.actor.ActorSystem +import org.mockito.Mockito.when import org.scalatest.GivenWhenThen import play.api.http.HeaderNames.USER_AGENT import play.api.http.Status.{BAD_REQUEST, OK} @@ -46,9 +47,9 @@ class PrepareUploadControllerSpec extends UnitSpec with StubControllerComponents private val clock = Clock.fixed(Clock.systemDefaultZone().instant(), Clock.systemDefaultZone().getZone) private trait WithGlobalFileSizeLimitFixture { - val GlobalFileSizeLimit = 1024 + val globalFileSizeLimit = 1024L val prepareUploadService = mock[PrepareUploadService] - when(prepareUploadService.globalFileSizeLimit).thenReturn(GlobalFileSizeLimit) + when(prepareUploadService.globalFileSizeLimit).thenReturn(globalFileSizeLimit) } private trait WithServiceConfiguration { @@ -289,7 +290,7 @@ class PrepareUploadControllerSpec extends UnitSpec with StubControllerComponents ("x-session-id", "some-session-id") ).withBody(Json.obj( "callbackUrl" -> "https://www.example.com", - "maximumFileSize" -> (GlobalFileSizeLimit + 1)) + "maximumFileSize" -> (globalFileSizeLimit + 1)) ) When("upload initiation has been requested") diff --git a/test/uk/gov/hmrc/upscaninitiate/test/UnitSpec.scala b/test/uk/gov/hmrc/upscaninitiate/test/UnitSpec.scala index bbd8fa0..3602ea0 100644 --- a/test/uk/gov/hmrc/upscaninitiate/test/UnitSpec.scala +++ b/test/uk/gov/hmrc/upscaninitiate/test/UnitSpec.scala @@ -16,8 +16,8 @@ package uk.gov.hmrc.upscaninitiate.test -import org.mockito.scalatest.MockitoSugar import org.scalatest.matchers.should import org.scalatest.wordspec.AnyWordSpecLike +import org.scalatestplus.mockito.MockitoSugar trait UnitSpec extends AnyWordSpecLike with should.Matchers with MockitoSugar From e2bf53c782ac61d9d1a1b4b3839bee14163e4f19 Mon Sep 17 00:00:00 2001 From: colin-lamed <9568290+colin-lamed@users.noreply.github.com> Date: Wed, 23 Oct 2024 17:32:51 +0100 Subject: [PATCH 83/96] Scala 3 syntax --- .../upscaninitiate/UpscanInitiateModule.scala | 2 +- .../config/ServiceConfiguration.scala | 69 +++-- .../connector/model/AwsCredentials.scala | 6 +- .../connector/model/ContentLengthRange.scala | 5 +- .../connector/model/UploadFormGenerator.scala | 3 +- .../connector/model/UploadParameters.scala | 10 +- .../connector/s3/PolicySigner.scala | 39 +-- .../connector/s3/S3Module.scala | 3 +- .../connector/s3/S3UploadFormGenerator.scala | 107 +++---- .../s3/S3UploadFormGeneratorProvider.scala | 15 +- .../controller/PrepareUploadController.scala | 84 +++--- .../model/PrepareUploadRequest.scala | 17 +- .../model/PreparedUploadResponse.scala | 8 +- .../controller/model/Reference.scala | 16 +- .../controller/model/UploadFormTemplate.scala | 9 +- .../service/PrepareUploadService.scala | 77 ++--- .../service/model/UploadSettings.scala | 9 +- .../upscaninitiate/util/UserAgentFilter.scala | 12 +- .../PrepareUploadControllerISpec.scala | 166 +++++------ .../connector/s3/PolicySignerSpec.scala | 13 +- .../s3/S3UploadFormGeneratorSpec.scala | 64 ++-- .../PrepareUploadControllerSpec.scala | 276 +++++++++--------- .../model/PrepareUploadRequestSpec.scala | 33 +-- .../model/PrepareUploadServiceSpec.scala | 104 +++---- .../hmrc/upscaninitiate/test/UnitSpec.scala | 5 +- .../util/UserAgentFilterSpec.scala | 31 +- 26 files changed, 554 insertions(+), 629 deletions(-) diff --git a/app/uk/gov/hmrc/upscaninitiate/UpscanInitiateModule.scala b/app/uk/gov/hmrc/upscaninitiate/UpscanInitiateModule.scala index 7311bfc..beeb1f2 100644 --- a/app/uk/gov/hmrc/upscaninitiate/UpscanInitiateModule.scala +++ b/app/uk/gov/hmrc/upscaninitiate/UpscanInitiateModule.scala @@ -26,6 +26,6 @@ class UpscanInitiateModule extends Module { override def bindings(environment: Environment, configuration: Configuration): Seq[Binding[_]] = Seq( bind[ServiceConfiguration].to[PlayBasedServiceConfiguration], - bind[Clock].toInstance(Clock.systemDefaultZone()) + bind[Clock ].toInstance(Clock.systemDefaultZone()) ) } diff --git a/app/uk/gov/hmrc/upscaninitiate/config/ServiceConfiguration.scala b/app/uk/gov/hmrc/upscaninitiate/config/ServiceConfiguration.scala index b09b7c6..4c05471 100644 --- a/app/uk/gov/hmrc/upscaninitiate/config/ServiceConfiguration.scala +++ b/app/uk/gov/hmrc/upscaninitiate/config/ServiceConfiguration.scala @@ -16,54 +16,51 @@ package uk.gov.hmrc.upscaninitiate.config -import com.typesafe.config.ConfigException -import org.apache.commons.lang3.StringUtils.isNotBlank import play.api.Configuration -import java.time.Duration import javax.inject.Inject - -trait ServiceConfiguration { - - def region: String - def uploadProxyUrl: String - def inboundBucketName: String - def sessionToken: Option[String] - def accessKeyId: String - def secretAccessKey: String - def fileExpirationPeriod: java.time.Duration - def globalFileSizeLimit: Long +import scala.concurrent.duration.FiniteDuration + +trait ServiceConfiguration: + def region : String + def uploadProxyUrl : String + def inboundBucketName : String + def sessionToken : Option[String] + def accessKeyId : String + def secretAccessKey : String + def fileExpirationPeriod : FiniteDuration + def globalFileSizeLimit : Long def allowedCallbackProtocols: Seq[String] -} -class PlayBasedServiceConfiguration @Inject()(configuration: Configuration) extends ServiceConfiguration { +class PlayBasedServiceConfiguration @Inject()( + configuration: Configuration +) extends ServiceConfiguration: - override val region: String = getRequired(configuration.getOptional[String](_), "aws.s3.region") + override val region: String = + configuration.get[String]("aws.s3.region") - override val uploadProxyUrl: String = getRequired(configuration.getOptional[String](_), "uploadProxy.url") + override val uploadProxyUrl: String = + configuration.get[String]("uploadProxy.url") - override val inboundBucketName: String = getRequired(configuration.getOptional[String](_), "aws.s3.bucket.inbound") + override val inboundBucketName: String = + configuration.get[String]("aws.s3.bucket.inbound") - override val fileExpirationPeriod: Duration = { - val readAsMillis: String => Option[Long] = configuration.getOptional[scala.concurrent.duration.Duration](_).map(_.toMillis) - Duration.ofMillis(getRequired(readAsMillis, "aws.s3.upload.link.validity.duration")) - } + override val fileExpirationPeriod: FiniteDuration = + configuration.get[FiniteDuration]("aws.s3.upload.link.validity.duration") - override val accessKeyId: String = getRequired(configuration.getOptional[String](_), "aws.accessKeyId") + override val accessKeyId: String = + configuration.get[String]("aws.accessKeyId") - override val secretAccessKey: String = getRequired(configuration.getOptional[String](_), "aws.secretAccessKey") + override val secretAccessKey: String = + configuration.get[String]("aws.secretAccessKey") - override val sessionToken: Option[String] = configuration.getOptional[String]("aws.sessionToken") + override val sessionToken: Option[String] = + configuration.getOptional[String]("aws.sessionToken") - override val globalFileSizeLimit: Long = getRequired(configuration.getOptional[Long](_), "global.file.size.limit") + override val globalFileSizeLimit: Long = + configuration.get[Long]("global.file.size.limit") override val allowedCallbackProtocols: Seq[String] = - commaSeparatedList(configuration.getOptional[String]("callbackValidation.allowedProtocols")) - - private def getRequired[T](read: String => Option[T], path: String): T = - read(path).getOrElse(throw new ConfigException.Missing(path)) - - private def commaSeparatedList(maybeString: Option[String]): List[String] = - maybeString.fold(List.empty[String])(_.split(",").toList.filter(isNotBlank)) - -} + configuration + .get[String]("callbackValidation.allowedProtocols") + .split(",").toList.map(_.trim).filter(_.nonEmpty) diff --git a/app/uk/gov/hmrc/upscaninitiate/connector/model/AwsCredentials.scala b/app/uk/gov/hmrc/upscaninitiate/connector/model/AwsCredentials.scala index ad3612a..cf5e3af 100644 --- a/app/uk/gov/hmrc/upscaninitiate/connector/model/AwsCredentials.scala +++ b/app/uk/gov/hmrc/upscaninitiate/connector/model/AwsCredentials.scala @@ -16,4 +16,8 @@ package uk.gov.hmrc.upscaninitiate.connector.model -final case class AwsCredentials(accessKeyId: String, secretKey: String, sessionToken: Option[String]) +case class AwsCredentials( + accessKeyId : String, + secretKey : String, + sessionToken: Option[String] +) diff --git a/app/uk/gov/hmrc/upscaninitiate/connector/model/ContentLengthRange.scala b/app/uk/gov/hmrc/upscaninitiate/connector/model/ContentLengthRange.scala index 8cc3012..01004fa 100644 --- a/app/uk/gov/hmrc/upscaninitiate/connector/model/ContentLengthRange.scala +++ b/app/uk/gov/hmrc/upscaninitiate/connector/model/ContentLengthRange.scala @@ -16,4 +16,7 @@ package uk.gov.hmrc.upscaninitiate.connector.model -case class ContentLengthRange(min: Long, max: Long) +case class ContentLengthRange( + min: Long, + max: Long +) diff --git a/app/uk/gov/hmrc/upscaninitiate/connector/model/UploadFormGenerator.scala b/app/uk/gov/hmrc/upscaninitiate/connector/model/UploadFormGenerator.scala index 6fe9cd9..4710807 100644 --- a/app/uk/gov/hmrc/upscaninitiate/connector/model/UploadFormGenerator.scala +++ b/app/uk/gov/hmrc/upscaninitiate/connector/model/UploadFormGenerator.scala @@ -16,6 +16,5 @@ package uk.gov.hmrc.upscaninitiate.connector.model -trait UploadFormGenerator { +trait UploadFormGenerator: def generateFormFields(uploadParameters: UploadParameters): Map[String, String] -} diff --git a/app/uk/gov/hmrc/upscaninitiate/connector/model/UploadParameters.scala b/app/uk/gov/hmrc/upscaninitiate/connector/model/UploadParameters.scala index 0ef4f71..140680c 100644 --- a/app/uk/gov/hmrc/upscaninitiate/connector/model/UploadParameters.scala +++ b/app/uk/gov/hmrc/upscaninitiate/connector/model/UploadParameters.scala @@ -20,11 +20,11 @@ import java.time.Instant case class UploadParameters( expirationDateTime: Instant, - bucketName: String, - objectKey: String, - acl: String, + bucketName : String, + objectKey : String, + acl : String, additionalMetadata: Map[String, String], contentLengthRange: ContentLengthRange, - successRedirect: Option[String], - errorRedirect: Option[String] + successRedirect : Option[String], + errorRedirect : Option[String] ) diff --git a/app/uk/gov/hmrc/upscaninitiate/connector/s3/PolicySigner.scala b/app/uk/gov/hmrc/upscaninitiate/connector/s3/PolicySigner.scala index 5c49d08..27d64a7 100644 --- a/app/uk/gov/hmrc/upscaninitiate/connector/s3/PolicySigner.scala +++ b/app/uk/gov/hmrc/upscaninitiate/connector/s3/PolicySigner.scala @@ -23,40 +23,41 @@ import java.nio.charset.Charset import javax.crypto.Mac import javax.crypto.spec.SecretKeySpec -trait PolicySigner { +trait PolicySigner: def signPolicy( - credentials: AwsCredentials, + credentials : AwsCredentials, formattedSigningDate: String, - regionName: String, - encodedPolicy: String): String -} + regionName : String, + encodedPolicy : String + ): String -object PolicySigner extends PolicySigner { +object PolicySigner extends PolicySigner: def signPolicy( - credentials: AwsCredentials, + credentials : AwsCredentials, formattedSigningDate: String, - regionName: String, - encodedPolicy: String): String = { + regionName : String, + encodedPolicy : String + ): String = val signingKey = newSigningKey(credentials, formattedSigningDate, regionName, "s3") val policySignature = sign(encodedPolicy, signingKey) BinaryUtils.toHex(policySignature) - } - private def newSigningKey(credentials: AwsCredentials, dateStamp: String, regionName: String, serviceName: String) = { + private def newSigningKey( + credentials: AwsCredentials, + dateStamp : String, + regionName : String, + serviceName: String + ) = val kSecret = ("AWS4" + credentials.secretKey).getBytes(Charset.forName("UTF-8")) - val kDate = sign(dateStamp, kSecret) - val kRegion = sign(regionName, kDate) + val kDate = sign(dateStamp , kSecret) + val kRegion = sign(regionName , kDate) val kService = sign(serviceName, kRegion) sign("aws4_request", kService) - } - private def sign(stringData: String, key: Array[Byte]): Array[Byte] = { + private def sign(stringData: String, key: Array[Byte]): Array[Byte] = val data = stringData.getBytes(StringUtils.UTF8) val algorithm = "HmacSHA256" val mac = Mac.getInstance(algorithm) - mac.init(new SecretKeySpec(key, algorithm)) + mac.init(SecretKeySpec(key, algorithm)) mac.doFinal(data) - } - -} diff --git a/app/uk/gov/hmrc/upscaninitiate/connector/s3/S3Module.scala b/app/uk/gov/hmrc/upscaninitiate/connector/s3/S3Module.scala index 0a49a6e..94a9cc5 100644 --- a/app/uk/gov/hmrc/upscaninitiate/connector/s3/S3Module.scala +++ b/app/uk/gov/hmrc/upscaninitiate/connector/s3/S3Module.scala @@ -20,9 +20,8 @@ import play.api.inject.{Binding, Module} import play.api.{Configuration, Environment} import uk.gov.hmrc.upscaninitiate.connector.model.UploadFormGenerator -class S3Module extends Module { +class S3Module extends Module: override def bindings(environment: Environment, configuration: Configuration): Seq[Binding[_]] = Seq( bind[UploadFormGenerator].toProvider[S3UploadFormGeneratorProvider] ) -} diff --git a/app/uk/gov/hmrc/upscaninitiate/connector/s3/S3UploadFormGenerator.scala b/app/uk/gov/hmrc/upscaninitiate/connector/s3/S3UploadFormGenerator.scala index ad30b65..8cafd22 100644 --- a/app/uk/gov/hmrc/upscaninitiate/connector/s3/S3UploadFormGenerator.scala +++ b/app/uk/gov/hmrc/upscaninitiate/connector/s3/S3UploadFormGenerator.scala @@ -24,13 +24,13 @@ import java.time.format.DateTimeFormatter import java.time.format.DateTimeFormatter.ISO_INSTANT class S3UploadFormGenerator( - credentials: AwsCredentials, - regionName: String, - currentTime: () => Instant, - policySigner: PolicySigner = PolicySigner) - extends UploadFormGenerator { + credentials : AwsCredentials, + regionName : String, + currentTime : () => Instant, + policySigner: PolicySigner = PolicySigner +) extends UploadFormGenerator: - def generateFormFields(uploadParameters: UploadParameters): Map[String, String] = { + def generateFormFields(uploadParameters: UploadParameters): Map[String, String] = val timestamp = currentTime() val formattedSigningDate = awsDate(timestamp) val signingCredentials = s"${credentials.accessKeyId}/$formattedSigningDate/$regionName/s3/aws4_request" @@ -47,7 +47,6 @@ class S3UploadFormGenerator( encodedPolicy, policySignature ) - } private def awsTimestamp(i: Instant): String = DateTimeFormatter.ofPattern("yyyyMMdd'T'HHmmss'Z'").withZone(ZoneOffset.UTC).format(i) @@ -55,18 +54,18 @@ class S3UploadFormGenerator( private def awsDate(i: Instant): String = DateTimeFormatter.ofPattern("yyyyMMdd").withZone(ZoneOffset.UTC).format(i) - private def base64encode(input: String): String = { + private def base64encode(input: String): String = val encodedBytes = java.util.Base64.getEncoder.encode(input.getBytes("UTF-8")) - new String(encodedBytes).replaceAll("\n", "").replaceAll("\r", "") - } + String(encodedBytes).replaceAll("\n", "").replaceAll("\r", "") private def buildFormFields( - uploadParameters: UploadParameters, - securityToken: Option[String], - timeStamp: String, + uploadParameters : UploadParameters, + securityToken : Option[String], + timeStamp : String, signingCredentials: String, - encodedPolicy: String, - policySignature: String): Map[String, String] = { + encodedPolicy : String, + policySignature : String + ): Map[String, String] = val fields = Map( "x-amz-algorithm" -> "AWS4-HMAC-SHA256", @@ -81,33 +80,40 @@ class S3UploadFormGenerator( ) val metadataFields = - uploadParameters.additionalMetadata.map { + uploadParameters.additionalMetadata.map: case (metadataKey, value) => s"x-amz-meta-$metadataKey" -> value - } - val sessionCredentials = securityToken.map(v => Map("x-amz-security-token" -> v)).getOrElse(Map.empty) + val sessionCredentials = + securityToken.map(v => Map("x-amz-security-token" -> v)).getOrElse(Map.empty) + val successRedirect = uploadParameters.successRedirect.map(v => Map("success_action_redirect" -> v)).getOrElse(Map.empty) val errorRedirect = uploadParameters.errorRedirect.map(v => Map("error_action_redirect" -> v)).getOrElse(Map.empty) - fields ++ metadataFields ++ sessionCredentials ++ successRedirect ++ errorRedirect - } + fields + ++ metadataFields + ++ sessionCredentials + ++ successRedirect + ++ errorRedirect private def buildPolicy( - uploadParameters: UploadParameters, - securityToken: Option[String], - timeStamp: String, - signingCredentials: String) = { + uploadParameters : UploadParameters, + securityToken : Option[String], + timeStamp : String, + signingCredentials: String + ) = val securityTokenJson = securityToken.map(t => Json.obj("x-amz-security-token" -> t)).toList - val metadataJson: Seq[JsValue] = uploadParameters.additionalMetadata.map { - case (k, v) => Json.obj(s"x-amz-meta-$k" -> v) - }.toSeq :+ - Json.arr("starts-with", "$x-amz-meta-original-filename", "") :+ - Json.arr("starts-with", "$x-amz-meta-upscan-initiate-response", "") + val metadataJson: Seq[JsValue] = + uploadParameters.additionalMetadata + .map: (k, v) => + Json.obj(s"x-amz-meta-$k" -> v) + .toSeq + :+ Json.arr("starts-with", "$x-amz-meta-original-filename" , "") + :+ Json.arr("starts-with", "$x-amz-meta-upscan-initiate-response", "") val successRedirectConstraint = uploadParameters.successRedirect.map(redirect => Json.obj("success_action_redirect" -> redirect)) @@ -115,28 +121,27 @@ class S3UploadFormGenerator( val errorRedirectConstraint = uploadParameters.errorRedirect.map(redirect => Json.obj("error_action_redirect" -> redirect)) - val policyDocument = Json.obj( - "expiration" -> ISO_INSTANT.format(uploadParameters.expirationDateTime), - "conditions" -> JsArray( - List( - Json.obj("bucket" -> uploadParameters.bucketName), - Json.obj("acl" -> uploadParameters.acl), - Json.obj("x-amz-credential" -> signingCredentials), - Json.obj("x-amz-algorithm" -> "AWS4-HMAC-SHA256"), - Json.obj("key" -> uploadParameters.objectKey), - Json.obj("x-amz-date" -> timeStamp), - Json.arr( - "content-length-range", - uploadParameters.contentLengthRange.min, - uploadParameters.contentLengthRange.max) - ) ++ securityTokenJson - ++ metadataJson - ++ successRedirectConstraint - ++ errorRedirectConstraint + val policyDocument = + Json.obj( + "expiration" -> ISO_INSTANT.format(uploadParameters.expirationDateTime), + "conditions" -> JsArray( + List( + Json.obj("bucket" -> uploadParameters.bucketName), + Json.obj("acl" -> uploadParameters.acl), + Json.obj("x-amz-credential" -> signingCredentials), + Json.obj("x-amz-algorithm" -> "AWS4-HMAC-SHA256"), + Json.obj("key" -> uploadParameters.objectKey), + Json.obj("x-amz-date" -> timeStamp), + Json.arr( + "content-length-range", + uploadParameters.contentLengthRange.min, + uploadParameters.contentLengthRange.max + ) + ) ++ securityTokenJson + ++ metadataJson + ++ successRedirectConstraint + ++ errorRedirectConstraint + ) ) - ) Json.stringify(policyDocument) - } - -} diff --git a/app/uk/gov/hmrc/upscaninitiate/connector/s3/S3UploadFormGeneratorProvider.scala b/app/uk/gov/hmrc/upscaninitiate/connector/s3/S3UploadFormGeneratorProvider.scala index 1912c93..5a3c7d8 100644 --- a/app/uk/gov/hmrc/upscaninitiate/connector/s3/S3UploadFormGeneratorProvider.scala +++ b/app/uk/gov/hmrc/upscaninitiate/connector/s3/S3UploadFormGeneratorProvider.scala @@ -23,18 +23,15 @@ import java.time.{Clock, Instant} import javax.inject.{Inject, Provider, Singleton} @Singleton -class S3UploadFormGeneratorProvider @Inject()(configuration: ServiceConfiguration, clock: Clock) - extends Provider[UploadFormGenerator] { - +class S3UploadFormGeneratorProvider @Inject()( + configuration: ServiceConfiguration, + clock : Clock +) extends Provider[UploadFormGenerator]: import configuration._ - private val tick: () => Instant = () => Instant.now(clock) - override def get() = - new S3UploadFormGenerator( + S3UploadFormGenerator( AwsCredentials(accessKeyId, secretAccessKey, sessionToken), regionName = region, - currentTime = tick + currentTime = () => Instant.now(clock) ) - -} diff --git a/app/uk/gov/hmrc/upscaninitiate/controller/PrepareUploadController.scala b/app/uk/gov/hmrc/upscaninitiate/controller/PrepareUploadController.scala index cd2a03a..50e697c 100644 --- a/app/uk/gov/hmrc/upscaninitiate/controller/PrepareUploadController.scala +++ b/app/uk/gov/hmrc/upscaninitiate/controller/PrepareUploadController.scala @@ -18,7 +18,7 @@ package uk.gov.hmrc.upscaninitiate.controller import play.api.Logging import play.api.libs.json._ -import play.api.mvc.{Action, ControllerComponents, Result} +import play.api.mvc.{Action, ControllerComponents, Request, Result} import uk.gov.hmrc.play.bootstrap.backend.controller.BackendController import uk.gov.hmrc.upscaninitiate.config.ServiceConfiguration import uk.gov.hmrc.upscaninitiate.controller.model.{PrepareUploadRequest, PreparedUploadResponse} @@ -35,40 +35,40 @@ import scala.util.{Failure, Success, Try} @Singleton class PrepareUploadController @Inject()( prepareUploadService: PrepareUploadService, - configuration: ServiceConfiguration, - clock: Clock, - controllerComponents: ControllerComponents) extends BackendController(controllerComponents) with UserAgentFilter with Logging { - - val prepareUploadRequestReadsV1: Reads[PrepareUploadRequest] = - PrepareUploadRequest.readsV1(prepareUploadService.globalFileSizeLimit) - - val prepareUploadRequestReadsV2: Reads[PrepareUploadRequest] = - PrepareUploadRequest.readsV2(prepareUploadService.globalFileSizeLimit) + configuration : ServiceConfiguration, + clock : Clock, + controllerComponents: ControllerComponents +) extends BackendController(controllerComponents) + with UserAgentFilter + with Logging: /** * V1 of the API supports direct upload to an S3 bucket and *does not support* error redirects in the event of failure */ - def prepareUploadV1: Action[JsValue] = { - val uploadUrl = s"https://${configuration.inboundBucketName}.s3.amazonaws.com" - prepareUpload(uploadUrl)(prepareUploadRequestReadsV1) - } + def prepareUploadV1: Action[JsValue] = + given Reads[PrepareUploadRequest] = PrepareUploadRequest.readsV1(prepareUploadService.globalFileSizeLimit) + prepareUpload(uploadUrl = s"https://${configuration.inboundBucketName}.s3.amazonaws.com") /** * V2 of the API supports upload to an S3 bucket via a proxy that additionally supports error redirects in the event of failure */ - def prepareUploadV2: Action[JsValue] = { - val uploadUrl = s"${configuration.uploadProxyUrl}/v1/uploads/${configuration.inboundBucketName}" - prepareUpload(uploadUrl)(prepareUploadRequestReadsV2) - } - - private def prepareUpload(uploadUrl: String)( - implicit reads: Reads[PrepareUploadRequest]): Action[JsValue] = - Action.async(parse.json) { implicit request => + def prepareUploadV2: Action[JsValue] = + given Reads[PrepareUploadRequest] = PrepareUploadRequest.readsV2(prepareUploadService.globalFileSizeLimit) + prepareUpload(uploadUrl = s"${configuration.uploadProxyUrl}/v1/uploads/${configuration.inboundBucketName}") + + private def prepareUpload( + uploadUrl: String + )(using + Reads[PrepareUploadRequest] + ): Action[JsValue] = + Action.async(parse.json): request => + given Request[JsValue] = request + val receivedAt = Instant.now(clock) - requireUserAgent[JsValue] { (_, userAgent) => - withJsonBody[PrepareUploadRequest] { prepareUploadRequest => - withAllowedCallbackProtocol(prepareUploadRequest.callbackUrl) { + requireUserAgent[JsValue]: (_, userAgent) => + withJsonBody[PrepareUploadRequest]: prepareUploadRequest => + withAllowedCallbackProtocol(prepareUploadRequest.callbackUrl): val sessionId = hc(request).sessionId.map(_.value).getOrElse("n/a") val requestId = hc(request).requestId.map(_.value).getOrElse("n/a") @@ -91,32 +91,30 @@ class PrepareUploadController @Inject()( ) Future.successful(Ok(Json.toJson(result)(PreparedUploadResponse.writes))) - } - } - } - } - private[controller] def withAllowedCallbackProtocol[A](callbackUrl: String)( - block: => Future[Result]): Future[Result] = { + private[controller] def withAllowedCallbackProtocol[A]( + callbackUrl: String + )( + block: => Future[Result] + ): Future[Result] = val allowedCallbackProtocols: Seq[String] = configuration.allowedCallbackProtocols - val isAllowedCallbackProtocol: Try[Boolean] = Try { - allowedCallbackProtocols.contains(new URL(callbackUrl).getProtocol) - } + val isAllowedCallbackProtocol: Try[Boolean] = + Try(allowedCallbackProtocols.contains(URL(callbackUrl).getProtocol)) + + isAllowedCallbackProtocol match + case Success(true) => + block - isAllowedCallbackProtocol match { - case Success(true) => block case Success(false) => logger.warn(s"Invalid callback url protocol: [$callbackUrl].") - Future.successful(BadRequest( - s"Invalid callback url protocol: [$callbackUrl]. Protocol must be in: [${allowedCallbackProtocols.mkString(",")}].")) + s"Invalid callback url protocol: [$callbackUrl]. Protocol must be in: [${allowedCallbackProtocols.mkString(",")}]." + )) - case Failure(e) => + case Failure(e) => logger.warn(s"Invalid callback url format: [$callbackUrl].") - Future.successful(BadRequest(s"Invalid callback url format: [$callbackUrl]. [${e.getMessage}]")) - } - } -} + +end PrepareUploadController diff --git a/app/uk/gov/hmrc/upscaninitiate/controller/model/PrepareUploadRequest.scala b/app/uk/gov/hmrc/upscaninitiate/controller/model/PrepareUploadRequest.scala index f5224a7..9ddf69d 100644 --- a/app/uk/gov/hmrc/upscaninitiate/controller/model/PrepareUploadRequest.scala +++ b/app/uk/gov/hmrc/upscaninitiate/controller/model/PrepareUploadRequest.scala @@ -18,14 +18,13 @@ package uk.gov.hmrc.upscaninitiate.controller.model import play.api.libs.functional.syntax._ import play.api.libs.json._ -import play.api.libs.json.Reads.{max, min} -final case class PrepareUploadRequest( - callbackUrl: String, - minimumFileSize: Option[Long], - maximumFileSize: Option[Long], - successRedirect: Option[String], - errorRedirect: Option[String], +case class PrepareUploadRequest( + callbackUrl : String, + minimumFileSize : Option[Long], + maximumFileSize : Option[Long], + successRedirect : Option[String], + errorRedirect : Option[String], consumingService: Option[String] ) @@ -37,8 +36,8 @@ object PrepareUploadRequest { def readsV2(maxFileSize: Long): Reads[PrepareUploadRequest] = ( (__ \ "callbackUrl" ).read[String] - ~ (__ \ "minimumFileSize" ).readNullable[Long](min(0L)) - ~ (__ \ "maximumFileSize" ).readNullable[Long](min(0L) keepAnd max(maxFileSize)) + ~ (__ \ "minimumFileSize" ).readNullable[Long](Reads.min(0L)) + ~ (__ \ "maximumFileSize" ).readNullable[Long](Reads.min(0L) keepAnd Reads.max(maxFileSize)) ~ (__ \ "successRedirect" ).readNullable[String] ~ (__ \ "errorRedirect" ).readNullable[String] ~ (__ \ "consumingService").readNullable[String] diff --git a/app/uk/gov/hmrc/upscaninitiate/controller/model/PreparedUploadResponse.scala b/app/uk/gov/hmrc/upscaninitiate/controller/model/PreparedUploadResponse.scala index a0e57db..2175999 100644 --- a/app/uk/gov/hmrc/upscaninitiate/controller/model/PreparedUploadResponse.scala +++ b/app/uk/gov/hmrc/upscaninitiate/controller/model/PreparedUploadResponse.scala @@ -19,12 +19,14 @@ package uk.gov.hmrc.upscaninitiate.controller.model import play.api.libs.functional.syntax._ import play.api.libs.json.{OWrites, __} -case class PreparedUploadResponse(reference: Reference, uploadRequest: UploadFormTemplate) +case class PreparedUploadResponse( + reference : Reference, + uploadRequest: UploadFormTemplate +) -object PreparedUploadResponse { +object PreparedUploadResponse: val writes: OWrites[PreparedUploadResponse] = ( (__ \ "reference" ).write(Reference.writes) ~ (__ \ "uploadRequest").write(UploadFormTemplate.writes) )(pur => Tuple.fromProductTyped(pur)) -} diff --git a/app/uk/gov/hmrc/upscaninitiate/controller/model/Reference.scala b/app/uk/gov/hmrc/upscaninitiate/controller/model/Reference.scala index bc7d23a..1a34fd7 100644 --- a/app/uk/gov/hmrc/upscaninitiate/controller/model/Reference.scala +++ b/app/uk/gov/hmrc/upscaninitiate/controller/model/Reference.scala @@ -20,14 +20,14 @@ import play.api.libs.json.Writes import java.util.UUID.randomUUID -case class Reference(value: String) extends AnyVal { - override def toString: String = value.toString -} +case class Reference(value: String) extends AnyVal: + override def toString: String = + value -object Reference { +object Reference: - val writes: Writes[Reference] = Writes.of[String].contramap(_.value) + val writes: Writes[Reference] = + Writes.of[String].contramap(_.value) - def generate(): Reference = Reference(randomUUID().toString) - -} + def generate(): Reference = + Reference(randomUUID().toString) diff --git a/app/uk/gov/hmrc/upscaninitiate/controller/model/UploadFormTemplate.scala b/app/uk/gov/hmrc/upscaninitiate/controller/model/UploadFormTemplate.scala index 00276f7..6e4d660 100644 --- a/app/uk/gov/hmrc/upscaninitiate/controller/model/UploadFormTemplate.scala +++ b/app/uk/gov/hmrc/upscaninitiate/controller/model/UploadFormTemplate.scala @@ -19,13 +19,14 @@ package uk.gov.hmrc.upscaninitiate.controller.model import play.api.libs.functional.syntax._ import play.api.libs.json.{OWrites, __} -case class UploadFormTemplate(href: String, fields: Map[String, String]) +case class UploadFormTemplate( + href : String, + fields: Map[String, String] +) -object UploadFormTemplate { +object UploadFormTemplate: val writes: OWrites[UploadFormTemplate] = ( (__ \ "href" ).write[String] ~ (__ \ "fields").write[Map[String, String]] )(uft => Tuple.fromProductTyped(uft)) - -} diff --git a/app/uk/gov/hmrc/upscaninitiate/service/PrepareUploadService.scala b/app/uk/gov/hmrc/upscaninitiate/service/PrepareUploadService.scala index eb4dbcb..e6f2035 100644 --- a/app/uk/gov/hmrc/upscaninitiate/service/PrepareUploadService.scala +++ b/app/uk/gov/hmrc/upscaninitiate/service/PrepareUploadService.scala @@ -26,61 +26,63 @@ import uk.gov.hmrc.upscaninitiate.service.model.UploadSettings import java.time.Instant import javax.inject.{Inject, Singleton} +import scala.jdk.DurationConverters.ScalaDurationOps @Singleton class PrepareUploadService @Inject()( - postSigner: UploadFormGenerator, + postSigner : UploadFormGenerator, configuration: ServiceConfiguration, - metrics: Metrics) extends Logging { + metrics : Metrics +) extends Logging: def prepareUpload( - settings: UploadSettings, - requestId: String, - sessionId: String, - receivedAt: Instant): PreparedUploadResponse = { + settings : UploadSettings, + requestId : String, + sessionId : String, + receivedAt: Instant + ): PreparedUploadResponse = + val reference = Reference.generate() - val expiration = receivedAt.plus(configuration.fileExpirationPeriod) + val expiration = receivedAt.plus(configuration.fileExpirationPeriod.toJava) val result = PreparedUploadResponse( reference = reference, - uploadRequest = - generatePost( - key = reference, - expiration = expiration, - settings = settings, - requestId = requestId, - sessionId = sessionId, - receivedAt = receivedAt - ) + uploadRequest = generatePost( + key = reference, + expiration = expiration, + settings = settings, + requestId = requestId, + sessionId = sessionId, + receivedAt = receivedAt + ) ) - try { + try MDC.put("file-reference", reference.toString) logger.info(s"Allocated key=[${reference.value}] to uploadRequest with requestId=[$requestId] sessionId=[$sessionId] from [${settings.consumingService}].") logger.debug(s"Prepared upload response [$result].") metrics.defaultRegistry.counter("uploadInitiated").inc() result - } finally { + finally MDC.remove("file-reference") - } - } private def generatePost( - key: Reference, + key : Reference, expiration: Instant, - settings: UploadSettings, - requestId: String, - sessionId: String, - receivedAt: Instant): UploadFormTemplate = { + settings : UploadSettings, + requestId : String, + sessionId : String, + receivedAt: Instant + ): UploadFormTemplate = val minFileSize = settings.prepareUploadRequest.minimumFileSize.getOrElse(0L) val maxFileSize = settings.prepareUploadRequest.maximumFileSize.getOrElse(globalFileSizeLimit) - require(minFileSize >= 0, "Minimum file size is less than 0") + require(minFileSize >= 0 , "Minimum file size is less than 0") require(maxFileSize <= globalFileSizeLimit, "Maximum file size is greater than global maximum file size") - require(minFileSize <= maxFileSize, "Minimum file size is greater than maximum file size") + require(minFileSize <= maxFileSize , "Minimum file size is greater than maximum file size") val uploadParameters = UploadParameters( expirationDateTime = expiration, @@ -88,21 +90,22 @@ class PrepareUploadService @Inject()( objectKey = key.value, acl = "private", additionalMetadata = Map( - "callback-url" -> settings.prepareUploadRequest.callbackUrl, - "consuming-service" -> settings.consumingService, - "session-id" -> sessionId, - "request-id" -> requestId, - "upscan-initiate-received" -> receivedAt.toString - ), + "callback-url" -> settings.prepareUploadRequest.callbackUrl, + "consuming-service" -> settings.consumingService, + "session-id" -> sessionId, + "request-id" -> requestId, + "upscan-initiate-received" -> receivedAt.toString + ), contentLengthRange = ContentLengthRange(minFileSize, maxFileSize), successRedirect = settings.prepareUploadRequest.successRedirect, errorRedirect = settings.prepareUploadRequest.errorRedirect ) - val form = postSigner.generateFormFields(uploadParameters) + val form = postSigner.generateFormFields(uploadParameters) UploadFormTemplate(settings.uploadUrl, form) - } - def globalFileSizeLimit: Long = configuration.globalFileSizeLimit -} + def globalFileSizeLimit: Long = + configuration.globalFileSizeLimit + +end PrepareUploadService diff --git a/app/uk/gov/hmrc/upscaninitiate/service/model/UploadSettings.scala b/app/uk/gov/hmrc/upscaninitiate/service/model/UploadSettings.scala index 8245225..b9c36b4 100644 --- a/app/uk/gov/hmrc/upscaninitiate/service/model/UploadSettings.scala +++ b/app/uk/gov/hmrc/upscaninitiate/service/model/UploadSettings.scala @@ -18,12 +18,11 @@ package uk.gov.hmrc.upscaninitiate.service.model import uk.gov.hmrc.upscaninitiate.controller.model.PrepareUploadRequest -final case class UploadSettings( - uploadUrl: String, - userAgent: String, +case class UploadSettings( + uploadUrl : String, + userAgent : String, prepareUploadRequest: PrepareUploadRequest -) { +): lazy val consumingService: String = prepareUploadRequest.consumingService.getOrElse(userAgent) -} diff --git a/app/uk/gov/hmrc/upscaninitiate/util/UserAgentFilter.scala b/app/uk/gov/hmrc/upscaninitiate/util/UserAgentFilter.scala index 4a5eb61..9cc7dab 100644 --- a/app/uk/gov/hmrc/upscaninitiate/util/UserAgentFilter.scala +++ b/app/uk/gov/hmrc/upscaninitiate/util/UserAgentFilter.scala @@ -23,15 +23,15 @@ import play.api.mvc.{Request, Result} import scala.concurrent.Future -trait UserAgentFilter { this: Logging => +trait UserAgentFilter: + this: Logging => + /* * We require the user agent to be set with the name of the client service. */ - def requireUserAgent[A](block: (Request[A], String) => Future[Result])(implicit request: Request[A]): Future[Result] = - request.headers.get(USER_AGENT).fold(onMissingUserAgent())(block(request, _)) + def requireUserAgent[A](block: (Request[A], String) => Future[Result])(using request: Request[A]): Future[Result] = + request.headers.get(USER_AGENT).fold(onMissingUserAgent)(block(request, _)) - private def onMissingUserAgent(): Future[Result] = { + private def onMissingUserAgent: Future[Result] = logger.warn(s"No $USER_AGENT Request Header found - unable to identify client service") Future.successful(BadRequest(s"Missing $USER_AGENT Header")) - } -} diff --git a/it/test/uk/gov/hmrc/upscaninitiate/controller/PrepareUploadControllerISpec.scala b/it/test/uk/gov/hmrc/upscaninitiate/controller/PrepareUploadControllerISpec.scala index b5b28bc..6d87fd5 100644 --- a/it/test/uk/gov/hmrc/upscaninitiate/controller/PrepareUploadControllerISpec.scala +++ b/it/test/uk/gov/hmrc/upscaninitiate/controller/PrepareUploadControllerISpec.scala @@ -18,39 +18,44 @@ package uk.gov.hmrc.upscaninitiate.controller import org.scalatest.GivenWhenThen import org.scalatest.matchers.should -import org.scalatest.wordspec.AnyWordSpecLike +import org.scalatest.wordspec.AnyWordSpec import org.scalatestplus.play.guice.GuiceOneAppPerSuite import play.api.Application -import play.api.http.HeaderNames.USER_AGENT -import play.api.http.Status.BAD_REQUEST import play.api.inject.guice.GuiceApplicationBuilder import play.api.libs.json.{JsValue, Json} import play.api.test.Helpers._ import play.api.test.{FakeHeaders, FakeRequest} import uk.gov.hmrc.http.HeaderNames.xSessionId -class PrepareUploadControllerISpec extends AnyWordSpecLike with should.Matchers with GuiceOneAppPerSuite with GivenWhenThen { +class PrepareUploadControllerISpec + extends AnyWordSpec + with should.Matchers + with GuiceOneAppPerSuite + with GivenWhenThen: import PrepareUploadControllerISpec._ - override implicit lazy val app: Application = new GuiceApplicationBuilder() - .configure( - "uploadProxy.url" -> "https://upload-proxy.tax.service.gov.uk", - "aws.s3.bucket.inbound" -> "inbound-bucket" + override implicit lazy val app: Application = + GuiceApplicationBuilder() + .configure( + "uploadProxy.url" -> "https://upload-proxy.tax.service.gov.uk", + "aws.s3.bucket.inbound" -> "inbound-bucket" + ) + .build() + + "PrepareUploadController prepareUploadV1 with all request values" in: + val postBodyJson = + Json.parse(s"""{ + | "callbackUrl" : "https://some-url/callback", + | "minimumFileSize" : 0, + | "maximumFileSize" : 1024, + | "successRedirect" : "https://some-url/success", + | "consumingService": "$someOtherConsumingService" + |}""".stripMargin ) - .build() - - "PrepareUploadController prepareUploadV1 with all request values" in { - val postBodyJson = Json.parse(s"""|{ - | "callbackUrl": "https://some-url/callback", - | "minimumFileSize" : 0, - | "maximumFileSize" : 1024, - | "successRedirect": "https://some-url/success", - | "consumingService": "$SomeOtherConsumingService" - |}""".stripMargin) Given("a request containing a User-Agent header") - val headers = FakeHeaders(Seq((USER_AGENT, SomeConsumingService))) + val headers = FakeHeaders(Seq((USER_AGENT, someConsumingService))) val initiateRequest = FakeRequest(POST, uri = "/upscan/initiate", headers, postBodyJson) When("a request is posted to the /initiate endpoint") @@ -65,18 +70,16 @@ class PrepareUploadControllerISpec extends AnyWordSpecLike with should.Matchers And("the response should contain the requested upload fields") val fields = (responseJson \ "uploadRequest" \ "fields").as[Map[String, String]] - fields.get("x-amz-meta-callback-url") should contain ("https://some-url/callback") - fields.get("success_action_redirect") should contain ("https://some-url/success") - fields.get("x-amz-meta-consuming-service") should contain (SomeOtherConsumingService) - } + fields.get("x-amz-meta-callback-url") should contain ("https://some-url/callback") + fields.get("success_action_redirect") should contain ("https://some-url/success") + fields.get("x-amz-meta-consuming-service") should contain (someOtherConsumingService) - "PrepareUploadController prepareUploadV1 with only mandatory request values" in { - val postBodyJson = Json.parse("""|{ - | "callbackUrl": "https://some-url/callback" - |}""".stripMargin) + "PrepareUploadController prepareUploadV1 with only mandatory request values" in: + val postBodyJson = + Json.parse("""{ "callbackUrl": "https://some-url/callback" }""") Given("a request containing a User-Agent header") - val headers = FakeHeaders(Seq((USER_AGENT, SomeConsumingService))) + val headers = FakeHeaders(Seq((USER_AGENT, someConsumingService))) val initiateRequest = FakeRequest(POST, uri = "/upscan/initiate", headers, postBodyJson) When("a request is posted to the /initiate endpoint") @@ -91,25 +94,23 @@ class PrepareUploadControllerISpec extends AnyWordSpecLike with should.Matchers And("the response should contain the requested upload fields") val fields = (responseJson \ "uploadRequest" \ "fields").as[Map[String, String]] - fields.get("x-amz-meta-callback-url") should contain ("https://some-url/callback") - fields.get("success_action_redirect") shouldBe empty - fields.get("x-amz-meta-consuming-service") should contain (SomeConsumingService) - } - - "PrepareUploadController prepareUploadV2 with all request values" in { - val postBodyJson = Json.parse(s""" - |{ - | "callbackUrl": "https://some-url/callback", - | "successRedirect": "https://some-url/success", - | "errorRedirect": "https://some-url/error", - | "minimumFileSize" : 0, - | "maximumFileSize" : 1024, - | "consumingService": "$SomeOtherConsumingService" - |} + fields.get("x-amz-meta-callback-url") should contain ("https://some-url/callback") + fields.get("success_action_redirect") shouldBe empty + fields.get("x-amz-meta-consuming-service") should contain (someConsumingService) + + "PrepareUploadController prepareUploadV2 with all request values" in: + val postBodyJson = Json.parse(s"""{ + | "callbackUrl" : "https://some-url/callback", + | "successRedirect" : "https://some-url/success", + | "errorRedirect" : "https://some-url/error", + | "minimumFileSize" : 0, + | "maximumFileSize" : 1024, + | "consumingService": "$someOtherConsumingService" + |} """.stripMargin) Given("a request containing a User-Agent header") - val headers = FakeHeaders(Seq((USER_AGENT, SomeConsumingService))) + val headers = FakeHeaders(Seq((USER_AGENT, someConsumingService))) val initiateRequest = FakeRequest(POST, uri = "/upscan/v2/initiate", headers, postBodyJson) When("a request is posted to the /initiate endpoint") @@ -127,16 +128,14 @@ class PrepareUploadControllerISpec extends AnyWordSpecLike with should.Matchers fields.get("x-amz-meta-callback-url") should contain ("https://some-url/callback") fields.get("success_action_redirect") should contain ("https://some-url/success") fields.get("error_action_redirect") should contain ("https://some-url/error") - fields.get("x-amz-meta-consuming-service") should contain (SomeOtherConsumingService) - } + fields.get("x-amz-meta-consuming-service") should contain (someOtherConsumingService) - "PrepareUploadController prepareUploadV2 with only mandatory request values" in { - val postBodyJson = Json.parse("""|{ - | "callbackUrl": "https://some-url/callback" - |}""".stripMargin) + "PrepareUploadController prepareUploadV2 with only mandatory request values" in: + val postBodyJson = + Json.parse("""{ "callbackUrl": "https://some-url/callback" }""") Given("a request containing a User-Agent header") - val headers = FakeHeaders(Seq((USER_AGENT, SomeConsumingService))) + val headers = FakeHeaders(Seq((USER_AGENT, someConsumingService))) val initiateRequest = FakeRequest(POST, uri = "/upscan/v2/initiate", headers, postBodyJson) When("a request is posted to the /initiate endpoint") @@ -151,27 +150,22 @@ class PrepareUploadControllerISpec extends AnyWordSpecLike with should.Matchers And("the response should contain the requested upload fields") val fields = (responseJson \ "uploadRequest" \ "fields").as[Map[String, String]] - fields.get("x-amz-meta-callback-url") should contain ("https://some-url/callback") - fields.get("error_action_redirect") shouldBe empty - fields.get("success_action_redirect") shouldBe empty - fields.get("x-amz-meta-consuming-service") should contain (SomeConsumingService) - } + fields.get("x-amz-meta-callback-url") should contain ("https://some-url/callback") + fields.get("error_action_redirect") shouldBe empty + fields.get("success_action_redirect") shouldBe empty + fields.get("x-amz-meta-consuming-service") should contain (someConsumingService) - "Upscan V1" should { + "Upscan V1" should: val requestJson = Json.parse("""{"callbackUrl": "https://some-url/callback"}""") - behave like upscanInitiate(uri = "/upscan/initiate", requestJson) - } - "Upscan V2" should { + "Upscan V2" should: val requestJson = Json.parse("""{"callbackUrl": "https://some-url/callback"}""") - behave like upscanInitiate(uri = "/upscan/v2/initiate", requestJson) - } //noinspection ScalaStyle - private def upscanInitiate(uri: String, requestJson: JsValue): Unit = { - "reject requests which do not include a User-Agent header" in { + private def upscanInitiate(uri: String, requestJson: JsValue): Unit = + "reject requests which do not include a User-Agent header" in: Given("a request not containing a User-Agent header") val initiateRequest = FakeRequest(POST, uri, FakeHeaders(Seq((xSessionId, "some-session-id"))), requestJson) @@ -180,11 +174,10 @@ class PrepareUploadControllerISpec extends AnyWordSpecLike with should.Matchers Then("the response should indicate the request is invalid") status(initiateResponse) shouldBe BAD_REQUEST - } - "include x-amz-meta-consuming-service to identify the client service" in { + "include x-amz-meta-consuming-service to identify the client service" in: Given("a request containing a User-Agent header") - val initiateRequest = FakeRequest(POST, uri, FakeHeaders(Seq((USER_AGENT, SomeConsumingService))), requestJson) + val initiateRequest = FakeRequest(POST, uri, FakeHeaders(Seq((USER_AGENT, someConsumingService))), requestJson) When("a request is posted to the /initiate endpoint") val initiateResponse = route(app, initiateRequest).get @@ -194,12 +187,11 @@ class PrepareUploadControllerISpec extends AnyWordSpecLike with should.Matchers And("the response should identify the client service") val responseJson = contentAsJson(initiateResponse) - (responseJson \ "uploadRequest" \ "fields" \ "x-amz-meta-consuming-service").as[String] shouldBe SomeConsumingService - } + (responseJson \ "uploadRequest" \ "fields" \ "x-amz-meta-consuming-service").as[String] shouldBe someConsumingService - "include x-amz-meta-session-id in the response when a session exists" in { + "include x-amz-meta-session-id in the response when a session exists" in: Given("a valid request containing a x-session-id header") - val headers = FakeHeaders(Seq((USER_AGENT, SomeConsumingService), (xSessionId, "some-session-id"))) + val headers = FakeHeaders(Seq((USER_AGENT, someConsumingService), (xSessionId, "some-session-id"))) val initiateRequest = FakeRequest(POST, uri, headers, requestJson) When("a request is posted to the /initiate endpoint") @@ -211,11 +203,10 @@ class PrepareUploadControllerISpec extends AnyWordSpecLike with should.Matchers And("the response should include the expected value for x-amz-meta-session-id") val responseJson = contentAsJson(initiateResponse) (responseJson \ "uploadRequest" \ "fields" \ "x-amz-meta-session-id").as[String] shouldBe "some-session-id" - } - "set a default x-amz-meta-session-id in the response if no session exists" in { + "set a default x-amz-meta-session-id in the response if no session exists" in: Given("a request containing a x-session-id header") - val initiateRequest = FakeRequest(POST, uri, FakeHeaders(Seq((USER_AGENT, SomeConsumingService))), requestJson) + val initiateRequest = FakeRequest(POST, uri, FakeHeaders(Seq((USER_AGENT, someConsumingService))), requestJson) When("a request is posted to the /initiate endpoint") val initiateResponse = route(app, initiateRequest).get @@ -226,11 +217,10 @@ class PrepareUploadControllerISpec extends AnyWordSpecLike with should.Matchers And("the response should include the expected value for x-amz-meta-consuming-service") val responseJson = contentAsJson(initiateResponse) (responseJson \ "uploadRequest" \ "fields" \ "x-amz-meta-session-id").as[String] shouldBe "n/a" - } - "include standard upload fields" in { + "include standard upload fields" in: Given("a request containing a User-Agent header") - val initiateRequest = FakeRequest(POST, uri, FakeHeaders(Seq((USER_AGENT, SomeConsumingService))), requestJson) + val initiateRequest = FakeRequest(POST, uri, FakeHeaders(Seq((USER_AGENT, someConsumingService))), requestJson) When("a request is posted to the /initiate endpoint") val initiateResponse = route(app, initiateRequest).get @@ -240,20 +230,18 @@ class PrepareUploadControllerISpec extends AnyWordSpecLike with should.Matchers And("the response should contain the standard upload fields") val responseJson = contentAsJson(initiateResponse) - val reference = (responseJson \ "reference").as[String] - val fields = (responseJson \ "uploadRequest" \ "fields").as[Map[String, String]] - fields.get("key") should contain(reference) - fields.get("acl") should contain("private") + val reference = (responseJson \ "reference").as[String] + val fields = (responseJson \ "uploadRequest" \ "fields").as[Map[String, String]] + fields.get("key") should contain(reference) + fields.get("acl") should contain("private") fields.get("x-amz-algorithm") should contain("AWS4-HMAC-SHA256") fields should contain key "x-amz-date" fields should contain key "x-amz-credential" fields should contain key "x-amz-signature" fields should contain key "policy" - } - } -} - -private object PrepareUploadControllerISpec { - val SomeConsumingService = "PrepareUploadControllerISpec" - val SomeOtherConsumingService = "some-other-consuming-service" -} + +end PrepareUploadControllerISpec + +private object PrepareUploadControllerISpec: + val someConsumingService = "PrepareUploadControllerISpec" + val someOtherConsumingService = "some-other-consuming-service" diff --git a/test/uk/gov/hmrc/upscaninitiate/connector/s3/PolicySignerSpec.scala b/test/uk/gov/hmrc/upscaninitiate/connector/s3/PolicySignerSpec.scala index 1567fa8..6445099 100644 --- a/test/uk/gov/hmrc/upscaninitiate/connector/s3/PolicySignerSpec.scala +++ b/test/uk/gov/hmrc/upscaninitiate/connector/s3/PolicySignerSpec.scala @@ -19,11 +19,10 @@ package uk.gov.hmrc.upscaninitiate.connector.s3 import uk.gov.hmrc.upscaninitiate.connector.model.AwsCredentials import uk.gov.hmrc.upscaninitiate.test.UnitSpec -class PolicySignerSpec extends UnitSpec { - - "Policy signer" should { - "sign policy document" in { +class PolicySignerSpec extends UnitSpec: + "Policy signer" should: + "sign policy document" in: /* The test tries to verify if signing function works according to AWS documentation, and @@ -37,7 +36,6 @@ class PolicySignerSpec extends UnitSpec { https://stackoverflow.com/questions/47701044/sigv4-post-example-using-python We assume that signature that has been mentioned on stackoverflow is the one that is correct. - */ val credentials = AwsCredentials("AKIAIOSFODNN7EXAMPLE", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", None) @@ -48,8 +46,3 @@ class PolicySignerSpec extends UnitSpec { val signature = PolicySigner.signPolicy(credentials, "20151229", "us-east-1", encodedPolicy) signature shouldBe "8afdbf4008c03f22c2cd3cdb72e4afbb1f6a588f3255ac628749a66d7f09699e" - - } - } - -} diff --git a/test/uk/gov/hmrc/upscaninitiate/connector/s3/S3UploadFormGeneratorSpec.scala b/test/uk/gov/hmrc/upscaninitiate/connector/s3/S3UploadFormGeneratorSpec.scala index e7989c8..3ea6a5e 100644 --- a/test/uk/gov/hmrc/upscaninitiate/connector/s3/S3UploadFormGeneratorSpec.scala +++ b/test/uk/gov/hmrc/upscaninitiate/connector/s3/S3UploadFormGeneratorSpec.scala @@ -26,10 +26,10 @@ import uk.gov.hmrc.upscaninitiate.test.UnitSpec import java.time.Instant import java.util.Base64 -class S3UploadFormGeneratorSpec extends UnitSpec with GivenWhenThen { +class S3UploadFormGeneratorSpec extends UnitSpec with GivenWhenThen: - "S3UploadFormGenerator" should { - "generate required fields for a presigned POST request" in { + "S3UploadFormGenerator" should: + "generate required fields for a presigned POST request" in: // based on: https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-HTTPPOSTForms.html#sigv4-HTTPPOSTFormFields @@ -40,13 +40,14 @@ class S3UploadFormGeneratorSpec extends UnitSpec with GivenWhenThen { val policySigner = mock[PolicySigner] val testSignature = "test-signature" - when(policySigner.signPolicy(any[AwsCredentials], any[String], any[String], any[String])).thenReturn(testSignature) + when(policySigner.signPolicy(any[AwsCredentials], any[String], any[String], any[String])) + .thenReturn(testSignature) - val generator = new S3UploadFormGenerator(credentials, regionName, currentTime, policySigner) + val generator = S3UploadFormGenerator(credentials, regionName, currentTime, policySigner) And("there are valid upload parameters") val expirationTimestamp = "1997-07-16T19:20:40Z" - val uploadParameters = UploadParameters( + val uploadParameters = UploadParameters( expirationDateTime = Instant.parse(expirationTimestamp), bucketName = "test-bucket", objectKey = "test-key", @@ -62,22 +63,20 @@ class S3UploadFormGeneratorSpec extends UnitSpec with GivenWhenThen { Then("valid POST policy is produced") val policy = decodePolicyFormResult(result("policy")) - (policy \ "expiration").as[String] shouldBe "1997-07-16T19:20:40Z" And("policy contains proper conditions") - - ((policy \ "conditions").get \\ "acl").head.as[String] shouldBe "private" - ((policy \ "conditions").get \\ "bucket").head.as[String] shouldBe "test-bucket" - ((policy \ "conditions").get \\ "key").head.as[String] shouldBe "test-key" - ((policy \ "conditions").get \\ "x-amz-algorithm").head.as[String] shouldBe "AWS4-HMAC-SHA256" - ((policy \ "conditions").get \\ "x-amz-date").head.as[String] shouldBe "19970716T192030Z" - ((policy \ "conditions").get \\ "x-amz-security-token").head.as[String] shouldBe "session-token" - ((policy \ "conditions").get \\ "x-amz-meta-key1").head.as[String] shouldBe "value1" + ((policy \ "conditions").get \\ "acl" ).head.as[String] shouldBe "private" + ((policy \ "conditions").get \\ "bucket" ).head.as[String] shouldBe "test-bucket" + ((policy \ "conditions").get \\ "key" ).head.as[String] shouldBe "test-key" + ((policy \ "conditions").get \\ "x-amz-algorithm" ).head.as[String] shouldBe "AWS4-HMAC-SHA256" + ((policy \ "conditions").get \\ "x-amz-date" ).head.as[String] shouldBe "19970716T192030Z" + ((policy \ "conditions").get \\ "x-amz-security-token" ).head.as[String] shouldBe "session-token" + ((policy \ "conditions").get \\ "x-amz-meta-key1" ).head.as[String] shouldBe "value1" ((policy \ "conditions").get \\ "success_action_redirect").head.as[String] shouldBe "http://test.com/abc" - ((policy \ "conditions").get \\ "error_action_redirect").head.as[String] shouldBe "http://test.com/error" + ((policy \ "conditions").get \\ "error_action_redirect" ).head.as[String] shouldBe "http://test.com/error" - val conditions = (policy \ "conditions").as[JsArray].value + val conditions = (policy \ "conditions").as[JsArray].value val arrayConditions = conditions.flatMap(_.asOpt[JsArray].map(_.value)) And("policy contains proper size constraints") @@ -104,10 +103,8 @@ class S3UploadFormGeneratorSpec extends UnitSpec with GivenWhenThen { result("x-amz-meta-key1") shouldBe "value1" result("x-amz-security-token") shouldBe "session-token" result("x-amz-meta-original-filename") shouldBe s"$${filename}" - } - - "generate a signed link with a success redirect" in { + "generate a signed link with a success redirect" in: Given("there is a properly configured form generator with AWS credentials") val credentials = AwsCredentials("accessKeyId", "secretKey", Some("session-token")) val regionName = "us-east-1" @@ -115,13 +112,14 @@ class S3UploadFormGeneratorSpec extends UnitSpec with GivenWhenThen { val policySigner = mock[PolicySigner] val testSignature = "test-signature" - when(policySigner.signPolicy(any[AwsCredentials], any[String], any[String], any[String])).thenReturn(testSignature) + when(policySigner.signPolicy(any[AwsCredentials], any[String], any[String], any[String])) + .thenReturn(testSignature) - val generator = new S3UploadFormGenerator(credentials, regionName, currentTime, policySigner) + val generator = S3UploadFormGenerator(credentials, regionName, currentTime, policySigner) And("there are valid upload parameters") val expirationTimestamp = "1997-07-16T19:20:40Z" - val uploadParameters = UploadParameters( + val uploadParameters = UploadParameters( expirationDateTime = Instant.parse(expirationTimestamp), bucketName = "test-bucket", objectKey = "test-key", @@ -134,12 +132,9 @@ class S3UploadFormGeneratorSpec extends UnitSpec with GivenWhenThen { When("form fields are generated") val result = generator.generateFormFields(uploadParameters) - result("success_action_redirect") shouldBe "http://test.server/success" - } - - "generate a signed link with a error redirect" in { + "generate a signed link with a error redirect" in: Given("there is a properly configured form generator with AWS credentials") val credentials = AwsCredentials("accessKeyId", "secretKey", Some("session-token")) val regionName = "us-east-1" @@ -147,9 +142,10 @@ class S3UploadFormGeneratorSpec extends UnitSpec with GivenWhenThen { val policySigner = mock[PolicySigner] val testSignature = "test-signature" - when(policySigner.signPolicy(any[AwsCredentials], any[String], any[String], any[String])).thenReturn(testSignature) + when(policySigner.signPolicy(any[AwsCredentials], any[String], any[String], any[String])) + .thenReturn(testSignature) - val generator = new S3UploadFormGenerator(credentials, regionName, currentTime, policySigner) + val generator = S3UploadFormGenerator(credentials, regionName, currentTime, policySigner) And("there are valid upload parameters") val expirationTimestamp = "1997-07-16T19:20:40Z" @@ -166,14 +162,8 @@ class S3UploadFormGeneratorSpec extends UnitSpec with GivenWhenThen { When("form fields are generated") val result = generator.generateFormFields(uploadParameters) - result("error_action_redirect") shouldBe "http://test.server/error" - } - } - def decodePolicyFormResult(base64EncodedPolicy: String): JsValue = { - val decoded = new String(Base64.getDecoder.decode(base64EncodedPolicy), "UTF-8") + def decodePolicyFormResult(base64EncodedPolicy: String): JsValue = + val decoded = String(Base64.getDecoder.decode(base64EncodedPolicy), "UTF-8") Json.parse(decoded) - } - -} diff --git a/test/uk/gov/hmrc/upscaninitiate/controller/PrepareUploadControllerSpec.scala b/test/uk/gov/hmrc/upscaninitiate/controller/PrepareUploadControllerSpec.scala index 48a14bd..d83117e 100644 --- a/test/uk/gov/hmrc/upscaninitiate/controller/PrepareUploadControllerSpec.scala +++ b/test/uk/gov/hmrc/upscaninitiate/controller/PrepareUploadControllerSpec.scala @@ -27,7 +27,6 @@ import play.api.mvc.Results.Ok import play.api.test.Helpers.{contentAsJson, contentAsString, status} import play.api.test.{FakeRequest, Helpers, StubControllerComponentsFactory} import uk.gov.hmrc.upscaninitiate.config.ServiceConfiguration -import uk.gov.hmrc.upscaninitiate.controller.PrepareUploadControllerSpec.{ConsumingService, UserAgent, requestTemplate, settingsTemplate} import uk.gov.hmrc.upscaninitiate.controller.model.{PrepareUploadRequest, PreparedUploadResponse, Reference, UploadFormTemplate} import uk.gov.hmrc.upscaninitiate.service.PrepareUploadService import uk.gov.hmrc.upscaninitiate.service.model.UploadSettings @@ -35,46 +34,53 @@ import uk.gov.hmrc.upscaninitiate.test.UnitSpec import java.time.Clock import scala.concurrent.Future -import scala.concurrent.duration._ -import scala.language.postfixOps +import scala.concurrent.duration.DurationInt -class PrepareUploadControllerSpec extends UnitSpec with StubControllerComponentsFactory with GivenWhenThen { +class PrepareUploadControllerSpec + extends UnitSpec + with StubControllerComponentsFactory + with GivenWhenThen: - private implicit val actorSystem: ActorSystem = ActorSystem() + import PrepareUploadControllerSpec._ - private implicit val timeout: org.apache.pekko.util.Timeout = 10 seconds + private given ActorSystem = ActorSystem() + + private given org.apache.pekko.util.Timeout = 10.seconds private val clock = Clock.fixed(Clock.systemDefaultZone().instant(), Clock.systemDefaultZone().getZone) private trait WithGlobalFileSizeLimitFixture { - val globalFileSizeLimit = 1024L + val globalFileSizeLimit = 1024L val prepareUploadService = mock[PrepareUploadService] - when(prepareUploadService.globalFileSizeLimit).thenReturn(globalFileSizeLimit) + when(prepareUploadService.globalFileSizeLimit) + .thenReturn(globalFileSizeLimit) } - private trait WithServiceConfiguration { + private trait WithServiceConfiguration: val config = mock[ServiceConfiguration] - } - private trait WithAllowedCallbackProtocol { this: WithServiceConfiguration => - when(config.allowedCallbackProtocols).thenReturn(List("https")) - } + private trait WithAllowedCallbackProtocol: + this: WithServiceConfiguration => + when(config.allowedCallbackProtocols) + .thenReturn(List("https")) - private trait WithInboundBucketName { this: WithServiceConfiguration => - when(config.inboundBucketName).thenReturn("inbound-bucket") - } + private trait WithInboundBucketName: + this: WithServiceConfiguration => + when(config.inboundBucketName) + .thenReturn("inbound-bucket") - private trait WithUploadProxyUrl { this: WithServiceConfiguration => - when(config.uploadProxyUrl).thenReturn("https://upload-proxy.com") - } + private trait WithUploadProxyUrl: + this: WithServiceConfiguration => + when(config.uploadProxyUrl) + .thenReturn("https://upload-proxy.com") - private trait WithAllowedCallbackProtocolFixture extends WithServiceConfiguration with WithAllowedCallbackProtocol - private trait WithV1SuccessFixture extends WithAllowedCallbackProtocolFixture with WithInboundBucketName - private trait WithV2SuccessFixture extends WithV1SuccessFixture with WithUploadProxyUrl - private trait WithV1BadRequestFixture extends WithServiceConfiguration with WithInboundBucketName - private trait WithV2BadRequestFixture extends WithV1BadRequestFixture with WithUploadProxyUrl + private trait WithAllowedCallbackProtocolFixture extends WithServiceConfiguration with WithAllowedCallbackProtocol + private trait WithV1SuccessFixture extends WithAllowedCallbackProtocolFixture with WithInboundBucketName + private trait WithV2SuccessFixture extends WithV1SuccessFixture with WithUploadProxyUrl + private trait WithV1BadRequestFixture extends WithServiceConfiguration with WithInboundBucketName + private trait WithV2BadRequestFixture extends WithV1BadRequestFixture with WithUploadProxyUrl - "V1 initiate with minimal request settings" should { + "V1 initiate with minimal request settings" should: val minimalRequestBody = Json.obj("callbackUrl" -> "https://www.example.com") @@ -84,7 +90,7 @@ class PrepareUploadControllerSpec extends UnitSpec with StubControllerComponents uploadUrl = "https://inbound-bucket.s3.amazonaws.com" ) - val _ = new WithV1SuccessFixture { + val _ = new WithV1SuccessFixture: behave like successfulInitiate( config = config, @@ -94,17 +100,15 @@ class PrepareUploadControllerSpec extends UnitSpec with StubControllerComponents sessionId = None, expectedUploadSettings = expectedUploadSettings ) - } - } - "V1 initiate with all request settings" should { + "V1 initiate with all request settings" should: val maximalRequestBody = Json.obj( "callbackUrl" -> "https://www.example.com", "minimumFileSize" -> 1, "maximumFileSize" -> 1024, "successRedirect" -> "https://www.example.com/success", - "consumingService" -> ConsumingService + "consumingService" -> consumingService ) val expectedUploadSettings = @@ -116,11 +120,11 @@ class PrepareUploadControllerSpec extends UnitSpec with StubControllerComponents minimumFileSize = Some(1), maximumFileSize = Some(1024), successRedirect = Some("https://www.example.com/success"), - consumingService = Some(ConsumingService) + consumingService = Some(consumingService) ) ) - val _ = new WithV1SuccessFixture { + val _ = new WithV1SuccessFixture: behave like successfulInitiate( config = config, @@ -130,10 +134,8 @@ class PrepareUploadControllerSpec extends UnitSpec with StubControllerComponents sessionId = Some("a-session-id"), expectedUploadSettings = expectedUploadSettings ) - } - } - "V2 initiate with minimal request settings" should { + "V2 initiate with minimal request settings" should: val minimalRequestBody = Json.obj("callbackUrl" -> "https://www.example.com") @@ -143,7 +145,7 @@ class PrepareUploadControllerSpec extends UnitSpec with StubControllerComponents uploadUrl = "https://upload-proxy.com/v1/uploads/inbound-bucket" ) - val _ = new WithV2SuccessFixture { + val _ = new WithV2SuccessFixture: behave like successfulInitiate( config = config, @@ -153,31 +155,28 @@ class PrepareUploadControllerSpec extends UnitSpec with StubControllerComponents sessionId = None, expectedUploadSettings = expectedUploadSettings ) - } - } "V2 initiate with all request settings" should { val maximalRequestBody = Json.obj( - "callbackUrl" -> "https://www.example.com", - "minimumFileSize" -> 1, - "maximumFileSize" -> 1024, - "successRedirect" -> "https://www.example.com/success", - "errorRedirect" -> "https://www.example.com/error", - "consumingService" -> ConsumingService + "callbackUrl" -> "https://www.example.com", + "minimumFileSize" -> 1, + "maximumFileSize" -> 1024, + "successRedirect" -> "https://www.example.com/success", + "errorRedirect" -> "https://www.example.com/error", + "consumingService" -> consumingService ) val expectedUploadSettings = settingsTemplate .copy( uploadUrl = "https://upload-proxy.com/v1/uploads/inbound-bucket", - prepareUploadRequest = - requestTemplate.copy( + prepareUploadRequest = requestTemplate.copy( minimumFileSize = Some(1), maximumFileSize = Some(1024), successRedirect = Some("https://www.example.com/success"), errorRedirect = Some("https://www.example.com/error"), - consumingService = Some(ConsumingService) + consumingService = Some(consumingService) ) ) @@ -195,30 +194,35 @@ class PrepareUploadControllerSpec extends UnitSpec with StubControllerComponents } //noinspection ScalaStyle - private def successfulInitiate(config: ServiceConfiguration, - actionUnderTest: PrepareUploadController => Action[JsValue], - requestBody: JsValue, - requestId: Option[String], - sessionId: Option[String], - expectedUploadSettings: UploadSettings): Unit = { - - "prepare upload request" in new WithGlobalFileSizeLimitFixture { - val controller = new PrepareUploadController(prepareUploadService, config, clock, stubControllerComponents()) + private def successfulInitiate( + config : ServiceConfiguration, + actionUnderTest : PrepareUploadController => Action[JsValue], + requestBody : JsValue, + requestId : Option[String], + sessionId : Option[String], + expectedUploadSettings: UploadSettings + ): Unit = + "prepare upload request" in new WithGlobalFileSizeLimitFixture: + val controller = PrepareUploadController(prepareUploadService, config, clock, stubControllerComponents()) Given("a valid initiate request") - val headers = (USER_AGENT, UserAgent) +: Seq( - requestId.map(Tuple2("x-request-id", _)), - sessionId.map(Tuple2("x-session-id", _)) - ).flatten + val headers = + (USER_AGENT, userAgent) + +: Seq( + requestId.map(Tuple2("x-request-id", _)), + sessionId.map(Tuple2("x-session-id", _)) + ).flatten + val request = FakeRequest().withHeaders(headers: _*).withBody(requestBody) - val expectedUploadResponse = PreparedUploadResponse( - reference = Reference("TEST"), - uploadRequest = UploadFormTemplate( - href = expectedUploadSettings.uploadUrl, - fields = Map("a" -> "b", "x" -> "y") + val expectedUploadResponse = + PreparedUploadResponse( + reference = Reference("TEST"), + uploadRequest = UploadFormTemplate( + href = expectedUploadSettings.uploadUrl, + fields = Map("a" -> "b", "x" -> "y") + ) ) - ) when(prepareUploadService.prepareUpload( expectedUploadSettings, @@ -237,73 +241,70 @@ class PrepareUploadControllerSpec extends UnitSpec with StubControllerComponents s""""${kvPair._1}": "${kvPair._2}"""" }.mkString("{", ",", "}") - contentAsJson(result) shouldBe Json.parse( - s"""|{ - | "reference": "${expectedUploadResponse.reference.value}", - | "uploadRequest": { - | "href": "${expectedUploadResponse.uploadRequest.href}", - | "fields": $expectedFieldsAsJson - | } - |}""".stripMargin) - } - } + contentAsJson(result) shouldBe Json.parse(s"""{ + | "reference": "${expectedUploadResponse.reference.value}", + | "uploadRequest": { + | "href" : "${expectedUploadResponse.uploadRequest.href}", + | "fields": $expectedFieldsAsJson + | } + |}""".stripMargin + ) - "V1 bad initiate request" should { - val _ = new WithV1BadRequestFixture { + "V1 bad initiate request" should: + val _ = new WithV1BadRequestFixture: behave like badRequestInitiate(config, _.prepareUploadV1()) - } - } - "V2 bad initiate request" should { - val _ = new WithV2BadRequestFixture { + "V2 bad initiate request" should: + val _ = new WithV2BadRequestFixture: behave like badRequestInitiate(config, _.prepareUploadV2()) - } - } private def badRequestInitiate(// scalastyle:ignore - config: ServiceConfiguration, - prepareUploadAction: PrepareUploadController => Action[JsValue]): Unit = { - - "return a bad request error when the request has the wrong structure" in new WithGlobalFileSizeLimitFixture { - val controller = new PrepareUploadController(prepareUploadService, config, clock, stubControllerComponents()) + config : ServiceConfiguration, + prepareUploadAction: PrepareUploadController => Action[JsValue] + ): Unit = + "return a bad request error when the request has the wrong structure" in new WithGlobalFileSizeLimitFixture: + val controller = PrepareUploadController(prepareUploadService, config, clock, stubControllerComponents()) Given("there is an invalid upload request") - val request = FakeRequest().withHeaders( - (USER_AGENT, UserAgent), - ("x-session-id", "some-session-id"), - ("x-request-id", "some-request-id") - ).withBody(Json.obj("invalid" -> "body")) + val request = + FakeRequest() + .withHeaders( + (USER_AGENT, userAgent), + ("x-session-id", "some-session-id"), + ("x-request-id", "some-request-id") + ) + .withBody(Json.obj("invalid" -> "body")) When("upload initiation has been requested") val result = prepareUploadAction(controller)(request) Then("service returns error response") withClue(Helpers.contentAsString(result)) { status(result) shouldBe BAD_REQUEST } - } - "return a bad request error when the maximum file size is in excess of the global maximum" in new WithGlobalFileSizeLimitFixture { - val controller = new PrepareUploadController(prepareUploadService, config, clock, stubControllerComponents()) + "return a bad request error when the maximum file size is in excess of the global maximum" in new WithGlobalFileSizeLimitFixture: + val controller = PrepareUploadController(prepareUploadService, config, clock, stubControllerComponents()) Given("there is an invalid upload request") - val request = FakeRequest().withHeaders( - (USER_AGENT, UserAgent), - ("x-session-id", "some-session-id") - ).withBody(Json.obj( - "callbackUrl" -> "https://www.example.com", - "maximumFileSize" -> (globalFileSizeLimit + 1)) - ) + val request = + FakeRequest() + .withHeaders( + (USER_AGENT, userAgent), + ("x-session-id", "some-session-id") + ) + .withBody(Json.obj( + "callbackUrl" -> "https://www.example.com", + "maximumFileSize" -> (globalFileSizeLimit + 1)) + ) When("upload initiation has been requested") val result = prepareUploadAction(controller)(request) Then("service returns error response") - withClue(Helpers.contentAsString(result)) { + withClue(Helpers.contentAsString(result)): status(result) shouldBe BAD_REQUEST - } - } - "return a bad request error when no user agent is supplied" in new WithGlobalFileSizeLimitFixture { - val controller = new PrepareUploadController(prepareUploadService, config, clock, stubControllerComponents()) + "return a bad request error when no user agent is supplied" in new WithGlobalFileSizeLimitFixture: + val controller = PrepareUploadController(prepareUploadService, config, clock, stubControllerComponents()) Given("there is an upload request without a user agent header") val request = FakeRequest().withHeaders( @@ -314,51 +315,45 @@ class PrepareUploadControllerSpec extends UnitSpec with StubControllerComponents val result = prepareUploadAction(controller)(request) Then("service returns error response") - withClue(Helpers.contentAsString(result)) { + withClue(Helpers.contentAsString(result)): status(result) shouldBe BAD_REQUEST - } - } - } - "withAllowedCallbackProtocol" should { - "allow https callback urls" in new WithGlobalFileSizeLimitFixture with WithAllowedCallbackProtocolFixture { - val controller = new PrepareUploadController(prepareUploadService, config, clock, stubControllerComponents()) + "withAllowedCallbackProtocol" should: + "allow https callback urls" in new WithGlobalFileSizeLimitFixture with WithAllowedCallbackProtocolFixture: + val controller = PrepareUploadController(prepareUploadService, config, clock, stubControllerComponents()) - val result = controller.withAllowedCallbackProtocol("https://my.callback.url") { - Future.successful(Ok) - } + val result = + controller.withAllowedCallbackProtocol("https://my.callback.url"): + Future.successful(Ok) status(result) shouldBe OK - } - "disallow http callback urls" in new WithGlobalFileSizeLimitFixture with WithAllowedCallbackProtocolFixture { - val controller = new PrepareUploadController(prepareUploadService, config, clock, stubControllerComponents()) + "disallow http callback urls" in new WithGlobalFileSizeLimitFixture with WithAllowedCallbackProtocolFixture: + val controller = PrepareUploadController(prepareUploadService, config, clock, stubControllerComponents()) - val result = controller.withAllowedCallbackProtocol("http://my.callback.url") { - Future.failed(new RuntimeException("This block should not have been invoked.")) - } + val result = + controller.withAllowedCallbackProtocol("http://my.callback.url"): + Future.failed(RuntimeException("This block should not have been invoked.")) - status(result) shouldBe BAD_REQUEST + status(result) shouldBe BAD_REQUEST contentAsString(result) should include("Invalid callback url protocol") - } - "disallow invalidly formatted callback urls" in new WithGlobalFileSizeLimitFixture with WithAllowedCallbackProtocolFixture { - val controller = new PrepareUploadController(prepareUploadService, config, clock, stubControllerComponents()) + "disallow invalidly formatted callback urls" in new WithGlobalFileSizeLimitFixture with WithAllowedCallbackProtocolFixture: + val controller = PrepareUploadController(prepareUploadService, config, clock, stubControllerComponents()) - val result = controller.withAllowedCallbackProtocol("123") { - Future.failed(new RuntimeException("This block should not have been invoked.")) - } + val result = + controller.withAllowedCallbackProtocol("123"): + Future.failed(RuntimeException("This block should not have been invoked.")) - status(result) shouldBe BAD_REQUEST + status(result) shouldBe BAD_REQUEST contentAsString(result) should include("Invalid callback url format") - } - } -} -object PrepareUploadControllerSpec { +end PrepareUploadControllerSpec + +object PrepareUploadControllerSpec: - val UserAgent = "user-agent" - val ConsumingService = "some-consuming-service" + val userAgent = "user-agent" + val consumingService = "some-consuming-service" val requestTemplate: PrepareUploadRequest = PrepareUploadRequest( @@ -373,7 +368,6 @@ object PrepareUploadControllerSpec { val settingsTemplate: UploadSettings = UploadSettings( uploadUrl = s"http://upload-proxy.com", - userAgent = UserAgent, + userAgent = userAgent, prepareUploadRequest = requestTemplate ) -} diff --git a/test/uk/gov/hmrc/upscaninitiate/controller/model/PrepareUploadRequestSpec.scala b/test/uk/gov/hmrc/upscaninitiate/controller/model/PrepareUploadRequestSpec.scala index 545617d..324b952 100644 --- a/test/uk/gov/hmrc/upscaninitiate/controller/model/PrepareUploadRequestSpec.scala +++ b/test/uk/gov/hmrc/upscaninitiate/controller/model/PrepareUploadRequestSpec.scala @@ -20,31 +20,28 @@ import play.api.libs.json.{JsSuccess, Json, Reads} import uk.gov.hmrc.upscaninitiate.controller.model.PrepareUploadRequestSpec._ import uk.gov.hmrc.upscaninitiate.test.UnitSpec -class PrepareUploadRequestSpec extends UnitSpec { +class PrepareUploadRequestSpec extends UnitSpec: - "V1 deserialisation" should { + "V1 deserialisation" should: behave like testDeserialisation( reads = PrepareUploadRequest.readsV1(maxFileSize), expectedErrorRedirect = None ) - } - "V2 deserialisation" should { + "V2 deserialisation" should: behave like testDeserialisation( reads = PrepareUploadRequest.readsV2(maxFileSize), expectedErrorRedirect = Some("https://www.example.com/error") ) - } //noinspection ScalaStyle private def testDeserialisation( reads: Reads[PrepareUploadRequest], expectedErrorRedirect: Option[String] - ) = { - - "deserialise from required fields" in { + ) = + "deserialise from required fields" in: val parse = Json.parse( """ @@ -58,9 +55,8 @@ class PrepareUploadRequestSpec extends UnitSpec { requestTemplate parse shouldBe JsSuccess(expectedPrepareUploadRequest) - } - "deserialise from all fields" in { + "deserialise from all fields" in: val parse = Json.parse( """ @@ -86,9 +82,8 @@ class PrepareUploadRequestSpec extends UnitSpec { ) parse shouldBe JsSuccess(expectedPrepareUploadRequest) - } - "reject out-of-bounds `minimumFileSize` values" in { + "reject out-of-bounds `minimumFileSize` values" in: val parse = Json.parse( """ @@ -100,9 +95,8 @@ class PrepareUploadRequestSpec extends UnitSpec { ).validate(reads) parse.isError shouldBe true - } - "reject out-of-bounds `maximumFileSize` values" in { + "reject out-of-bounds `maximumFileSize` values" in: def parse(withMaximumFileSizeValue: Long) = Json.parse( s""" @@ -113,11 +107,10 @@ class PrepareUploadRequestSpec extends UnitSpec { |""".stripMargin ).validate(reads) - parse(withMaximumFileSizeValue = -1).isError shouldBe true + parse(withMaximumFileSizeValue = -1 ).isError shouldBe true parse(withMaximumFileSizeValue = maxFileSize + 1).isError shouldBe true - } - "reject erroneous `minimumFileSize` & `maximumFileSize` combinations" in { + "reject erroneous `minimumFileSize` & `maximumFileSize` combinations" in: val parse = Json.parse( """ @@ -130,11 +123,8 @@ class PrepareUploadRequestSpec extends UnitSpec { ).validate(reads) parse.isError shouldBe true - } - } -} -object PrepareUploadRequestSpec { +object PrepareUploadRequestSpec: val maxFileSize: Long = 100 @@ -148,4 +138,3 @@ object PrepareUploadRequestSpec { errorRedirect = None, consumingService = None ) -} diff --git a/test/uk/gov/hmrc/upscaninitiate/model/PrepareUploadServiceSpec.scala b/test/uk/gov/hmrc/upscaninitiate/model/PrepareUploadServiceSpec.scala index 66879d5..4aee1fe 100644 --- a/test/uk/gov/hmrc/upscaninitiate/model/PrepareUploadServiceSpec.scala +++ b/test/uk/gov/hmrc/upscaninitiate/model/PrepareUploadServiceSpec.scala @@ -27,63 +27,58 @@ import uk.gov.hmrc.upscaninitiate.service.PrepareUploadService import uk.gov.hmrc.upscaninitiate.service.model.UploadSettings import uk.gov.hmrc.upscaninitiate.test.UnitSpec -import java.time import java.time.Instant +import scala.concurrent.duration.{DurationInt, FiniteDuration} -class PrepareUploadServiceSpec extends UnitSpec with GivenWhenThen { +class PrepareUploadServiceSpec extends UnitSpec with GivenWhenThen: - private val serviceConfiguration = new ServiceConfiguration { + private val serviceConfiguration = + new ServiceConfiguration: + override def accessKeyId: String = ??? - override def accessKeyId: String = ??? + override def secretAccessKey: String = ??? - override def secretAccessKey: String = ??? + override def uploadProxyUrl: String = ??? - override def uploadProxyUrl: String = ??? + override def inboundBucketName: String = "test-bucket" - override def inboundBucketName: String = "test-bucket" + override def sessionToken: Option[String] = ??? - override def sessionToken: Option[String] = ??? + override def region: String = ??? - override def region: String = ??? + override def fileExpirationPeriod: FiniteDuration = 7.days - override def fileExpirationPeriod: time.Duration = time.Duration.ofDays(7) + override def globalFileSizeLimit = 1024 - override def globalFileSizeLimit = 1024 + override def allowedCallbackProtocols: List[String] = List("https") - override def allowedCallbackProtocols: List[String] = List("https") - } + private def metricsStub() = + new Metrics: + override val defaultRegistry: MetricRegistry = MetricRegistry() - private def metricsStub() = new Metrics { - - override val defaultRegistry: MetricRegistry = new MetricRegistry - - } - - private val s3PostSigner = new UploadFormGenerator { + private val s3PostSigner = new UploadFormGenerator: override def generateFormFields(uploadParameters: UploadParameters): Map[String, String] = Map( "bucket" -> uploadParameters.bucketName, "key" -> uploadParameters.objectKey, "minSize" -> uploadParameters.contentLengthRange.min.toString, "maxSize" -> uploadParameters.contentLengthRange.max.toString - ) ++ - uploadParameters.additionalMetadata.map { case (k, v) => s"x-amz-meta-$k" -> v } ++ - uploadParameters.successRedirect.map { "success_redirect_url" -> _ } ++ - uploadParameters.errorRedirect.map { "error_redirect_url" -> _ } - } + ) + ++ uploadParameters.additionalMetadata.map { case (k, v) => s"x-amz-meta-$k" -> v } + ++ uploadParameters.successRedirect.map { "success_redirect_url" -> _ } + ++ uploadParameters.errorRedirect.map { "error_redirect_url" -> _ } val receivedAt: Instant = Instant.now() - "S3 Prepare Upload Service" should { - - def service(metrics: Metrics) = new PrepareUploadService(s3PostSigner, serviceConfiguration, metrics) + "S3 Prepare Upload Service" should: - "create post form that allows to upload the file" in { + def service(metrics: Metrics) = + PrepareUploadService(s3PostSigner, serviceConfiguration, metrics) + "create post form that allows to upload the file" in: val metrics = metricsStub() Given("there are valid upload settings") - val settings = settingsTemplate @@ -116,14 +111,10 @@ class PrepareUploadServiceSpec extends UnitSpec with GivenWhenThen { And("uploadInitiated counter has been incremented") metrics.defaultRegistry.counter("uploadInitiated").getCount shouldBe 1 - } - - "take in account file size limits if provided in request" in { - + "take in account file size limits if provided in request" in: val metrics = metricsStub() Given("there are valid upload settings with size limits") - val settings = settingsTemplate .copy( @@ -143,14 +134,11 @@ class PrepareUploadServiceSpec extends UnitSpec with GivenWhenThen { result.uploadRequest.fields("minSize") shouldBe "100" result.uploadRequest.fields("maxSize") shouldBe "200" - } - - "fail when minimum file size is less than 0" in { - - Given("there are upload settings with minimum file size less than zero") + "fail when minimum file size is less than 0" in: val metrics = metricsStub() + Given("there are upload settings with minimum file size less than zero") val settings = settingsTemplate .copy( @@ -170,14 +158,10 @@ class PrepareUploadServiceSpec extends UnitSpec with GivenWhenThen { metrics.defaultRegistry.counter("uploadInitiated").getCount shouldBe 0 - } - - "fail when maximum file size is greater than global limit" in { - - Given("there upload settings with maximum file size greater than global limit") - + "fail when maximum file size is greater than global limit" in: val metrics = metricsStub() + Given("there upload settings with maximum file size greater than global limit") val settings = settingsTemplate .copy( @@ -190,20 +174,16 @@ class PrepareUploadServiceSpec extends UnitSpec with GivenWhenThen { When("we setup the upload") Then("an exception should be thrown") - val thrown = the[IllegalArgumentException] thrownBy service(metrics) .prepareUpload(settings, "some-request-id", "some-session-id", receivedAt) thrown.getMessage should include("Maximum file size is greater than global maximum file size") metrics.defaultRegistry.counter("uploadInitiated").getCount shouldBe 0 - } - - "fail when minimum file size is greater than maximum file size" in { - - Given("there are upload settings with minimum file size greater than maximum size") + "fail when minimum file size is greater than maximum file size" in: val metrics = metricsStub() + Given("there are upload settings with minimum file size greater than maximum size") val settings = settingsTemplate .copy( @@ -216,20 +196,16 @@ class PrepareUploadServiceSpec extends UnitSpec with GivenWhenThen { When("we setup the upload") Then("an exception should be thrown") - val thrown = the[IllegalArgumentException] thrownBy service(metrics) .prepareUpload(settings, "some-request-id", "some-session-id", receivedAt) thrown.getMessage should include("Minimum file size is greater than maximum file size") metrics.defaultRegistry.counter("uploadInitiated").getCount shouldBe 0 - } - - "create post form that allows to upload the file with redirect on success" in { + "create post form that allows to upload the file with redirect on success" in: val metrics = metricsStub() Given("there are valid upload settings") - val settings = settingsTemplate .copy( @@ -245,7 +221,6 @@ class PrepareUploadServiceSpec extends UnitSpec with GivenWhenThen { .prepareUpload(settings, "some-request-id", "some-session-id", receivedAt) Then("proper upload request form definition should be returned") - result.uploadRequest.href shouldBe settings.uploadUrl result.uploadRequest.fields shouldBe Map( "bucket" -> serviceConfiguration.inboundBucketName, @@ -263,14 +238,10 @@ class PrepareUploadServiceSpec extends UnitSpec with GivenWhenThen { And("uploadInitiated counter has been incremented") metrics.defaultRegistry.counter("uploadInitiated").getCount shouldBe 1 - } - - "create post form that allows to upload the file with redirect on error" in { - + "create post form that allows to upload the file with redirect on error" in: val metrics = metricsStub() Given("there are valid upload settings") - val settings = settingsTemplate .copy( @@ -281,12 +252,10 @@ class PrepareUploadServiceSpec extends UnitSpec with GivenWhenThen { ) When("we setup the upload") - val result = service(metrics) .prepareUpload(settings, "some-request-id", "some-session-id", receivedAt) Then("proper upload request form definition should be returned") - result.uploadRequest.href shouldBe settings.uploadUrl result.uploadRequest.fields shouldBe Map( "bucket" -> serviceConfiguration.inboundBucketName, @@ -304,11 +273,6 @@ class PrepareUploadServiceSpec extends UnitSpec with GivenWhenThen { And("uploadInitiated counter has been incremented") metrics.defaultRegistry.counter("uploadInitiated").getCount shouldBe 1 - } - - } -} - object PrepareUploadServiceSpec { val requestTemplate: PrepareUploadRequest = diff --git a/test/uk/gov/hmrc/upscaninitiate/test/UnitSpec.scala b/test/uk/gov/hmrc/upscaninitiate/test/UnitSpec.scala index 3602ea0..ba1392d 100644 --- a/test/uk/gov/hmrc/upscaninitiate/test/UnitSpec.scala +++ b/test/uk/gov/hmrc/upscaninitiate/test/UnitSpec.scala @@ -20,4 +20,7 @@ import org.scalatest.matchers.should import org.scalatest.wordspec.AnyWordSpecLike import org.scalatestplus.mockito.MockitoSugar -trait UnitSpec extends AnyWordSpecLike with should.Matchers with MockitoSugar +trait UnitSpec + extends AnyWordSpecLike + with should.Matchers + with MockitoSugar diff --git a/test/uk/gov/hmrc/upscaninitiate/util/UserAgentFilterSpec.scala b/test/uk/gov/hmrc/upscaninitiate/util/UserAgentFilterSpec.scala index 4340bac..497cca7 100644 --- a/test/uk/gov/hmrc/upscaninitiate/util/UserAgentFilterSpec.scala +++ b/test/uk/gov/hmrc/upscaninitiate/util/UserAgentFilterSpec.scala @@ -30,43 +30,40 @@ import uk.gov.hmrc.upscaninitiate.test.UnitSpec import scala.concurrent.Future import scala.concurrent.duration._ -class UserAgentFilterSpec extends UnitSpec with GivenWhenThen { +class UserAgentFilterSpec extends UnitSpec with GivenWhenThen: import UserAgentFilterSpec._ - private implicit val timeout: Timeout = Timeout(3.seconds) + private given Timeout = 3.seconds - "UserAgentFilter" should { - "accept a request when the User-Agent header is specified" in { + "UserAgentFilter" should: + "accept a request when the User-Agent header is specified" in: Given("a request that specifies a User-Agent header") - val request = FakeRequest().withHeaders((USER_AGENT, SomeUserAgent)) + val request = FakeRequest().withHeaders((USER_AGENT, someUserAgent)) When("the request is received") - val result = UserAgentFilter.requireUserAgent(block)(request) + val result = UserAgentFilter.requireUserAgent(block)(using request) Then("the request should be accepted") status(result) shouldBe OK And("the User-Agent header value should be passed to the block") - contentAsString(result) shouldBe SomeUserAgent - } + contentAsString(result) shouldBe someUserAgent - "reject a request when the User-Agent header is not specified" in { + "reject a request when the User-Agent header is not specified" in: Given("a request that does not specify a User-Agent header") val request = FakeRequest() When("the request is received") - val result = UserAgentFilter.requireUserAgent(block)(request) + val result = UserAgentFilter.requireUserAgent(block)(using request) Then("the request should be rejected") status(result) shouldBe BAD_REQUEST - } - } -} -private object UserAgentFilterSpec { - val SomeUserAgent = "SOME_USER-AGENT" - val block: (Request[_], String) => Future[Result] = (_, userAgent) => Future.successful(Ok(userAgent)) +private object UserAgentFilterSpec: + val someUserAgent = "SOME_USER-AGENT" + + val block: (Request[_], String) => Future[Result] = + (_, userAgent) => Future.successful(Ok(userAgent)) object UserAgentFilter extends Logging with UserAgentFilter -} From a629ebf513e1d6369ccda56494682ad78eab871b Mon Sep 17 00:00:00 2001 From: colin-lamed <9568290+colin-lamed@users.noreply.github.com> Date: Thu, 24 Oct 2024 10:52:07 +0100 Subject: [PATCH 84/96] BDOG-3255 Inject MetricRegistry rather than Metrics wrapper --- .../service/PrepareUploadService.scala | 10 +- conf/application.conf | 15 +- .../model/PrepareUploadServiceSpec.scala | 132 +++++++----------- 3 files changed, 60 insertions(+), 97 deletions(-) diff --git a/app/uk/gov/hmrc/upscaninitiate/service/PrepareUploadService.scala b/app/uk/gov/hmrc/upscaninitiate/service/PrepareUploadService.scala index e6f2035..4051b49 100644 --- a/app/uk/gov/hmrc/upscaninitiate/service/PrepareUploadService.scala +++ b/app/uk/gov/hmrc/upscaninitiate/service/PrepareUploadService.scala @@ -16,9 +16,9 @@ package uk.gov.hmrc.upscaninitiate.service +import com.codahale.metrics.MetricRegistry import org.slf4j.MDC import play.api.Logging -import uk.gov.hmrc.play.bootstrap.metrics.Metrics import uk.gov.hmrc.upscaninitiate.config.ServiceConfiguration import uk.gov.hmrc.upscaninitiate.connector.model.{ContentLengthRange, UploadFormGenerator, UploadParameters} import uk.gov.hmrc.upscaninitiate.controller.model.{PreparedUploadResponse, Reference, UploadFormTemplate} @@ -30,9 +30,9 @@ import scala.jdk.DurationConverters.ScalaDurationOps @Singleton class PrepareUploadService @Inject()( - postSigner : UploadFormGenerator, - configuration: ServiceConfiguration, - metrics : Metrics + postSigner : UploadFormGenerator, + configuration : ServiceConfiguration, + metricRegistry: MetricRegistry ) extends Logging: def prepareUpload( @@ -62,7 +62,7 @@ class PrepareUploadService @Inject()( MDC.put("file-reference", reference.toString) logger.info(s"Allocated key=[${reference.value}] to uploadRequest with requestId=[$requestId] sessionId=[$sessionId] from [${settings.consumingService}].") logger.debug(s"Prepared upload response [$result].") - metrics.defaultRegistry.counter("uploadInitiated").inc() + metricRegistry.counter("uploadInitiated").inc() result finally diff --git a/conf/application.conf b/conf/application.conf index af00491..2c9249e 100644 --- a/conf/application.conf +++ b/conf/application.conf @@ -16,7 +16,7 @@ # ~~~~~ include "backend.conf" -appName=upscan-initiate +appName = upscan-initiate play.modules.enabled += "uk.gov.hmrc.upscaninitiate.UpscanInitiateModule" play.modules.enabled += "uk.gov.hmrc.upscaninitiate.connector.s3.S3Module" @@ -31,14 +31,9 @@ play.modules.enabled += "uk.gov.hmrc.upscaninitiate.connector.s3.S3Module" # you may need to define a router file `conf/my.application.routes`. # Default to Routes in the root package (and conf/routes) # !!!WARNING!!! DO NOT CHANGE THIS ROUTER -play.http.router=prod.Routes +play.http.router = prod.Routes -# Metrics plugin settings - graphite reporting is configured on a per env basis -metrics { - enabled = true -} - uploadProxy.url = "ENTER UPLOAD PROXY URL" aws { @@ -49,11 +44,11 @@ aws { upload.link.validity.duration = "7 days" } - accessKeyId = "ENTER YOUR KEY" + accessKeyId = "ENTER YOUR KEY" secretAccessKey = "ENYER YOUR SECRET" - accessKeyId = ${?AWS_ACCESS_KEY_ID} + accessKeyId = ${?AWS_ACCESS_KEY_ID} secretAccessKey = ${?AWS_SECRET_ACCESS_KEY} - sessionToken = ${?AWS_SESSION_TOKEN} + sessionToken = ${?AWS_SESSION_TOKEN} } callbackValidation.allowedProtocols = "https" diff --git a/test/uk/gov/hmrc/upscaninitiate/model/PrepareUploadServiceSpec.scala b/test/uk/gov/hmrc/upscaninitiate/model/PrepareUploadServiceSpec.scala index 4aee1fe..4a4626c 100644 --- a/test/uk/gov/hmrc/upscaninitiate/model/PrepareUploadServiceSpec.scala +++ b/test/uk/gov/hmrc/upscaninitiate/model/PrepareUploadServiceSpec.scala @@ -17,8 +17,8 @@ package uk.gov.hmrc.upscaninitiate.model import com.codahale.metrics.MetricRegistry +import org.mockito.Mockito.when import org.scalatest.GivenWhenThen -import uk.gov.hmrc.play.bootstrap.metrics.Metrics import uk.gov.hmrc.upscaninitiate.config.ServiceConfiguration import uk.gov.hmrc.upscaninitiate.connector.model.{UploadFormGenerator, UploadParameters} import uk.gov.hmrc.upscaninitiate.controller.model.PrepareUploadRequest @@ -28,64 +28,21 @@ import uk.gov.hmrc.upscaninitiate.service.model.UploadSettings import uk.gov.hmrc.upscaninitiate.test.UnitSpec import java.time.Instant -import scala.concurrent.duration.{DurationInt, FiniteDuration} +import scala.concurrent.duration.DurationInt class PrepareUploadServiceSpec extends UnitSpec with GivenWhenThen: - private val serviceConfiguration = - new ServiceConfiguration: - override def accessKeyId: String = ??? - - override def secretAccessKey: String = ??? - - override def uploadProxyUrl: String = ??? - - override def inboundBucketName: String = "test-bucket" - - override def sessionToken: Option[String] = ??? - - override def region: String = ??? - - override def fileExpirationPeriod: FiniteDuration = 7.days - - override def globalFileSizeLimit = 1024 - - override def allowedCallbackProtocols: List[String] = List("https") - - private def metricsStub() = - new Metrics: - override val defaultRegistry: MetricRegistry = MetricRegistry() - - private val s3PostSigner = new UploadFormGenerator: - override def generateFormFields(uploadParameters: UploadParameters): Map[String, String] = - Map( - "bucket" -> uploadParameters.bucketName, - "key" -> uploadParameters.objectKey, - "minSize" -> uploadParameters.contentLengthRange.min.toString, - "maxSize" -> uploadParameters.contentLengthRange.max.toString - ) - ++ uploadParameters.additionalMetadata.map { case (k, v) => s"x-amz-meta-$k" -> v } - ++ uploadParameters.successRedirect.map { "success_redirect_url" -> _ } - ++ uploadParameters.errorRedirect.map { "error_redirect_url" -> _ } - val receivedAt: Instant = Instant.now() "S3 Prepare Upload Service" should: - - def service(metrics: Metrics) = - PrepareUploadService(s3PostSigner, serviceConfiguration, metrics) - - "create post form that allows to upload the file" in: - val metrics = metricsStub() - + "create post form that allows to upload the file" in new Setup: Given("there are valid upload settings") - val settings = - settingsTemplate + val settings = settingsTemplate When("we setup the upload") val result = - service(metrics) + service .prepareUpload( settings, "some-request-id", @@ -109,11 +66,9 @@ class PrepareUploadServiceSpec extends UnitSpec with GivenWhenThen: ) And("uploadInitiated counter has been incremented") - metrics.defaultRegistry.counter("uploadInitiated").getCount shouldBe 1 - - "take in account file size limits if provided in request" in: - val metrics = metricsStub() + metricRegistry.counter("uploadInitiated").getCount shouldBe 1 + "take in account file size limits if provided in request" in new Setup: Given("there are valid upload settings with size limits") val settings = settingsTemplate @@ -126,18 +81,14 @@ class PrepareUploadServiceSpec extends UnitSpec with GivenWhenThen: ) When("we setup the upload") - - val result = service(metrics) - .prepareUpload(settings, "some-request-id", "some-session-id", receivedAt) + val result = service.prepareUpload(settings, "some-request-id", "some-session-id", receivedAt) Then("upload request should contain requested min/max size") result.uploadRequest.fields("minSize") shouldBe "100" result.uploadRequest.fields("maxSize") shouldBe "200" - "fail when minimum file size is less than 0" in: - val metrics = metricsStub() - + "fail when minimum file size is less than 0" in new Setup: Given("there are upload settings with minimum file size less than zero") val settings = settingsTemplate @@ -152,15 +103,13 @@ class PrepareUploadServiceSpec extends UnitSpec with GivenWhenThen: When("we setup the upload") Then("an exception should be thrown") - val thrown = the[IllegalArgumentException] thrownBy service(metrics) + val thrown = the[IllegalArgumentException] thrownBy service .prepareUpload(settings, "some-request-id", "some-session-id", receivedAt) thrown.getMessage should include("Minimum file size is less than 0") - metrics.defaultRegistry.counter("uploadInitiated").getCount shouldBe 0 - - "fail when maximum file size is greater than global limit" in: - val metrics = metricsStub() + metricRegistry.counter("uploadInitiated").getCount shouldBe 0 + "fail when maximum file size is greater than global limit" in new Setup: Given("there upload settings with maximum file size greater than global limit") val settings = settingsTemplate @@ -174,15 +123,13 @@ class PrepareUploadServiceSpec extends UnitSpec with GivenWhenThen: When("we setup the upload") Then("an exception should be thrown") - val thrown = the[IllegalArgumentException] thrownBy service(metrics) + val thrown = the[IllegalArgumentException] thrownBy service .prepareUpload(settings, "some-request-id", "some-session-id", receivedAt) thrown.getMessage should include("Maximum file size is greater than global maximum file size") - metrics.defaultRegistry.counter("uploadInitiated").getCount shouldBe 0 - - "fail when minimum file size is greater than maximum file size" in: - val metrics = metricsStub() + metricRegistry.counter("uploadInitiated").getCount shouldBe 0 + "fail when minimum file size is greater than maximum file size" in new Setup: Given("there are upload settings with minimum file size greater than maximum size") val settings = settingsTemplate @@ -196,15 +143,13 @@ class PrepareUploadServiceSpec extends UnitSpec with GivenWhenThen: When("we setup the upload") Then("an exception should be thrown") - val thrown = the[IllegalArgumentException] thrownBy service(metrics) + val thrown = the[IllegalArgumentException] thrownBy service .prepareUpload(settings, "some-request-id", "some-session-id", receivedAt) thrown.getMessage should include("Minimum file size is greater than maximum file size") - metrics.defaultRegistry.counter("uploadInitiated").getCount shouldBe 0 - - "create post form that allows to upload the file with redirect on success" in: - val metrics = metricsStub() + metricRegistry.counter("uploadInitiated").getCount shouldBe 0 + "create post form that allows to upload the file with redirect on success" in new Setup: Given("there are valid upload settings") val settings = settingsTemplate @@ -217,8 +162,7 @@ class PrepareUploadServiceSpec extends UnitSpec with GivenWhenThen: When("we setup the upload") - val result = service(metrics) - .prepareUpload(settings, "some-request-id", "some-session-id", receivedAt) + val result = service.prepareUpload(settings, "some-request-id", "some-session-id", receivedAt) Then("proper upload request form definition should be returned") result.uploadRequest.href shouldBe settings.uploadUrl @@ -236,11 +180,9 @@ class PrepareUploadServiceSpec extends UnitSpec with GivenWhenThen: ) And("uploadInitiated counter has been incremented") - metrics.defaultRegistry.counter("uploadInitiated").getCount shouldBe 1 - - "create post form that allows to upload the file with redirect on error" in: - val metrics = metricsStub() + metricRegistry.counter("uploadInitiated").getCount shouldBe 1 + "create post form that allows to upload the file with redirect on error" in new Setup: Given("there are valid upload settings") val settings = settingsTemplate @@ -252,8 +194,7 @@ class PrepareUploadServiceSpec extends UnitSpec with GivenWhenThen: ) When("we setup the upload") - val result = service(metrics) - .prepareUpload(settings, "some-request-id", "some-session-id", receivedAt) + val result = service.prepareUpload(settings, "some-request-id", "some-session-id", receivedAt) Then("proper upload request form definition should be returned") result.uploadRequest.href shouldBe settings.uploadUrl @@ -271,7 +212,34 @@ class PrepareUploadServiceSpec extends UnitSpec with GivenWhenThen: ) And("uploadInitiated counter has been incremented") - metrics.defaultRegistry.counter("uploadInitiated").getCount shouldBe 1 + metricRegistry.counter("uploadInitiated").getCount shouldBe 1 + + trait Setup: + val serviceConfiguration = mock[ServiceConfiguration] + when(serviceConfiguration.inboundBucketName) + .thenReturn("test-bucket") + when(serviceConfiguration.fileExpirationPeriod) + .thenReturn(7.days) + when(serviceConfiguration.globalFileSizeLimit) + .thenReturn(1024L) + when(serviceConfiguration.allowedCallbackProtocols) + .thenReturn(List("https")) + + val s3PostSigner = new UploadFormGenerator: + override def generateFormFields(uploadParameters: UploadParameters): Map[String, String] = + Map( + "bucket" -> uploadParameters.bucketName, + "key" -> uploadParameters.objectKey, + "minSize" -> uploadParameters.contentLengthRange.min.toString, + "maxSize" -> uploadParameters.contentLengthRange.max.toString + ) + ++ uploadParameters.additionalMetadata.map { case (k, v) => s"x-amz-meta-$k" -> v } + ++ uploadParameters.successRedirect.map { "success_redirect_url" -> _ } + ++ uploadParameters.errorRedirect.map { "error_redirect_url" -> _ } + + + val metricRegistry = MetricRegistry() + val service = PrepareUploadService(s3PostSigner, serviceConfiguration, metricRegistry) object PrepareUploadServiceSpec { From 61137e146f3b38942860115ba827c3e93f12a20f Mon Sep 17 00:00:00 2001 From: colin-lamed <9568290+colin-lamed@users.noreply.github.com> Date: Thu, 24 Oct 2024 13:26:17 +0100 Subject: [PATCH 85/96] BDOG-3255 Drop dependency on aws sdk --- .../hmrc/upscaninitiate/connector/s3/PolicySigner.scala | 9 +++++---- project/AppDependencies.scala | 6 ++---- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/app/uk/gov/hmrc/upscaninitiate/connector/s3/PolicySigner.scala b/app/uk/gov/hmrc/upscaninitiate/connector/s3/PolicySigner.scala index 27d64a7..7a49a85 100644 --- a/app/uk/gov/hmrc/upscaninitiate/connector/s3/PolicySigner.scala +++ b/app/uk/gov/hmrc/upscaninitiate/connector/s3/PolicySigner.scala @@ -16,7 +16,6 @@ package uk.gov.hmrc.upscaninitiate.connector.s3 -import com.amazonaws.util.{BinaryUtils, StringUtils} import uk.gov.hmrc.upscaninitiate.connector.model.AwsCredentials import java.nio.charset.Charset @@ -41,7 +40,7 @@ object PolicySigner extends PolicySigner: ): String = val signingKey = newSigningKey(credentials, formattedSigningDate, regionName, "s3") val policySignature = sign(encodedPolicy, signingKey) - BinaryUtils.toHex(policySignature) + toHex(policySignature) private def newSigningKey( credentials: AwsCredentials, @@ -56,8 +55,10 @@ object PolicySigner extends PolicySigner: sign("aws4_request", kService) private def sign(stringData: String, key: Array[Byte]): Array[Byte] = - val data = stringData.getBytes(StringUtils.UTF8) val algorithm = "HmacSHA256" val mac = Mac.getInstance(algorithm) mac.init(SecretKeySpec(key, algorithm)) - mac.doFinal(data) + mac.doFinal(stringData.getBytes("UTF-8")) + + private def toHex(bytes: Array[Byte]): String = + java.lang.String.format("%032x", new java.math.BigInteger(1, bytes)) diff --git a/project/AppDependencies.scala b/project/AppDependencies.scala index 16e430b..c121bb7 100644 --- a/project/AppDependencies.scala +++ b/project/AppDependencies.scala @@ -4,13 +4,11 @@ object AppDependencies { private val bootstrapVersion = "9.5.0" private val compile = Seq( - "uk.gov.hmrc" %% "bootstrap-backend-play-30" % bootstrapVersion, - "com.amazonaws" % "aws-java-sdk-s3" % "1.12.606", - "org.apache.commons" % "commons-lang3" % "3.12.0" + "uk.gov.hmrc" %% "bootstrap-backend-play-30" % bootstrapVersion ) private val test = Seq( - "uk.gov.hmrc" %% "bootstrap-test-play-30" % bootstrapVersion % Test + "uk.gov.hmrc" %% "bootstrap-test-play-30" % bootstrapVersion % Test ) def apply(): Seq[ModuleID] = compile ++ test From 934e862538f310c8574454aee46392eefef39e23 Mon Sep 17 00:00:00 2001 From: colin-lamed <9568290+colin-lamed@users.noreply.github.com> Date: Thu, 24 Oct 2024 13:44:00 +0100 Subject: [PATCH 86/96] BDOG-3255 Pr comments --- build.sbt | 8 ++------ conf/application-json-logger.xml | 15 --------------- conf/logback.xml | 11 ----------- 3 files changed, 2 insertions(+), 32 deletions(-) delete mode 100644 conf/application-json-logger.xml diff --git a/build.sbt b/build.sbt index 3d0f609..2ff3777 100644 --- a/build.sbt +++ b/build.sbt @@ -3,10 +3,6 @@ import sbt.Keys._ import sbt._ import scoverage.ScoverageKeys import uk.gov.hmrc.DefaultBuildSettings -import uk.gov.hmrc.sbtdistributables.SbtDistributablesPlugin - - -val appName = "upscan-initiate" ThisBuild / majorVersion := 0 ThisBuild / scalaVersion := "3.3.4" @@ -23,8 +19,8 @@ lazy val scoverageSettings = Seq( ScoverageKeys.coverageHighlighting := true ) -lazy val microservice = Project(appName, file(".")) - .enablePlugins(play.sbt.PlayScala, SbtAutoBuildPlugin, SbtDistributablesPlugin) +lazy val microservice = Project("upscan-initiate", file(".")) + .enablePlugins(PlayScala, SbtDistributablesPlugin) .disablePlugins(JUnitXmlReportPlugin) //Required to prevent https://github.com/scalatest/scalatest/issues/1427 .settings(scoverageSettings: _*) .settings(scalacOptions += "-Wconf:src=routes/.*:s") diff --git a/conf/application-json-logger.xml b/conf/application-json-logger.xml deleted file mode 100644 index 3a22bb0..0000000 --- a/conf/application-json-logger.xml +++ /dev/null @@ -1,15 +0,0 @@ - - - - - - - - - - - - - - - diff --git a/conf/logback.xml b/conf/logback.xml index ff38ea5..636877b 100644 --- a/conf/logback.xml +++ b/conf/logback.xml @@ -14,13 +14,6 @@ - - logs/access.log - - %message%n - - - logs/connector.log @@ -29,10 +22,6 @@ - - - - From 3f31204a586136762b526f9817d6ed2fd44f2530 Mon Sep 17 00:00:00 2001 From: colin-lamed <9568290+colin-lamed@users.noreply.github.com> Date: Thu, 24 Oct 2024 17:24:49 +0100 Subject: [PATCH 87/96] BDOG-3255 Use toHex implementation from aws sdk --- .../gov/hmrc/upscaninitiate/connector/s3/PolicySigner.scala | 6 ++---- project/AppDependencies.scala | 5 +++-- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/app/uk/gov/hmrc/upscaninitiate/connector/s3/PolicySigner.scala b/app/uk/gov/hmrc/upscaninitiate/connector/s3/PolicySigner.scala index 7a49a85..0af2ccf 100644 --- a/app/uk/gov/hmrc/upscaninitiate/connector/s3/PolicySigner.scala +++ b/app/uk/gov/hmrc/upscaninitiate/connector/s3/PolicySigner.scala @@ -16,6 +16,7 @@ package uk.gov.hmrc.upscaninitiate.connector.s3 +import software.amazon.awssdk.utils.BinaryUtils import uk.gov.hmrc.upscaninitiate.connector.model.AwsCredentials import java.nio.charset.Charset @@ -40,7 +41,7 @@ object PolicySigner extends PolicySigner: ): String = val signingKey = newSigningKey(credentials, formattedSigningDate, regionName, "s3") val policySignature = sign(encodedPolicy, signingKey) - toHex(policySignature) + BinaryUtils.toHex(policySignature) private def newSigningKey( credentials: AwsCredentials, @@ -59,6 +60,3 @@ object PolicySigner extends PolicySigner: val mac = Mac.getInstance(algorithm) mac.init(SecretKeySpec(key, algorithm)) mac.doFinal(stringData.getBytes("UTF-8")) - - private def toHex(bytes: Array[Byte]): String = - java.lang.String.format("%032x", new java.math.BigInteger(1, bytes)) diff --git a/project/AppDependencies.scala b/project/AppDependencies.scala index c121bb7..e271f04 100644 --- a/project/AppDependencies.scala +++ b/project/AppDependencies.scala @@ -4,11 +4,12 @@ object AppDependencies { private val bootstrapVersion = "9.5.0" private val compile = Seq( - "uk.gov.hmrc" %% "bootstrap-backend-play-30" % bootstrapVersion + "uk.gov.hmrc" %% "bootstrap-backend-play-30" % bootstrapVersion, + "software.amazon.awssdk" % "s3" % "2.28.19" ) private val test = Seq( - "uk.gov.hmrc" %% "bootstrap-test-play-30" % bootstrapVersion % Test + "uk.gov.hmrc" %% "bootstrap-test-play-30" % bootstrapVersion % Test ) def apply(): Seq[ModuleID] = compile ++ test From 5013289a47ee31bbfab6c594a452bf14e08c867f Mon Sep 17 00:00:00 2001 From: macgrewal <7713555+macgrewal@users.noreply.github.com> Date: Thu, 9 Jan 2025 10:59:19 +0000 Subject: [PATCH 88/96] BDOG-3271: Updated max upload size to 1024MB --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 73d998c..45065b6 100644 --- a/README.md +++ b/README.md @@ -250,7 +250,7 @@ Example request: |--------------|-----------|--------| |callbackUrl |Url that will be called to report the outcome of file checking and upload, including retrieval details if successful. Notification format is detailed further down in this file. Must be https.| yes| |minimumFileSize|Minimum file size (in Bytes). Default is 0.|no| -|maximumFileSize|Maximum file size (in Bytes). Cannot be greater than 100MB. Default is 100MB.|no| +|maximumFileSize|Maximum file size (in Bytes). Cannot be greater than 1024MB. Default is 1024MB.|no| |successRedirect|Url to redirect to after file has been successfully uploaded.|no| From 6f4e8b69af8825c75b312704fbb2e7c8eeb1f2af Mon Sep 17 00:00:00 2001 From: macgrewal <7713555+macgrewal@users.noreply.github.com> Date: Thu, 9 Jan 2025 13:15:57 +0000 Subject: [PATCH 89/96] BDOG-3271: Changed default max file size to 100MB whilst allowing files up to 1GB --- README.md | 4 ++-- .../upscaninitiate/config/ServiceConfiguration.scala | 10 +++++++--- .../controller/PrepareUploadController.scala | 4 ++-- .../upscaninitiate/service/PrepareUploadService.scala | 11 +++++++---- conf/application.conf | 5 ++++- .../controller/PrepareUploadControllerSpec.scala | 11 +++++++---- .../model/PrepareUploadServiceSpec.scala | 10 ++++++---- 7 files changed, 35 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 45065b6..be00800 100644 --- a/README.md +++ b/README.md @@ -146,7 +146,7 @@ Another example `upscan/v2/initiate` requests that explicitly identifies the con |successRedirect|Url to redirect to after file has been successfully uploaded.|no| |errorRedirect|Url to redirect to if error encountered during upload.|no| |minimumFileSize|Minimum file size (in Bytes). Default is 0.|no| -|maximumFileSize|Maximum file size (in Bytes). Cannot be greater than 100MB. Default is 100MB.|no| +|maximumFileSize|Maximum file size (in Bytes). Cannot be greater than 1024MB. Default is 100MB.|no| Example response @@ -250,7 +250,7 @@ Example request: |--------------|-----------|--------| |callbackUrl |Url that will be called to report the outcome of file checking and upload, including retrieval details if successful. Notification format is detailed further down in this file. Must be https.| yes| |minimumFileSize|Minimum file size (in Bytes). Default is 0.|no| -|maximumFileSize|Maximum file size (in Bytes). Cannot be greater than 1024MB. Default is 1024MB.|no| +|maximumFileSize|Maximum file size (in Bytes). Cannot be greater than 1024MB. Default is 100MB.|no| |successRedirect|Url to redirect to after file has been successfully uploaded.|no| diff --git a/app/uk/gov/hmrc/upscaninitiate/config/ServiceConfiguration.scala b/app/uk/gov/hmrc/upscaninitiate/config/ServiceConfiguration.scala index 4c05471..94b6d8e 100644 --- a/app/uk/gov/hmrc/upscaninitiate/config/ServiceConfiguration.scala +++ b/app/uk/gov/hmrc/upscaninitiate/config/ServiceConfiguration.scala @@ -29,7 +29,8 @@ trait ServiceConfiguration: def accessKeyId : String def secretAccessKey : String def fileExpirationPeriod : FiniteDuration - def globalFileSizeLimit : Long + def maxFileSizeLimit : Long + def defaultMaxFileSize : Long def allowedCallbackProtocols: Seq[String] class PlayBasedServiceConfiguration @Inject()( @@ -57,8 +58,11 @@ class PlayBasedServiceConfiguration @Inject()( override val sessionToken: Option[String] = configuration.getOptional[String]("aws.sessionToken") - override val globalFileSizeLimit: Long = - configuration.get[Long]("global.file.size.limit") + override val maxFileSizeLimit: Long = + configuration.get[Long]("maxFileSize.limit") + + override val defaultMaxFileSize: Long = + configuration.get[Long]("maxFileSize.default") override val allowedCallbackProtocols: Seq[String] = configuration diff --git a/app/uk/gov/hmrc/upscaninitiate/controller/PrepareUploadController.scala b/app/uk/gov/hmrc/upscaninitiate/controller/PrepareUploadController.scala index 50e697c..7b0bde9 100644 --- a/app/uk/gov/hmrc/upscaninitiate/controller/PrepareUploadController.scala +++ b/app/uk/gov/hmrc/upscaninitiate/controller/PrepareUploadController.scala @@ -46,14 +46,14 @@ class PrepareUploadController @Inject()( * V1 of the API supports direct upload to an S3 bucket and *does not support* error redirects in the event of failure */ def prepareUploadV1: Action[JsValue] = - given Reads[PrepareUploadRequest] = PrepareUploadRequest.readsV1(prepareUploadService.globalFileSizeLimit) + given Reads[PrepareUploadRequest] = PrepareUploadRequest.readsV1(prepareUploadService.maxFileSizeLimit) prepareUpload(uploadUrl = s"https://${configuration.inboundBucketName}.s3.amazonaws.com") /** * V2 of the API supports upload to an S3 bucket via a proxy that additionally supports error redirects in the event of failure */ def prepareUploadV2: Action[JsValue] = - given Reads[PrepareUploadRequest] = PrepareUploadRequest.readsV2(prepareUploadService.globalFileSizeLimit) + given Reads[PrepareUploadRequest] = PrepareUploadRequest.readsV2(prepareUploadService.maxFileSizeLimit) prepareUpload(uploadUrl = s"${configuration.uploadProxyUrl}/v1/uploads/${configuration.inboundBucketName}") private def prepareUpload( diff --git a/app/uk/gov/hmrc/upscaninitiate/service/PrepareUploadService.scala b/app/uk/gov/hmrc/upscaninitiate/service/PrepareUploadService.scala index 4051b49..bb4a938 100644 --- a/app/uk/gov/hmrc/upscaninitiate/service/PrepareUploadService.scala +++ b/app/uk/gov/hmrc/upscaninitiate/service/PrepareUploadService.scala @@ -78,10 +78,10 @@ class PrepareUploadService @Inject()( ): UploadFormTemplate = val minFileSize = settings.prepareUploadRequest.minimumFileSize.getOrElse(0L) - val maxFileSize = settings.prepareUploadRequest.maximumFileSize.getOrElse(globalFileSizeLimit) + val maxFileSize = settings.prepareUploadRequest.maximumFileSize.getOrElse(defaultMaxFileSize) require(minFileSize >= 0 , "Minimum file size is less than 0") - require(maxFileSize <= globalFileSizeLimit, "Maximum file size is greater than global maximum file size") + require(maxFileSize <= maxFileSizeLimit , "Maximum file size is greater than global maximum file size") require(minFileSize <= maxFileSize , "Minimum file size is greater than maximum file size") val uploadParameters = UploadParameters( @@ -105,7 +105,10 @@ class PrepareUploadService @Inject()( UploadFormTemplate(settings.uploadUrl, form) - def globalFileSizeLimit: Long = - configuration.globalFileSizeLimit + def maxFileSizeLimit: Long = + configuration.maxFileSizeLimit + + def defaultMaxFileSize: Long = + configuration.defaultMaxFileSize end PrepareUploadService diff --git a/conf/application.conf b/conf/application.conf index 2c9249e..8585c06 100644 --- a/conf/application.conf +++ b/conf/application.conf @@ -53,4 +53,7 @@ aws { callbackValidation.allowedProtocols = "https" -global.file.size.limit = 104857600 +maxFileSize { + limit = 1073741824 # 1024MB + default = 104857600 # 100MB +} \ No newline at end of file diff --git a/test/uk/gov/hmrc/upscaninitiate/controller/PrepareUploadControllerSpec.scala b/test/uk/gov/hmrc/upscaninitiate/controller/PrepareUploadControllerSpec.scala index d83117e..c59d9b3 100644 --- a/test/uk/gov/hmrc/upscaninitiate/controller/PrepareUploadControllerSpec.scala +++ b/test/uk/gov/hmrc/upscaninitiate/controller/PrepareUploadControllerSpec.scala @@ -50,10 +50,13 @@ class PrepareUploadControllerSpec private val clock = Clock.fixed(Clock.systemDefaultZone().instant(), Clock.systemDefaultZone().getZone) private trait WithGlobalFileSizeLimitFixture { - val globalFileSizeLimit = 1024L + val maxFileSizeLimit = 1024L + val defaultMaxFileSize = 100L val prepareUploadService = mock[PrepareUploadService] - when(prepareUploadService.globalFileSizeLimit) - .thenReturn(globalFileSizeLimit) + when(prepareUploadService.maxFileSizeLimit) + .thenReturn(maxFileSizeLimit) + when(prepareUploadService.defaultMaxFileSize) + .thenReturn(defaultMaxFileSize) } private trait WithServiceConfiguration: @@ -293,7 +296,7 @@ class PrepareUploadControllerSpec ) .withBody(Json.obj( "callbackUrl" -> "https://www.example.com", - "maximumFileSize" -> (globalFileSizeLimit + 1)) + "maximumFileSize" -> (maxFileSizeLimit + 1)) ) When("upload initiation has been requested") diff --git a/test/uk/gov/hmrc/upscaninitiate/model/PrepareUploadServiceSpec.scala b/test/uk/gov/hmrc/upscaninitiate/model/PrepareUploadServiceSpec.scala index 4a4626c..092b6e8 100644 --- a/test/uk/gov/hmrc/upscaninitiate/model/PrepareUploadServiceSpec.scala +++ b/test/uk/gov/hmrc/upscaninitiate/model/PrepareUploadServiceSpec.scala @@ -61,7 +61,7 @@ class PrepareUploadServiceSpec extends UnitSpec with GivenWhenThen: "x-amz-meta-session-id" -> "some-session-id", "x-amz-meta-request-id" -> "some-request-id", "minSize" -> "0", - "maxSize" -> "1024", + "maxSize" -> "100", "x-amz-meta-upscan-initiate-received" -> receivedAt.toString ) @@ -174,7 +174,7 @@ class PrepareUploadServiceSpec extends UnitSpec with GivenWhenThen: "x-amz-meta-session-id" -> "some-session-id", "x-amz-meta-request-id" -> "some-request-id", "minSize" -> "0", - "maxSize" -> "1024", + "maxSize" -> "100", "x-amz-meta-upscan-initiate-received" -> receivedAt.toString, "success_redirect_url" -> settings.prepareUploadRequest.successRedirect.get ) @@ -206,7 +206,7 @@ class PrepareUploadServiceSpec extends UnitSpec with GivenWhenThen: "x-amz-meta-session-id" -> "some-session-id", "x-amz-meta-request-id" -> "some-request-id", "minSize" -> "0", - "maxSize" -> "1024", + "maxSize" -> "100", "x-amz-meta-upscan-initiate-received" -> receivedAt.toString, "error_redirect_url" -> settings.prepareUploadRequest.errorRedirect.get ) @@ -220,8 +220,10 @@ class PrepareUploadServiceSpec extends UnitSpec with GivenWhenThen: .thenReturn("test-bucket") when(serviceConfiguration.fileExpirationPeriod) .thenReturn(7.days) - when(serviceConfiguration.globalFileSizeLimit) + when(serviceConfiguration.maxFileSizeLimit) .thenReturn(1024L) + when(serviceConfiguration.defaultMaxFileSize) + .thenReturn(100L) when(serviceConfiguration.allowedCallbackProtocols) .thenReturn(List("https")) From b751142cbf88b5c7992efc3e3f69b60b83b722a5 Mon Sep 17 00:00:00 2001 From: macgrewal <7713555+macgrewal@users.noreply.github.com> Date: Thu, 9 Jan 2025 13:25:24 +0000 Subject: [PATCH 90/96] Default and Max file sizes set to 1GB --- README.md | 4 ++-- conf/application.conf | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index be00800..5c5a46e 100644 --- a/README.md +++ b/README.md @@ -146,7 +146,7 @@ Another example `upscan/v2/initiate` requests that explicitly identifies the con |successRedirect|Url to redirect to after file has been successfully uploaded.|no| |errorRedirect|Url to redirect to if error encountered during upload.|no| |minimumFileSize|Minimum file size (in Bytes). Default is 0.|no| -|maximumFileSize|Maximum file size (in Bytes). Cannot be greater than 1024MB. Default is 100MB.|no| +|maximumFileSize|Maximum file size (in Bytes). Cannot be greater than 1024MB. Default is 1024MB.|no| Example response @@ -250,7 +250,7 @@ Example request: |--------------|-----------|--------| |callbackUrl |Url that will be called to report the outcome of file checking and upload, including retrieval details if successful. Notification format is detailed further down in this file. Must be https.| yes| |minimumFileSize|Minimum file size (in Bytes). Default is 0.|no| -|maximumFileSize|Maximum file size (in Bytes). Cannot be greater than 1024MB. Default is 100MB.|no| +|maximumFileSize|Maximum file size (in Bytes). Cannot be greater than 1024MB. Default is 1024MB.|no| |successRedirect|Url to redirect to after file has been successfully uploaded.|no| diff --git a/conf/application.conf b/conf/application.conf index 8585c06..83f80a4 100644 --- a/conf/application.conf +++ b/conf/application.conf @@ -55,5 +55,5 @@ callbackValidation.allowedProtocols = "https" maxFileSize { limit = 1073741824 # 1024MB - default = 104857600 # 100MB + default = 1073741824 # 1024MB } \ No newline at end of file From 6caf1276d5852f15ab1ac99f1a7305099dfcdab2 Mon Sep 17 00:00:00 2001 From: macgrewal <7713555+macgrewal@users.noreply.github.com> Date: Fri, 10 Jan 2025 10:58:34 +0000 Subject: [PATCH 91/96] BDOG-3271: Updated config/documentation to reflect new max/default filesizes --- README.md | 4 ++-- conf/application.conf | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 5c5a46e..be00800 100644 --- a/README.md +++ b/README.md @@ -146,7 +146,7 @@ Another example `upscan/v2/initiate` requests that explicitly identifies the con |successRedirect|Url to redirect to after file has been successfully uploaded.|no| |errorRedirect|Url to redirect to if error encountered during upload.|no| |minimumFileSize|Minimum file size (in Bytes). Default is 0.|no| -|maximumFileSize|Maximum file size (in Bytes). Cannot be greater than 1024MB. Default is 1024MB.|no| +|maximumFileSize|Maximum file size (in Bytes). Cannot be greater than 1024MB. Default is 100MB.|no| Example response @@ -250,7 +250,7 @@ Example request: |--------------|-----------|--------| |callbackUrl |Url that will be called to report the outcome of file checking and upload, including retrieval details if successful. Notification format is detailed further down in this file. Must be https.| yes| |minimumFileSize|Minimum file size (in Bytes). Default is 0.|no| -|maximumFileSize|Maximum file size (in Bytes). Cannot be greater than 1024MB. Default is 1024MB.|no| +|maximumFileSize|Maximum file size (in Bytes). Cannot be greater than 1024MB. Default is 100MB.|no| |successRedirect|Url to redirect to after file has been successfully uploaded.|no| diff --git a/conf/application.conf b/conf/application.conf index 83f80a4..8585c06 100644 --- a/conf/application.conf +++ b/conf/application.conf @@ -55,5 +55,5 @@ callbackValidation.allowedProtocols = "https" maxFileSize { limit = 1073741824 # 1024MB - default = 1073741824 # 1024MB + default = 104857600 # 100MB } \ No newline at end of file From 347fe4f3882f8babbacfd8fb3abefbffbe8e140e Mon Sep 17 00:00:00 2001 From: colin-lamed <9568290+colin-lamed@users.noreply.github.com> Date: Fri, 28 Feb 2025 17:21:19 +0000 Subject: [PATCH 92/96] Bump dependencies --- build.sbt | 2 +- project/AppDependencies.scala | 4 ++-- project/plugins.sbt | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/build.sbt b/build.sbt index 2ff3777..4eddccc 100644 --- a/build.sbt +++ b/build.sbt @@ -5,7 +5,7 @@ import scoverage.ScoverageKeys import uk.gov.hmrc.DefaultBuildSettings ThisBuild / majorVersion := 0 -ThisBuild / scalaVersion := "3.3.4" +ThisBuild / scalaVersion := "3.3.5" ThisBuild / scalacOptions += "-Wconf:msg=Flag.*repeatedly:s" lazy val scoverageSettings = Seq( diff --git a/project/AppDependencies.scala b/project/AppDependencies.scala index e271f04..5eef449 100644 --- a/project/AppDependencies.scala +++ b/project/AppDependencies.scala @@ -1,11 +1,11 @@ import sbt._ object AppDependencies { - private val bootstrapVersion = "9.5.0" + private val bootstrapVersion = "9.10.0" private val compile = Seq( "uk.gov.hmrc" %% "bootstrap-backend-play-30" % bootstrapVersion, - "software.amazon.awssdk" % "s3" % "2.28.19" + "software.amazon.awssdk" % "s3" % "2.30.30" ) private val test = Seq( diff --git a/project/plugins.sbt b/project/plugins.sbt index d0f92be..b438315 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -4,8 +4,8 @@ resolvers += MavenRepository("HMRC-open-artefacts-maven2", "https://open.artefac resolvers += Resolver.url("HMRC-open-artefacts-ivy2", url("https://open.artefacts.tax.service.gov.uk/ivy2"))( Resolver.ivyStylePatterns) -addSbtPlugin("org.playframework" % "sbt-plugin" % "3.0.5") -addSbtPlugin("uk.gov.hmrc" % "sbt-auto-build" % "3.22.0") -addSbtPlugin("uk.gov.hmrc" % "sbt-distributables" % "2.5.0") +addSbtPlugin("org.playframework" % "sbt-plugin" % "3.0.6") +addSbtPlugin("uk.gov.hmrc" % "sbt-auto-build" % "3.24.0") +addSbtPlugin("uk.gov.hmrc" % "sbt-distributables" % "2.6.0") addSbtPlugin("org.scoverage" % "sbt-scoverage" % "2.0.11") addSbtPlugin("org.scalastyle" %% "scalastyle-sbt-plugin" % "1.0.0" exclude("org.scala-lang.modules", "scala-xml_2.12")) From d376dfa057bb6bd8d0ceb471c685882c5ce21c7b Mon Sep 17 00:00:00 2001 From: Shnick <1132919+Shnick@users.noreply.github.com> Date: Mon, 3 Mar 2025 11:53:58 +0000 Subject: [PATCH 93/96] BDOG-3398 add Upscan digital service repository config --- repository.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/repository.yaml b/repository.yaml index f7c6ae2..64cc805 100644 --- a/repository.yaml +++ b/repository.yaml @@ -1 +1,3 @@ repoVisibility: public_0C3F0CE3E6E6448FAD341E7BFA50FCD333E06A20CFF05FCACE61154DDBBADF71 +digital-service: Upscan + From ee9dba1c0d8a9a762bde982b91dfd59cc3bb327b Mon Sep 17 00:00:00 2001 From: jordanrowe <37838443+jordanrowe@users.noreply.github.com> Date: Tue, 8 Apr 2025 16:30:31 +0100 Subject: [PATCH 94/96] BDOG-3426 retrieve aws credentials from secrets manager --- .gitignore | 5 ++- .../config/ServiceConfiguration.scala | 4 ++ .../connector/s3/S3Module.scala | 21 ++++++++++ .../s3/S3UploadFormGeneratorProvider.scala | 41 +++++++++++++++++-- conf/application.conf | 3 +- .../PrepareUploadControllerISpec.scala | 15 +++++++ project/AppDependencies.scala | 5 ++- project/build.properties | 2 +- 8 files changed, 87 insertions(+), 9 deletions(-) diff --git a/.gitignore b/.gitignore index 974d7fa..6ab70f8 100644 --- a/.gitignore +++ b/.gitignore @@ -31,11 +31,12 @@ tmp .idea_modules bin **/Scratchpad.scala - *.log t nohup.out nohup .metals/ .vscode/ - +.bloop/ +project/.bloop/ +project/metals.sbt diff --git a/app/uk/gov/hmrc/upscaninitiate/config/ServiceConfiguration.scala b/app/uk/gov/hmrc/upscaninitiate/config/ServiceConfiguration.scala index 94b6d8e..862743f 100644 --- a/app/uk/gov/hmrc/upscaninitiate/config/ServiceConfiguration.scala +++ b/app/uk/gov/hmrc/upscaninitiate/config/ServiceConfiguration.scala @@ -28,6 +28,7 @@ trait ServiceConfiguration: def sessionToken : Option[String] def accessKeyId : String def secretAccessKey : String + def secretArn : String def fileExpirationPeriod : FiniteDuration def maxFileSizeLimit : Long def defaultMaxFileSize : Long @@ -55,6 +56,9 @@ class PlayBasedServiceConfiguration @Inject()( override val secretAccessKey: String = configuration.get[String]("aws.secretAccessKey") + override val secretArn: String = + configuration.get[String]("aws.secretArn") + override val sessionToken: Option[String] = configuration.getOptional[String]("aws.sessionToken") diff --git a/app/uk/gov/hmrc/upscaninitiate/connector/s3/S3Module.scala b/app/uk/gov/hmrc/upscaninitiate/connector/s3/S3Module.scala index 94a9cc5..c2f0428 100644 --- a/app/uk/gov/hmrc/upscaninitiate/connector/s3/S3Module.scala +++ b/app/uk/gov/hmrc/upscaninitiate/connector/s3/S3Module.scala @@ -19,9 +19,30 @@ package uk.gov.hmrc.upscaninitiate.connector.s3 import play.api.inject.{Binding, Module} import play.api.{Configuration, Environment} import uk.gov.hmrc.upscaninitiate.connector.model.UploadFormGenerator +import uk.gov.hmrc.upscaninitiate.config.ServiceConfiguration +import org.apache.pekko.actor.ActorSystem +import javax.inject.{Inject, Provider} +import software.amazon.awssdk.auth.credentials.ContainerCredentialsProvider +import software.amazon.awssdk.services.secretsmanager.SecretsManagerClient +import software.amazon.awssdk.regions.Region class S3Module extends Module: override def bindings(environment: Environment, configuration: Configuration): Seq[Binding[_]] = Seq( + bind[SecretsManagerClient].toProvider[SecretsManagerClientProvider], bind[UploadFormGenerator].toProvider[S3UploadFormGeneratorProvider] ) + +class SecretsManagerClientProvider @Inject()( + configuration: ServiceConfiguration, + actorSystem : ActorSystem +) extends Provider[SecretsManagerClient]: + override def get(): SecretsManagerClient = + val containerCredentials = ContainerCredentialsProvider.builder().build() + val client = + SecretsManagerClient.builder() + .credentialsProvider(containerCredentials) + .region(Region.of(configuration.region)) + .build() + actorSystem.registerOnTermination(client.close()) + client diff --git a/app/uk/gov/hmrc/upscaninitiate/connector/s3/S3UploadFormGeneratorProvider.scala b/app/uk/gov/hmrc/upscaninitiate/connector/s3/S3UploadFormGeneratorProvider.scala index 5a3c7d8..38370a6 100644 --- a/app/uk/gov/hmrc/upscaninitiate/connector/s3/S3UploadFormGeneratorProvider.scala +++ b/app/uk/gov/hmrc/upscaninitiate/connector/s3/S3UploadFormGeneratorProvider.scala @@ -21,17 +21,52 @@ import uk.gov.hmrc.upscaninitiate.connector.model.{AwsCredentials, UploadFormGen import java.time.{Clock, Instant} import javax.inject.{Inject, Provider, Singleton} +import play.api.libs.functional.syntax.toFunctionalBuilderOps +import play.api.libs.json.* +import software.amazon.awssdk.services.secretsmanager.SecretsManagerClient +import software.amazon.awssdk.services.secretsmanager.model.GetSecretValueRequest +import play.api.Logger @Singleton class S3UploadFormGeneratorProvider @Inject()( - configuration: ServiceConfiguration, - clock : Clock + configuration : ServiceConfiguration, + clock : Clock, + secretsManagerClient: SecretsManagerClient ) extends Provider[UploadFormGenerator]: import configuration._ + private val logger = Logger(getClass) + override def get() = + val credentials = sessionToken match + case Some(_) => + AwsCredentials(accessKeyId, secretAccessKey, sessionToken) + case None => + try + val request = GetSecretValueRequest.builder() + .secretId(secretArn) + .build() + + val secretResponse = secretsManagerClient.getSecretValue(request) + val creds = Json.parse(secretResponse.secretString()).as[RetrievedCredentials](RetrievedCredentials.reads) + + AwsCredentials(creds.accessKeyId, creds.secretAccessKey, sessionToken = None) + catch + case e: Exception => + logger.error(s"Failed to retrieve AWS credentials from Secrets Manager: ${e.getMessage}", e) + throw e + S3UploadFormGenerator( - AwsCredentials(accessKeyId, secretAccessKey, sessionToken), + credentials, regionName = region, currentTime = () => Instant.now(clock) ) + +final case class RetrievedCredentials(accessKeyId: String, secretAccessKey: String): + override def toString(): String = s"RetrievedCredentials($accessKeyId, REDACTED)" + +object RetrievedCredentials: + val reads: Reads[RetrievedCredentials] = + ( (__ \ "accessKeyId" ).read[String] + ~ (__ \ "secretAccessKey").read[String] + )(apply) diff --git a/conf/application.conf b/conf/application.conf index 8585c06..10d4063 100644 --- a/conf/application.conf +++ b/conf/application.conf @@ -44,8 +44,9 @@ aws { upload.link.validity.duration = "7 days" } + secretArn = "ENTER FULL ARN" # not used when running locally with aws-profile session accessKeyId = "ENTER YOUR KEY" - secretAccessKey = "ENYER YOUR SECRET" + secretAccessKey = "ENTER YOUR SECRET" accessKeyId = ${?AWS_ACCESS_KEY_ID} secretAccessKey = ${?AWS_SECRET_ACCESS_KEY} sessionToken = ${?AWS_SESSION_TOKEN} diff --git a/it/test/uk/gov/hmrc/upscaninitiate/controller/PrepareUploadControllerISpec.scala b/it/test/uk/gov/hmrc/upscaninitiate/controller/PrepareUploadControllerISpec.scala index 6d87fd5..e554087 100644 --- a/it/test/uk/gov/hmrc/upscaninitiate/controller/PrepareUploadControllerISpec.scala +++ b/it/test/uk/gov/hmrc/upscaninitiate/controller/PrepareUploadControllerISpec.scala @@ -22,10 +22,14 @@ import org.scalatest.wordspec.AnyWordSpec import org.scalatestplus.play.guice.GuiceOneAppPerSuite import play.api.Application import play.api.inject.guice.GuiceApplicationBuilder +import play.api.inject.bind import play.api.libs.json.{JsValue, Json} import play.api.test.Helpers._ import play.api.test.{FakeHeaders, FakeRequest} import uk.gov.hmrc.http.HeaderNames.xSessionId +import uk.gov.hmrc.upscaninitiate.connector.model.{AwsCredentials, UploadFormGenerator} +import uk.gov.hmrc.upscaninitiate.connector.s3.S3UploadFormGenerator +import java.time.{Clock, Instant} class PrepareUploadControllerISpec extends AnyWordSpec @@ -41,6 +45,17 @@ class PrepareUploadControllerISpec "uploadProxy.url" -> "https://upload-proxy.tax.service.gov.uk", "aws.s3.bucket.inbound" -> "inbound-bucket" ) + .overrides( + bind[UploadFormGenerator].toInstance(new S3UploadFormGenerator( + AwsCredentials( + accessKeyId = "test-key-id", + secretKey = "test-secret-key", + sessionToken = None + ), + regionName = "eu-west-2", + currentTime = () => Instant.now(Clock.systemUTC()) + )) + ) .build() "PrepareUploadController prepareUploadV1 with all request values" in: diff --git a/project/AppDependencies.scala b/project/AppDependencies.scala index 5eef449..47a747a 100644 --- a/project/AppDependencies.scala +++ b/project/AppDependencies.scala @@ -1,11 +1,12 @@ import sbt._ object AppDependencies { - private val bootstrapVersion = "9.10.0" + private val bootstrapVersion = "9.11.0" private val compile = Seq( "uk.gov.hmrc" %% "bootstrap-backend-play-30" % bootstrapVersion, - "software.amazon.awssdk" % "s3" % "2.30.30" + "software.amazon.awssdk" % "s3" % "2.30.30", + "software.amazon.awssdk" % "secretsmanager" % "2.30.30" ) private val test = Seq( diff --git a/project/build.properties b/project/build.properties index 04267b1..e97b272 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1 +1 @@ -sbt.version=1.9.9 +sbt.version=1.10.10 From c7bcfe3b0e9d0027519e1e9a3f25b517b4904819 Mon Sep 17 00:00:00 2001 From: macgrewal <7713555+macgrewal@users.noreply.github.com> Date: Wed, 7 May 2025 11:36:13 +0100 Subject: [PATCH 95/96] BDOG-3406: Updated README to reflect new default TTL on presigned download URLs --- README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index be00800..bb3f2c3 100644 --- a/README.md +++ b/README.md @@ -484,12 +484,12 @@ These commands will give you an access to SBT shell where you can run the servic ## Appendix ### Quick reference figures -| Metric | Value | Comments | -| ------------------------------------------------------- | ----------------|----------| -| Expiration of S3 upload pre-signed URL | 7 days | A relatively long period, since we can't control exactly when users will initiate the upload process | -| Expiration of S3 download pre-signed URL (scanned docs) | 1 day (default) | Configurable per-service up to 7 days. Upscan is not intended as a storage solution for services | -| Callback request retry time | 60 seconds | | -| Maximum callback notification retries | 30 | | +| Metric | Value | Comments | +| ------------------------------------------------------- | ------------------|----------| +| Expiration of S3 upload pre-signed URL | 7 days | A relatively long period, since we can't control exactly when users will initiate the upload process | +| Expiration of S3 download pre-signed URL (scanned docs) | 6 hours (default) | Configurable per-service up to 6 hourss. Upscan is not intended as a storage solution for services, for storage needs beyond 6 hours, integrate with [object-store](https://github.com/hmrc/object-store) | +| Callback request retry time | 60 seconds | | +| Maximum callback notification retries | 30 | | [[Back to the top]](#top) From b63018524d2ad33b080308ae93702254a1280f8b Mon Sep 17 00:00:00 2001 From: macgrewal <7713555+macgrewal@users.noreply.github.com> Date: Wed, 7 May 2025 13:14:13 +0100 Subject: [PATCH 96/96] Fixed typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index bb3f2c3..00c2860 100644 --- a/README.md +++ b/README.md @@ -487,7 +487,7 @@ These commands will give you an access to SBT shell where you can run the servic | Metric | Value | Comments | | ------------------------------------------------------- | ------------------|----------| | Expiration of S3 upload pre-signed URL | 7 days | A relatively long period, since we can't control exactly when users will initiate the upload process | -| Expiration of S3 download pre-signed URL (scanned docs) | 6 hours (default) | Configurable per-service up to 6 hourss. Upscan is not intended as a storage solution for services, for storage needs beyond 6 hours, integrate with [object-store](https://github.com/hmrc/object-store) | +| Expiration of S3 download pre-signed URL (scanned docs) | 6 hours (default) | Configurable per-service up to 6 hours. Upscan is not intended as a storage solution for services, for storage needs beyond 6 hours, integrate with [object-store](https://github.com/hmrc/object-store) | | Callback request retry time | 60 seconds | | | Maximum callback notification retries | 30 | |