diff --git a/docs/config-app.md b/docs/config-app.md index 569e4c09b15..7ee227f2631 100644 --- a/docs/config-app.md +++ b/docs/config-app.md @@ -27,6 +27,7 @@ options. ## Server - `server.max-headers-size` - set the maximum length of all headers. +- `server.max-body-size` - set the maximum length of body. - `server.ssl` - enable SSL/TLS support. - `server.jks-path` - path to the java keystore (if ssl is enabled). - `server.jks-password` - password for the keystore (if ssl is enabled). @@ -117,7 +118,6 @@ Removes and downloads file again if depending service cant process probably corr - `auction.biddertmax.max` - maximum operation timeout for OpenRTB Auction requests. - `auction.biddertmax.percent` - adjustment factor for `request.tmax` for bidders. - `auction.tmax-upstream-response-time` - the amount of time that PBS needs to respond to the original caller. -- `auction.max-request-size` - set the maximum size in bytes of OpenRTB Auction request. - `auction.stored-requests-timeout-ms` - timeout for stored requests fetching. - `auction.ad-server-currency` - default currency for auction, if its value was not specified in request. Important note: PBS uses ISO-4217 codes for the representation of currencies. diff --git a/src/main/java/org/prebid/server/auction/requestfactory/AuctionRequestFactory.java b/src/main/java/org/prebid/server/auction/requestfactory/AuctionRequestFactory.java index d873af3e696..e857262275b 100644 --- a/src/main/java/org/prebid/server/auction/requestfactory/AuctionRequestFactory.java +++ b/src/main/java/org/prebid/server/auction/requestfactory/AuctionRequestFactory.java @@ -38,7 +38,6 @@ */ public class AuctionRequestFactory { - private final long maxRequestSize; private final Ortb2RequestFactory ortb2RequestFactory; private final StoredRequestProcessor storedRequestProcessor; private final ProfilesProcessor profilesProcessor; @@ -57,8 +56,7 @@ public class AuctionRequestFactory { private static final String ENDPOINT = Endpoint.openrtb2_auction.value(); - public AuctionRequestFactory(long maxRequestSize, - Ortb2RequestFactory ortb2RequestFactory, + public AuctionRequestFactory(Ortb2RequestFactory ortb2RequestFactory, StoredRequestProcessor storedRequestProcessor, ProfilesProcessor profilesProcessor, BidRequestOrtbVersionConversionManager ortbVersionConversionManager, @@ -74,7 +72,6 @@ public AuctionRequestFactory(long maxRequestSize, GeoLocationServiceWrapper geoLocationServiceWrapper, BidAdjustmentsEnricher bidAdjustmentsEnricher) { - this.maxRequestSize = maxRequestSize; this.ortb2RequestFactory = Objects.requireNonNull(ortb2RequestFactory); this.storedRequestProcessor = Objects.requireNonNull(storedRequestProcessor); this.profilesProcessor = Objects.requireNonNull(profilesProcessor); @@ -166,10 +163,6 @@ private String extractAndValidateBody(RoutingContext routingContext) { throw new InvalidRequestException("Incoming request has no body"); } - if (body.length() > maxRequestSize) { - throw new InvalidRequestException("Request size exceeded max size of %d bytes.".formatted(maxRequestSize)); - } - return body; } diff --git a/src/main/java/org/prebid/server/auction/requestfactory/VideoRequestFactory.java b/src/main/java/org/prebid/server/auction/requestfactory/VideoRequestFactory.java index 811db804fb9..6b20a444ebd 100644 --- a/src/main/java/org/prebid/server/auction/requestfactory/VideoRequestFactory.java +++ b/src/main/java/org/prebid/server/auction/requestfactory/VideoRequestFactory.java @@ -50,7 +50,6 @@ public class VideoRequestFactory { private static final int DEFAULT_CACHE_LOG_TTL = 3600; private static final String ENDPOINT = Endpoint.openrtb2_video.value(); - private final int maxRequestSize; private final boolean enforceStoredRequest; private final Pattern escapeLogCacheRegexPattern; @@ -63,8 +62,7 @@ public class VideoRequestFactory { private final JacksonMapper mapper; private final GeoLocationServiceWrapper geoLocationServiceWrapper; - public VideoRequestFactory(int maxRequestSize, - boolean enforceStoredRequest, + public VideoRequestFactory(boolean enforceStoredRequest, String escapeLogCacheRegex, Ortb2RequestFactory ortb2RequestFactory, VideoStoredRequestProcessor storedRequestProcessor, @@ -76,7 +74,6 @@ public VideoRequestFactory(int maxRequestSize, GeoLocationServiceWrapper geoLocationServiceWrapper) { this.enforceStoredRequest = enforceStoredRequest; - this.maxRequestSize = maxRequestSize; this.ortb2RequestFactory = Objects.requireNonNull(ortb2RequestFactory); this.storedRequestProcessor = Objects.requireNonNull(storedRequestProcessor); this.ortbVersionConversionManager = Objects.requireNonNull(ortbVersionConversionManager); @@ -120,9 +117,9 @@ public Future> fromRequest(RoutingContext routingC .map(auctionContext -> auctionContext.with(debugResolver.debugContextFrom(auctionContext))) .compose(auctionContext -> ortb2RequestFactory.limitImpressions( - auctionContext.getAccount(), - auctionContext.getBidRequest(), - auctionContext.getDebugWarnings()) + auctionContext.getAccount(), + auctionContext.getBidRequest(), + auctionContext.getDebugWarnings()) .map(auctionContext::with)) .compose(auctionContext -> ortb2RequestFactory.validateRequest( @@ -177,10 +174,6 @@ private String extractAndValidateBody(RoutingContext routingContext) { throw new InvalidRequestException("Incoming request has no body"); } - if (body.length() > maxRequestSize) { - throw new InvalidRequestException("Request size exceeded max size of %d bytes.".formatted(maxRequestSize)); - } - return body; } diff --git a/src/main/java/org/prebid/server/spring/config/ServiceConfiguration.java b/src/main/java/org/prebid/server/spring/config/ServiceConfiguration.java index 597feb1f1ac..a3e14218c5e 100644 --- a/src/main/java/org/prebid/server/spring/config/ServiceConfiguration.java +++ b/src/main/java/org/prebid/server/spring/config/ServiceConfiguration.java @@ -139,7 +139,6 @@ import org.springframework.context.annotation.Scope; import org.springframework.context.annotation.ScopedProxyMode; -import jakarta.validation.constraints.Min; import java.io.IOException; import java.time.Clock; import java.util.ArrayList; @@ -470,7 +469,6 @@ Ortb2RequestFactory openRtb2RequestFactory( @Bean AuctionRequestFactory auctionRequestFactory( - @Value("${auction.max-request-size}") @Min(0) int maxRequestSize, Ortb2RequestFactory ortb2RequestFactory, StoredRequestProcessor storedRequestProcessor, ProfilesProcessor profilesProcessor, @@ -487,7 +485,6 @@ AuctionRequestFactory auctionRequestFactory( BidAdjustmentsEnricher bidAdjustmentsEnricher) { return new AuctionRequestFactory( - maxRequestSize, ortb2RequestFactory, storedRequestProcessor, profilesProcessor, @@ -555,7 +552,6 @@ AmpRequestFactory ampRequestFactory(Ortb2RequestFactory ortb2RequestFactory, @Bean VideoRequestFactory videoRequestFactory( - @Value("${auction.max-request-size}") int maxRequestSize, @Value("${video.stored-request-required}") boolean enforceStoredRequest, @Value("${auction.video.escape-log-cache-regex:#{null}}") String escapeLogCacheRegex, Ortb2RequestFactory ortb2RequestFactory, @@ -568,7 +564,6 @@ VideoRequestFactory videoRequestFactory( GeoLocationServiceWrapper geoLocationServiceWrapper) { return new VideoRequestFactory( - maxRequestSize, enforceStoredRequest, escapeLogCacheRegex, ortb2RequestFactory, diff --git a/src/main/java/org/prebid/server/spring/config/VertxConfiguration.java b/src/main/java/org/prebid/server/spring/config/VertxConfiguration.java index 3ac62c8fd59..844a4f930dc 100644 --- a/src/main/java/org/prebid/server/spring/config/VertxConfiguration.java +++ b/src/main/java/org/prebid/server/spring/config/VertxConfiguration.java @@ -12,10 +12,13 @@ import org.prebid.server.log.LoggerFactory; import org.prebid.server.spring.config.metrics.MetricsConfiguration; import org.prebid.server.vertx.ContextRunner; +import org.prebid.server.vertx.http.ParametrizedDecompressionHandler; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import jakarta.validation.constraints.Min; + @Configuration public class VertxConfiguration { @@ -55,8 +58,18 @@ FileSystem fileSystem(Vertx vertx) { } @Bean - BodyHandler bodyHandler(@Value("${vertx.uploads-dir}") String uploadsDir) { - return BodyHandler.create(uploadsDir); + BodyHandler bodyHandler(@Value("${vertx.uploads-dir}") String uploadsDir, + @Value("${server.max-body-size}") @Min(0) long maxBodySize) { + + return BodyHandler.create(uploadsDir) + .setBodyLimit(maxBodySize); + } + + @Bean + ParametrizedDecompressionHandler gzipParamDecompressionHandler( + @Value("${server.max-body-size}") @Min(0) int maxBodySize) { + + return new ParametrizedDecompressionHandler(maxBodySize); } @Bean diff --git a/src/main/java/org/prebid/server/spring/config/server/admin/AdminServerConfiguration.java b/src/main/java/org/prebid/server/spring/config/server/admin/AdminServerConfiguration.java index e88e04be03d..e64ea153aaa 100644 --- a/src/main/java/org/prebid/server/spring/config/server/admin/AdminServerConfiguration.java +++ b/src/main/java/org/prebid/server/spring/config/server/admin/AdminServerConfiguration.java @@ -4,6 +4,7 @@ import io.vertx.core.net.SocketAddress; import io.vertx.ext.web.Router; import io.vertx.ext.web.handler.BodyHandler; +import org.prebid.server.vertx.http.ParametrizedDecompressionHandler; import org.prebid.server.vertx.verticles.VerticleDefinition; import org.prebid.server.vertx.verticles.server.ServerVerticle; import org.springframework.beans.factory.annotation.Value; @@ -18,10 +19,12 @@ public class AdminServerConfiguration { @Bean Router adminPortAdminServerRouter(Vertx vertx, AdminResourcesBinder adminPortAdminResourcesBinder, - BodyHandler bodyHandler) { + BodyHandler bodyHandler, + ParametrizedDecompressionHandler parametrizedDecompressionHandler) { final Router router = Router.router(vertx); router.route().handler(bodyHandler); + router.route().handler(parametrizedDecompressionHandler); adminPortAdminResourcesBinder.bind(router); return router; diff --git a/src/main/java/org/prebid/server/spring/config/server/application/ApplicationServerConfiguration.java b/src/main/java/org/prebid/server/spring/config/server/application/ApplicationServerConfiguration.java index a6b56622c42..629cc42be6c 100644 --- a/src/main/java/org/prebid/server/spring/config/server/application/ApplicationServerConfiguration.java +++ b/src/main/java/org/prebid/server/spring/config/server/application/ApplicationServerConfiguration.java @@ -63,6 +63,7 @@ import org.prebid.server.util.HttpUtil; import org.prebid.server.validation.BidderParamValidator; import org.prebid.server.version.PrebidVersionProvider; +import org.prebid.server.vertx.http.ParametrizedDecompressionHandler; import org.prebid.server.vertx.verticles.VerticleDefinition; import org.prebid.server.vertx.verticles.server.ServerVerticle; import org.prebid.server.vertx.verticles.server.application.ApplicationResource; @@ -166,6 +167,7 @@ ExceptionHandler exceptionHandler(Metrics metrics) { @Bean Router applicationServerRouter(Vertx vertx, BodyHandler bodyHandler, + ParametrizedDecompressionHandler parametrizedDecompressionHandler, NoCacheHandler noCacheHandler, CorsHandler corsHandler, List resources, @@ -174,6 +176,7 @@ Router applicationServerRouter(Vertx vertx, final Router router = Router.router(vertx); router.route().handler(bodyHandler); + router.route().handler(parametrizedDecompressionHandler); router.route().handler(noCacheHandler); router.route().handler(corsHandler); diff --git a/src/main/java/org/prebid/server/vertx/http/ParametrizedDecompressionHandler.java b/src/main/java/org/prebid/server/vertx/http/ParametrizedDecompressionHandler.java new file mode 100644 index 00000000000..20af702492f --- /dev/null +++ b/src/main/java/org/prebid/server/vertx/http/ParametrizedDecompressionHandler.java @@ -0,0 +1,81 @@ +package org.prebid.server.vertx.http; + +import io.netty.handler.codec.http.HttpResponseStatus; +import io.vertx.core.Handler; +import io.vertx.core.buffer.Buffer; +import io.vertx.ext.web.RoutingContext; +import io.vertx.ext.web.impl.RoutingContextInternal; +import org.apache.commons.lang3.StringUtils; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.zip.GZIPInputStream; + +public class ParametrizedDecompressionHandler implements Handler { + + private final ThreadLocal intermediateBuffer; + private final ThreadLocal inputBuffer; + private final ThreadLocal outputBuffer; + + public ParametrizedDecompressionHandler(int maxBodySize) { + intermediateBuffer = ThreadLocal.withInitial(() -> new byte[16384]); + inputBuffer = ThreadLocal.withInitial(() -> new byte[maxBodySize]); + outputBuffer = ThreadLocal.withInitial(() -> new byte[2 * maxBodySize]); + } + + @Override + public void handle(RoutingContext routingContext) { + if (!StringUtils.equalsAny(routingContext.request().getParam("gzip"), "1", "true")) { + routingContext.next(); + return; + } + + try { + final Buffer decompressed = decompressGzip(routingContext.body().buffer()); + ((RoutingContextInternal) routingContext).setBody(decompressed); + routingContext.next(); + } catch (IOException e) { + respondWithBadRequest(routingContext, "Invalid body: " + e.getMessage()); + } + } + + private static void respondWithBadRequest(RoutingContext routingContext, String message) { + routingContext.response() + .setStatusCode(HttpResponseStatus.BAD_REQUEST.code()) + .end(message); + } + + private Buffer decompressGzip(Buffer compressed) throws IOException { + final byte[] decompressionBuffer = intermediateBuffer.get(); + final byte[] compressedBuffer = inputBuffer.get(); + final byte[] decompressedBuffer = outputBuffer.get(); + + compressed.getBytes(compressedBuffer); + try (ByteArrayInputStream input = new ByteArrayInputStream(compressedBuffer); + GZIPInputStream gzip = new GZIPInputStream(input); + FastByteArrayOutputStream baos = new FastByteArrayOutputStream(decompressedBuffer)) { + + int totalLen = 0; + int len; + while ((len = gzip.read(decompressionBuffer)) > 0) { + baos.write(decompressionBuffer, 0, len); + totalLen += len; + } + + compressed.setBytes(0, baos.getBuffer(), 0, totalLen); + return compressed.slice(0, totalLen); + } + } + + private static class FastByteArrayOutputStream extends ByteArrayOutputStream { + + FastByteArrayOutputStream(byte[] buf) { + this.buf = buf; + } + + public byte[] getBuffer() { + return buf; + } + } +} diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index 55822047954..fb84010f765 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -10,6 +10,7 @@ vertx: server: max-initial-line-length: 8092 max-headers-size: 16384 + max-body-size: 262144 ssl: false jks-path: jks-password: @@ -119,7 +120,6 @@ auction: log-result: false log-failure-only: false log-sampling-rate: 0.0 - max-request-size: 262144 generate-bid-id: false cache: expected-request-time-ms: 10 diff --git a/src/test/java/org/prebid/server/auction/requestfactory/AuctionRequestFactoryTest.java b/src/test/java/org/prebid/server/auction/requestfactory/AuctionRequestFactoryTest.java index 59615bdbaab..ea694557289 100644 --- a/src/test/java/org/prebid/server/auction/requestfactory/AuctionRequestFactoryTest.java +++ b/src/test/java/org/prebid/server/auction/requestfactory/AuctionRequestFactoryTest.java @@ -210,7 +210,6 @@ public void setUp() { given(bidAdjustmentsEnricher.enrichBidRequest(any())).willReturn(defaultBidRequest); target = new AuctionRequestFactory( - Integer.MAX_VALUE, ortb2RequestFactory, storedRequestProcessor, profilesProcessor, @@ -243,39 +242,6 @@ public void shouldReturnFailedFutureIfRequestBodyIsMissing() { .hasMessage("Incoming request has no body"); } - @Test - public void shouldReturnFailedFutureIfRequestBodyExceedsMaxRequestSize() { - // given - target = new AuctionRequestFactory( - 1, - ortb2RequestFactory, - storedRequestProcessor, - profilesProcessor, - ortbVersionConversionManager, - auctionGppService, - cookieDeprecationService, - paramsExtractor, - paramsResolver, - interstitialProcessor, - ortbTypesResolver, - auctionPrivacyContextFactory, - debugResolver, - jacksonMapper, - geoLocationServiceWrapper, - bidAdjustmentsEnricher); - - given(requestBody.asString()).willReturn("body"); - - // when - final Future future = target.parseRequest(routingContext, 0L); - - // then - assertThat(future.failed()).isTrue(); - assertThat(future.cause()) - .isInstanceOf(InvalidRequestException.class) - .hasMessage("Request size exceeded max size of 1 bytes."); - } - @Test public void shouldReturnFailedFutureIfRequestBodyCouldNotBeParsed() { // given diff --git a/src/test/java/org/prebid/server/auction/requestfactory/VideoRequestFactoryTest.java b/src/test/java/org/prebid/server/auction/requestfactory/VideoRequestFactoryTest.java index ad154e06b49..ec2c147a281 100644 --- a/src/test/java/org/prebid/server/auction/requestfactory/VideoRequestFactoryTest.java +++ b/src/test/java/org/prebid/server/auction/requestfactory/VideoRequestFactoryTest.java @@ -137,7 +137,6 @@ public void setUp() { .willReturn(Future.succeededFuture(defaultPrivacyContext)); target = new VideoRequestFactory( - Integer.MAX_VALUE, false, null, ortb2RequestFactory, @@ -173,7 +172,6 @@ public void shouldReturnFailedFutureIfStoredRequestIsEnforcedAndIdIsNotProvided( given(routingContext.request().headers()).willReturn(MultiMap.caseInsensitiveMultiMap() .add(HttpUtil.USER_AGENT_HEADER, "123")); target = new VideoRequestFactory( - Integer.MAX_VALUE, true, null, ortb2RequestFactory, @@ -195,34 +193,6 @@ public void shouldReturnFailedFutureIfStoredRequestIsEnforcedAndIdIsNotProvided( .hasMessage("Unable to find required stored request id"); } - @Test - public void shouldReturnFailedFutureIfRequestBodyExceedsMaxRequestSize() { - // given - target = new VideoRequestFactory( - 2, - true, - null, - ortb2RequestFactory, - videoStoredRequestProcessor, - ortbVersionConversionManager, - paramsResolver, - auctionPrivacyContextFactory, - debugResolver, - jacksonMapper, - geoLocationServiceWrapper); - - given(requestBody.asString()).willReturn("body"); - - // when - final Future future = target.fromRequest(routingContext, 0L); - - // then - assertThat(future.failed()).isTrue(); - assertThat(future.cause()) - .isInstanceOf(InvalidRequestException.class) - .hasMessage("Request size exceeded max size of 2 bytes."); - } - @Test public void shouldReturnFailedFutureIfRequestBodyCouldNotBeParsed() { // given diff --git a/src/test/java/org/prebid/server/vertx/http/ParametrizedDecompressionHandlerTest.java b/src/test/java/org/prebid/server/vertx/http/ParametrizedDecompressionHandlerTest.java new file mode 100644 index 00000000000..198d838dc73 --- /dev/null +++ b/src/test/java/org/prebid/server/vertx/http/ParametrizedDecompressionHandlerTest.java @@ -0,0 +1,130 @@ +package org.prebid.server.vertx.http; + +import io.netty.handler.codec.http.HttpResponseStatus; +import io.vertx.core.buffer.Buffer; +import io.vertx.core.http.HttpServerRequest; +import io.vertx.core.http.HttpServerResponse; +import io.vertx.ext.web.RequestBody; +import io.vertx.ext.web.impl.RoutingContextInternal; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.zip.GZIPOutputStream; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.BDDMockito.given; +import static org.mockito.Mock.Strictness.LENIENT; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +@ExtendWith(MockitoExtension.class) +public class ParametrizedDecompressionHandlerTest { + + @Mock(strictness = LENIENT) + private RoutingContextInternal routingContext; + @Mock(strictness = LENIENT) + private HttpServerRequest request; + @Mock(strictness = LENIENT) + private HttpServerResponse response; + @Mock(strictness = LENIENT) + private RequestBody requestBody; + + private final ParametrizedDecompressionHandler target = new ParametrizedDecompressionHandler(1024); + + @BeforeEach + public void setUp() { + given(routingContext.request()).willReturn(request); + given(routingContext.response()).willReturn(response); + given(routingContext.body()).willReturn(requestBody); + + given(response.setStatusCode(anyInt())).willReturn(response); + } + + @Test + public void handleShouldPassRequestToNextHandlerWhenGzipNotSet() { + // given and when + target.handle(routingContext); + + // then + verify(routingContext).next(); + verify(routingContext, times(0)).setBody(any()); + } + + @Test + public void handleShouldDecompressRequestAndSetBodyWhenGzipParamIsSetToOne() { + // given + final byte[] body = "decompressed body".getBytes(StandardCharsets.UTF_8); + + given(request.getParam(eq("gzip"))).willReturn("1"); + given(requestBody.buffer()).willReturn(Buffer.buffer(gzip(body))); + + // when + target.handle(routingContext); + + // then + verify(routingContext, times(1)).setBody(eq(Buffer.buffer(body))); + verify(routingContext).next(); + } + + @Test + public void handleShouldDecompressRequestAndSetBodyWhenGzipParamIsSetToTrue() { + // given + final byte[] body = "decompressed body".getBytes(StandardCharsets.UTF_8); + + given(request.getParam(eq("gzip"))).willReturn("true"); + given(requestBody.buffer()).willReturn(Buffer.buffer(gzip(body))); + + // when + target.handle(routingContext); + + // then + verify(routingContext, times(1)).setBody(eq(Buffer.buffer(body))); + verify(routingContext).next(); + } + + @Test + public void handleShouldRespondWithBadRequestWhenCompressedBodyIsCorrupted() { + // given + final byte[] body = new byte[]{1, 2, 3, 4}; + + given(request.getParam(eq("gzip"))).willReturn("true"); + given(requestBody.buffer()).willReturn(Buffer.buffer(body)); + + // when + target.handle(routingContext); + + // then + verify(response).setStatusCode(HttpResponseStatus.BAD_REQUEST.code()); + + final ArgumentCaptor responseBodyCaptor = ArgumentCaptor.forClass(String.class); + verify(response).end(responseBodyCaptor.capture()); + assertThat(responseBodyCaptor.getValue()).asString().startsWith("Invalid body: "); + + verify(routingContext, times(0)).setBody(any()); + verify(routingContext, times(0)).next(); + } + + private static byte[] gzip(byte[] input) { + try ( + ByteArrayOutputStream obj = new ByteArrayOutputStream(); + GZIPOutputStream gzip = new GZIPOutputStream(obj)) { + + gzip.write(input); + gzip.finish(); + + return obj.toByteArray(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } +}