Gzip: Add decompression handler

docs/config-app.md

@@ -27,6 +27,7 @@ options.
 ## Server
 
 - `server.max-headers-size` - set the maximum length of all headers.
+- `server.max-body-size` - set the maximum length of body.
 - `server.ssl` - enable SSL/TLS support.
 - `server.jks-path` - path to the java keystore (if ssl is enabled).
 - `server.jks-password` - password for the keystore (if ssl is enabled).
@@ -117,7 +118,6 @@ Removes and downloads file again if depending service cant process probably corr
 - `auction.biddertmax.max` - maximum operation timeout for OpenRTB Auction requests.
 - `auction.biddertmax.percent` - adjustment factor for `request.tmax` for bidders.
 - `auction.tmax-upstream-response-time` - the amount of time that PBS needs to respond to the original caller.
-- `auction.max-request-size` - set the maximum size in bytes of OpenRTB Auction request.
 - `auction.stored-requests-timeout-ms` - timeout for stored requests fetching.
 - `auction.ad-server-currency` - default currency for auction, if its value was not specified in request. Important
   note: PBS uses ISO-4217 codes for the representation of currencies.

src/main/java/org/prebid/server/auction/requestfactory/AuctionRequestFactory.java

@@ -38,7 +38,6 @@
  */
 public class AuctionRequestFactory {
 
-    private final long maxRequestSize;
     private final Ortb2RequestFactory ortb2RequestFactory;
     private final StoredRequestProcessor storedRequestProcessor;
     private final ProfilesProcessor profilesProcessor;
@@ -57,8 +56,7 @@ public class AuctionRequestFactory {
 
     private static final String ENDPOINT = Endpoint.openrtb2_auction.value();
 
-    public AuctionRequestFactory(long maxRequestSize,
-                                 Ortb2RequestFactory ortb2RequestFactory,
+    public AuctionRequestFactory(Ortb2RequestFactory ortb2RequestFactory,
                                  StoredRequestProcessor storedRequestProcessor,
                                  ProfilesProcessor profilesProcessor,
                                  BidRequestOrtbVersionConversionManager ortbVersionConversionManager,
@@ -74,7 +72,6 @@ public AuctionRequestFactory(long maxRequestSize,
                                  GeoLocationServiceWrapper geoLocationServiceWrapper,
                                  BidAdjustmentsEnricher bidAdjustmentsEnricher) {
 
-        this.maxRequestSize = maxRequestSize;
         this.ortb2RequestFactory = Objects.requireNonNull(ortb2RequestFactory);
         this.storedRequestProcessor = Objects.requireNonNull(storedRequestProcessor);
         this.profilesProcessor = Objects.requireNonNull(profilesProcessor);
@@ -166,10 +163,6 @@ private String extractAndValidateBody(RoutingContext routingContext) {
             throw new InvalidRequestException("Incoming request has no body");
         }
 
-        if (body.length() > maxRequestSize) {
-            throw new InvalidRequestException("Request size exceeded max size of %d bytes.".formatted(maxRequestSize));
-        }
-
         return body;
     }
 

src/main/java/org/prebid/server/auction/requestfactory/VideoRequestFactory.java

@@ -50,7 +50,6 @@ public class VideoRequestFactory {
     private static final int DEFAULT_CACHE_LOG_TTL = 3600;
     private static final String ENDPOINT = Endpoint.openrtb2_video.value();
 
-    private final int maxRequestSize;
     private final boolean enforceStoredRequest;
     private final Pattern escapeLogCacheRegexPattern;
 
@@ -63,8 +62,7 @@ public class VideoRequestFactory {
     private final JacksonMapper mapper;
     private final GeoLocationServiceWrapper geoLocationServiceWrapper;
 
-    public VideoRequestFactory(int maxRequestSize,
-                               boolean enforceStoredRequest,
+    public VideoRequestFactory(boolean enforceStoredRequest,
                                String escapeLogCacheRegex,
                                Ortb2RequestFactory ortb2RequestFactory,
                                VideoStoredRequestProcessor storedRequestProcessor,
@@ -76,7 +74,6 @@ public VideoRequestFactory(int maxRequestSize,
                                GeoLocationServiceWrapper geoLocationServiceWrapper) {
 
         this.enforceStoredRequest = enforceStoredRequest;
-        this.maxRequestSize = maxRequestSize;
         this.ortb2RequestFactory = Objects.requireNonNull(ortb2RequestFactory);
         this.storedRequestProcessor = Objects.requireNonNull(storedRequestProcessor);
         this.ortbVersionConversionManager = Objects.requireNonNull(ortbVersionConversionManager);
@@ -120,9 +117,9 @@ public Future<WithPodErrors<AuctionContext>> fromRequest(RoutingContext routingC
                 .map(auctionContext -> auctionContext.with(debugResolver.debugContextFrom(auctionContext)))
 
                 .compose(auctionContext -> ortb2RequestFactory.limitImpressions(
-                        auctionContext.getAccount(),
-                        auctionContext.getBidRequest(),
-                        auctionContext.getDebugWarnings())
+                                auctionContext.getAccount(),
+                                auctionContext.getBidRequest(),
+                                auctionContext.getDebugWarnings())
                         .map(auctionContext::with))
 
                 .compose(auctionContext -> ortb2RequestFactory.validateRequest(
@@ -177,10 +174,6 @@ private String extractAndValidateBody(RoutingContext routingContext) {
             throw new InvalidRequestException("Incoming request has no body");
         }
 
-        if (body.length() > maxRequestSize) {
-            throw new InvalidRequestException("Request size exceeded max size of %d bytes.".formatted(maxRequestSize));
-        }
-
         return body;
     }
 

src/main/java/org/prebid/server/spring/config/ServiceConfiguration.java

@@ -139,7 +139,6 @@
 import org.springframework.context.annotation.Scope;
 import org.springframework.context.annotation.ScopedProxyMode;
 
-import jakarta.validation.constraints.Min;
 import java.io.IOException;
 import java.time.Clock;
 import java.util.ArrayList;
@@ -470,7 +469,6 @@ Ortb2RequestFactory openRtb2RequestFactory(
 
     @Bean
     AuctionRequestFactory auctionRequestFactory(
-            @Value("${auction.max-request-size}") @Min(0) int maxRequestSize,
             Ortb2RequestFactory ortb2RequestFactory,
             StoredRequestProcessor storedRequestProcessor,
             ProfilesProcessor profilesProcessor,
@@ -487,7 +485,6 @@ AuctionRequestFactory auctionRequestFactory(
             BidAdjustmentsEnricher bidAdjustmentsEnricher) {
 
         return new AuctionRequestFactory(
-                maxRequestSize,
                 ortb2RequestFactory,
                 storedRequestProcessor,
                 profilesProcessor,
@@ -555,7 +552,6 @@ AmpRequestFactory ampRequestFactory(Ortb2RequestFactory ortb2RequestFactory,
 
     @Bean
     VideoRequestFactory videoRequestFactory(
-            @Value("${auction.max-request-size}") int maxRequestSize,
             @Value("${video.stored-request-required}") boolean enforceStoredRequest,
             @Value("${auction.video.escape-log-cache-regex:#{null}}") String escapeLogCacheRegex,
             Ortb2RequestFactory ortb2RequestFactory,
@@ -568,7 +564,6 @@ VideoRequestFactory videoRequestFactory(
             GeoLocationServiceWrapper geoLocationServiceWrapper) {
 
         return new VideoRequestFactory(
-                maxRequestSize,
                 enforceStoredRequest,
                 escapeLogCacheRegex,
                 ortb2RequestFactory,

src/main/java/org/prebid/server/spring/config/VertxConfiguration.java

@@ -12,10 +12,13 @@
 import org.prebid.server.log.LoggerFactory;
 import org.prebid.server.spring.config.metrics.MetricsConfiguration;
 import org.prebid.server.vertx.ContextRunner;
+import org.prebid.server.vertx.http.ParametrizedDecompressionHandler;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 
+import jakarta.validation.constraints.Min;
+
 @Configuration
 public class VertxConfiguration {
 
@@ -55,8 +58,18 @@ FileSystem fileSystem(Vertx vertx) {
     }
 
     @Bean
-    BodyHandler bodyHandler(@Value("${vertx.uploads-dir}") String uploadsDir) {
-        return BodyHandler.create(uploadsDir);
+    BodyHandler bodyHandler(@Value("${vertx.uploads-dir}") String uploadsDir,
+                            @Value("${server.max-body-size}") @Min(0) long maxBodySize) {
+
+        return BodyHandler.create(uploadsDir)
+                .setBodyLimit(maxBodySize);
+    }
+
+    @Bean
+    ParametrizedDecompressionHandler gzipParamDecompressionHandler(
+            @Value("${server.max-body-size}") @Min(0) int maxBodySize) {
+
+        return new ParametrizedDecompressionHandler(maxBodySize);
     }
 
     @Bean

src/main/java/org/prebid/server/spring/config/server/admin/AdminServerConfiguration.java

@@ -4,6 +4,7 @@
 import io.vertx.core.net.SocketAddress;
 import io.vertx.ext.web.Router;
 import io.vertx.ext.web.handler.BodyHandler;
+import org.prebid.server.vertx.http.ParametrizedDecompressionHandler;
 import org.prebid.server.vertx.verticles.VerticleDefinition;
 import org.prebid.server.vertx.verticles.server.ServerVerticle;
 import org.springframework.beans.factory.annotation.Value;
@@ -18,10 +19,12 @@ public class AdminServerConfiguration {
     @Bean
     Router adminPortAdminServerRouter(Vertx vertx,
                                       AdminResourcesBinder adminPortAdminResourcesBinder,
-                                      BodyHandler bodyHandler) {
+                                      BodyHandler bodyHandler,
+                                      ParametrizedDecompressionHandler parametrizedDecompressionHandler) {
 
         final Router router = Router.router(vertx);
         router.route().handler(bodyHandler);
+        router.route().handler(parametrizedDecompressionHandler);
 
         adminPortAdminResourcesBinder.bind(router);
         return router;

src/main/java/org/prebid/server/spring/config/server/application/ApplicationServerConfiguration.java

@@ -63,6 +63,7 @@
 import org.prebid.server.util.HttpUtil;
 import org.prebid.server.validation.BidderParamValidator;
 import org.prebid.server.version.PrebidVersionProvider;
+import org.prebid.server.vertx.http.ParametrizedDecompressionHandler;
 import org.prebid.server.vertx.verticles.VerticleDefinition;
 import org.prebid.server.vertx.verticles.server.ServerVerticle;
 import org.prebid.server.vertx.verticles.server.application.ApplicationResource;
@@ -166,6 +167,7 @@ ExceptionHandler exceptionHandler(Metrics metrics) {
     @Bean
     Router applicationServerRouter(Vertx vertx,
                                    BodyHandler bodyHandler,
+                                   ParametrizedDecompressionHandler parametrizedDecompressionHandler,
                                    NoCacheHandler noCacheHandler,
                                    CorsHandler corsHandler,
                                    List<ApplicationResource> resources,
@@ -174,6 +176,7 @@ Router applicationServerRouter(Vertx vertx,
 
         final Router router = Router.router(vertx);
         router.route().handler(bodyHandler);
+        router.route().handler(parametrizedDecompressionHandler);
         router.route().handler(noCacheHandler);
         router.route().handler(corsHandler);
 

src/main/java/org/prebid/server/vertx/http/ParametrizedDecompressionHandler.java

@@ -0,0 +1,81 @@
+package org.prebid.server.vertx.http;
+
+import io.netty.handler.codec.http.HttpResponseStatus;
+import io.vertx.core.Handler;
+import io.vertx.core.buffer.Buffer;
+import io.vertx.ext.web.RoutingContext;
+import io.vertx.ext.web.impl.RoutingContextInternal;
+import org.apache.commons.lang3.StringUtils;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.util.zip.GZIPInputStream;
+
+public class ParametrizedDecompressionHandler implements Handler<RoutingContext> {
+
+    private final ThreadLocal<byte[]> intermediateBuffer;
+    private final ThreadLocal<byte[]> inputBuffer;
+    private final ThreadLocal<byte[]> outputBuffer;
+
+    public ParametrizedDecompressionHandler(int maxBodySize) {
+        intermediateBuffer = ThreadLocal.withInitial(() -> new byte[16384]);
+        inputBuffer = ThreadLocal.withInitial(() -> new byte[maxBodySize]);
+        outputBuffer = ThreadLocal.withInitial(() -> new byte[2 * maxBodySize]);
+    }
+
+    @Override
+    public void handle(RoutingContext routingContext) {
+        if (!StringUtils.equalsAny(routingContext.request().getParam("gzip"), "1", "true")) {
+            routingContext.next();
+            return;
+        }
+
+        try {
+            final Buffer decompressed = decompressGzip(routingContext.body().buffer());
+            ((RoutingContextInternal) routingContext).setBody(decompressed);
+            routingContext.next();
+        } catch (IOException e) {
+            respondWithBadRequest(routingContext, "Invalid body: " + e.getMessage());
+        }
+    }
+
+    private static void respondWithBadRequest(RoutingContext routingContext, String message) {
+        routingContext.response()
+                .setStatusCode(HttpResponseStatus.BAD_REQUEST.code())
+                .end(message);
+    }
+
+    private Buffer decompressGzip(Buffer compressed) throws IOException {
+        final byte[] decompressionBuffer = intermediateBuffer.get();
+        final byte[] compressedBuffer = inputBuffer.get();
+        final byte[] decompressedBuffer = outputBuffer.get();
+
+        compressed.getBytes(compressedBuffer);
+        try (ByteArrayInputStream input = new ByteArrayInputStream(compressedBuffer);
+                GZIPInputStream gzip = new GZIPInputStream(input);
+                FastByteArrayOutputStream baos = new FastByteArrayOutputStream(decompressedBuffer)) {
+
+            int totalLen = 0;
+            int len;
+            while ((len = gzip.read(decompressionBuffer)) > 0) {
+                baos.write(decompressionBuffer, 0, len);
+                totalLen += len;
+            }
+
+            compressed.setBytes(0, baos.getBuffer(), 0, totalLen);
+            return compressed.slice(0, totalLen);
+        }
+    }
+
+    private static class FastByteArrayOutputStream extends ByteArrayOutputStream {
+
+        FastByteArrayOutputStream(byte[] buf) {
+            this.buf = buf;
+        }
+
+        public byte[] getBuffer() {
+            return buf;
+        }
+    }
+}

src/main/resources/application.yaml

@@ -10,6 +10,7 @@ vertx:
 server:
   max-initial-line-length: 8092
   max-headers-size: 16384
+  max-body-size: 262144
   ssl: false
   jks-path:
   jks-password:
@@ -119,7 +120,6 @@ auction:
     log-result: false
     log-failure-only: false
     log-sampling-rate: 0.0
-  max-request-size: 262144
   generate-bid-id: false
   cache:
     expected-request-time-ms: 10

src/test/java/org/prebid/server/auction/requestfactory/AuctionRequestFactoryTest.java

@@ -210,7 +210,6 @@ public void setUp() {
         given(bidAdjustmentsEnricher.enrichBidRequest(any())).willReturn(defaultBidRequest);
 
         target = new AuctionRequestFactory(
-                Integer.MAX_VALUE,
                 ortb2RequestFactory,
                 storedRequestProcessor,
                 profilesProcessor,
@@ -243,39 +242,6 @@ public void shouldReturnFailedFutureIfRequestBodyIsMissing() {
                 .hasMessage("Incoming request has no body");
     }
 
-    @Test
-    public void shouldReturnFailedFutureIfRequestBodyExceedsMaxRequestSize() {
-        // given
-        target = new AuctionRequestFactory(
-                1,
-                ortb2RequestFactory,
-                storedRequestProcessor,
-                profilesProcessor,
-                ortbVersionConversionManager,
-                auctionGppService,
-                cookieDeprecationService,
-                paramsExtractor,
-                paramsResolver,
-                interstitialProcessor,
-                ortbTypesResolver,
-                auctionPrivacyContextFactory,
-                debugResolver,
-                jacksonMapper,
-                geoLocationServiceWrapper,
-                bidAdjustmentsEnricher);
-
-        given(requestBody.asString()).willReturn("body");
-
-        // when
-        final Future<?> future = target.parseRequest(routingContext, 0L);
-
-        // then
-        assertThat(future.failed()).isTrue();
-        assertThat(future.cause())
-                .isInstanceOf(InvalidRequestException.class)
-                .hasMessage("Request size exceeded max size of 1 bytes.");
-    }
-
     @Test
     public void shouldReturnFailedFutureIfRequestBodyCouldNotBeParsed() {
         // given