Skip to content

fix: limit request body size and stream multipart form data #6950

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
hengyuss wants to merge 1 commit into
base: master
Choose a base branch
from
Open

fix: limit request body size and stream multipart form data #6950

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion .../shenyu-plugin-base/src/main/java/org/apache/shenyu/plugin/base/utils/MediaTypeUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ public static boolean isByteType(final MediaType mediaType) {
//APPLICATION_STREAM_JSON is deprecated
|| MediaType.APPLICATION_NDJSON.isCompatibleWith(mediaType)
|| MediaType.APPLICATION_PDF.isCompatibleWith(mediaType)
|| MediaType.APPLICATION_OCTET_STREAM.isCompatibleWith(mediaType);
|| MediaType.APPLICATION_OCTET_STREAM.isCompatibleWith(mediaType)
|| MediaType.MULTIPART_FORM_DATA.isCompatibleWith(mediaType);
}
}
8 changes: 6 additions & 2 deletions ...-plugin-httpclient/src/main/java/org/apache/shenyu/plugin/httpclient/WebClientPlugin.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,13 +43,17 @@ public class WebClientPlugin extends AbstractHttpClientPlugin<ResponseEntity<Flu

private final WebClient webClient;

private final int maxInMemorySize;

/**
* Instantiates a new Web client plugin.
*
* @param webClient the web client
* @param maxInMemorySize the maximum number of bytes to buffer in memory
*/
public WebClientPlugin(final WebClient webClient) {
public WebClientPlugin(final WebClient webClient, final int maxInMemorySize) {
this.webClient = webClient;
this.maxInMemorySize = maxInMemorySize;
}

@Override
Expand Down Expand Up @@ -80,7 +84,7 @@ protected Mono<ResponseEntity<Flux<DataBuffer>>> doRequest(final ServerWebExchan
return outputMessage.writeWith(body);
}
// fix chinese garbled code
return outputMessage.writeWith(DataBufferUtils.join(body));
return outputMessage.writeWith(DataBufferUtils.join(body, maxInMemorySize));
});
}
final WebClient.ResponseSpec responseSpec = requestHeadersSpec
Expand Down
44 changes: 39 additions & 5 deletions ...gin-httpclient/src/test/java/org/apache/shenyu/plugin/httpclient/WebClientPluginTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,19 +34,29 @@
import org.mockito.junit.jupiter.MockitoSettings;
import org.mockito.quality.Strictness;
import org.springframework.context.ConfigurableApplicationContext;
import org.springframework.core.io.buffer.DataBuffer;
import org.springframework.core.io.buffer.DataBufferLimitException;
import org.springframework.core.io.buffer.DefaultDataBufferFactory;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpMethod;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.mock.http.client.reactive.MockClientHttpRequest;
import org.springframework.mock.http.server.reactive.MockServerHttpRequest;
import org.springframework.mock.web.server.MockServerWebExchange;
import org.springframework.web.reactive.function.BodyInserter;
import org.springframework.web.reactive.function.client.ClientRequest;
import org.springframework.web.reactive.function.client.ClientResponse;
import org.springframework.web.reactive.function.client.ExchangeFunction;
import org.springframework.web.reactive.function.client.WebClient;
import org.springframework.web.server.ServerWebExchange;
import reactor.core.publisher.Mono;
import reactor.core.publisher.Flux;
import reactor.test.StepVerifier;

import java.net.URI;
import java.nio.charset.StandardCharsets;
import java.util.Collections;

import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
Expand Down Expand Up @@ -77,7 +87,7 @@ public void setup() {
when(context.getBean(ShenyuResult.class)).thenReturn(mock(ShenyuResult.class));

WebClient webClient = mockWebClientOK();
webClientPlugin = new WebClientPlugin(webClient);
webClientPlugin = new WebClientPlugin(webClient, Constants.BYTES_PER_MB);
}

/**
Expand All @@ -90,7 +100,7 @@ public void testExecuted() {
ServerWebExchange exchangeNoPathTest = MockServerWebExchange
.from(MockServerHttpRequest.get("/test").build());
exchangeNoPathTest.getAttributes().put(Constants.CONTEXT, mock(ShenyuContext.class));
WebClientPlugin webClientPluginNoPathTest = new WebClientPlugin(webClientNoPathTest);
WebClientPlugin webClientPluginNoPathTest = new WebClientPlugin(webClientNoPathTest, Constants.BYTES_PER_MB);
Mono<Void> monoNoPathTest = webClientPluginNoPathTest.execute(exchangeNoPathTest, chainNoPathTest);
StepVerifier.create(monoNoPathTest).expectSubscription().verifyComplete();

Expand All @@ -100,19 +110,19 @@ public void testExecuted() {
.from(MockServerHttpRequest.post("/test123?param=1").build());
exchangePostTest.getAttributes().put(Constants.CONTEXT, mock(ShenyuContext.class));
exchangePostTest.getAttributes().put(Constants.HTTP_URI, URI.create("/test123?param=1"));
WebClientPlugin webClientPluginPostTest = new WebClientPlugin(webClientPostTest);
WebClientPlugin webClientPluginPostTest = new WebClientPlugin(webClientPostTest, Constants.BYTES_PER_MB);
Mono<Void> monoPostTest = webClientPluginPostTest.execute(exchangePostTest, chainPostTest);
StepVerifier.create(monoPostTest).expectSubscription().verifyError();

final ShenyuPluginChain chainOkTest = mock(ShenyuPluginChain.class);
final WebClient webClientOkTest = mockWebClientOK();
WebClientPlugin webClientPluginOkTest = new WebClientPlugin(webClientOkTest);
WebClientPlugin webClientPluginOkTest = new WebClientPlugin(webClientOkTest, Constants.BYTES_PER_MB);
Mono<Void> monoOkTest = webClientPluginOkTest.execute(generateServerWebExchange(), chainOkTest);
StepVerifier.create(monoOkTest).expectSubscription().verifyError();

final ShenyuPluginChain chainErrorTest = mock(ShenyuPluginChain.class);
final WebClient webClientErrorTest = mockWebClientError();
WebClientPlugin webClientPluginErrorTest = new WebClientPlugin(webClientErrorTest);
WebClientPlugin webClientPluginErrorTest = new WebClientPlugin(webClientErrorTest, Constants.BYTES_PER_MB);
Mono<Void> monoErrorTest = webClientPluginErrorTest.execute(generateServerWebExchange(), chainErrorTest);
StepVerifier.create(monoErrorTest).expectSubscription().verifyError();
}
Expand Down Expand Up @@ -147,6 +157,30 @@ public void testNamed() {
assertEquals(PluginEnum.WEB_CLIENT.getName(), webClientPlugin.named());
}

/**
* Test that a non-binary request body cannot exceed the configured in-memory limit.
*/
@Test
public void testRequestBodyExceedsMaxInMemorySize() {
final int maxInMemorySize = 4;
final WebClientPlugin plugin = new WebClientPlugin(mockWebClientOK(), maxInMemorySize);
final DataBuffer body = DefaultDataBufferFactory.sharedInstance
.wrap("12345".getBytes(StandardCharsets.UTF_8));
final ServerWebExchange exchange = MockServerWebExchange.from(MockServerHttpRequest.post("/test")
.contentType(MediaType.APPLICATION_JSON)
.build());

plugin.doRequest(exchange, HttpMethod.POST.name(), URI.create("/test"), Flux.just(body)).subscribe();
final ClientRequest request = captor.getValue();
final MockClientHttpRequest outputMessage = new MockClientHttpRequest(HttpMethod.POST, URI.create("/test"));
final BodyInserter.Context context = mock(BodyInserter.Context.class);
when(context.messageWriters()).thenReturn(Collections.emptyList());

StepVerifier.create(request.body().insert(outputMessage, context))
.expectError(DataBufferLimitException.class)
.verify();
}

private ServerWebExchange generateServerWebExchange() {
ServerWebExchange exchange = MockServerWebExchange.from(MockServerHttpRequest.get("/test").build());
exchange.getAttributes().put(Constants.CONTEXT, mock(ShenyuContext.class));
Expand Down
6 changes: 4 additions & 2 deletions ...org/apache/shenyu/springboot/starter/plugin/httpclient/HttpClientPluginConfiguration.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,15 +102,17 @@ static class WebClientConfiguration {
public ShenyuPlugin webClientPlugin(
final HttpClientProperties properties,
final ObjectProvider<HttpClient> httpClient) {
int maxInMemorySize =
properties.getMaxInMemorySize() * Constants.BYTES_PER_MB;
WebClient webClient = WebClient.builder()
// fix Exceeded limit on max bytes to buffer
// detail see https://stackoverflow.com/questions/59326351/configure-spring-codec-max-in-memory-size-when-using-reactiveelasticsearchclient
.exchangeStrategies(ExchangeStrategies.builder()
.codecs(codecs -> codecs.defaultCodecs().maxInMemorySize(properties.getMaxInMemorySize() * Constants.BYTES_PER_MB))
.codecs(codecs -> codecs.defaultCodecs().maxInMemorySize(maxInMemorySize))
.build())
.clientConnector(new ReactorClientHttpConnector(Objects.requireNonNull(httpClient.getIfAvailable())))
.build();
return new WebClientPlugin(webClient);
return new WebClientPlugin(webClient, maxInMemorySize);
}
}

Expand Down
Loading