From d6fe135ec464b1290905ed162a07936452d3e18d Mon Sep 17 00:00:00 2001 From: xloya <982052490@qq.com> Date: Tue, 30 Apr 2024 19:17:56 +0800 Subject: [PATCH] [#3165] fix(client-java): Fix the http client `toJson` encode without UTF-8 (#3179) ### What changes were proposed in this pull request? Currently in the java client, the json result is not encoded with `UTF-8` when requesting the server, which will cause some Chinese characters to be garbled.It will use `ISO_8859_1` as default. This PR fixed this. ![image](https://github.com/datastrato/gravitino/assets/26177232/9342dd32-1ded-4670-a3c7-37d9a5673955) ### Why are the changes needed? Fix: #3165 ### How was this patch tested? Add some ITs. --------- Co-authored-by: xiaojiebao --- .../integration/test/HadoopCatalogIT.java | 27 +++++++++++++++++++ .../gravitino/client/HTTPClient.java | 2 +- .../integration/test/client/CatalogIT.java | 21 +++++++++++++++ .../integration/test/client/MetalakeIT.java | 19 +++++++++++++ 4 files changed, 68 insertions(+), 1 deletion(-) diff --git a/catalogs/catalog-hadoop/src/test/java/com/datastrato/gravitino/catalog/hadoop/integration/test/HadoopCatalogIT.java b/catalogs/catalog-hadoop/src/test/java/com/datastrato/gravitino/catalog/hadoop/integration/test/HadoopCatalogIT.java index 916f7c6183a..6668d7fb694 100644 --- a/catalogs/catalog-hadoop/src/test/java/com/datastrato/gravitino/catalog/hadoop/integration/test/HadoopCatalogIT.java +++ b/catalogs/catalog-hadoop/src/test/java/com/datastrato/gravitino/catalog/hadoop/integration/test/HadoopCatalogIT.java @@ -197,6 +197,33 @@ public void testCreateFileset() throws IOException { Fileset.Type.MANAGED, fileset3.type(), "fileset type should be MANAGED by default"); } + @Test + public void testCreateFilesetWithChinese() throws IOException { + // create fileset + String filesetName = "test_create_fileset_with_chinese"; + String storageLocation = storageLocation(filesetName) + "/中文目录test"; + Assertions.assertFalse( + hdfs.exists(new Path(storageLocation)), "storage location should not exists"); + Fileset fileset = + createFileset( + filesetName, + "这是中文comment", + Fileset.Type.MANAGED, + storageLocation, + ImmutableMap.of("k1", "v1", "test", "中文测试test", "中文key", "test1")); + + // verify fileset is created + assertFilesetExists(filesetName); + Assertions.assertNotNull(fileset, "fileset should be created"); + Assertions.assertEquals("这是中文comment", fileset.comment()); + Assertions.assertEquals(Fileset.Type.MANAGED, fileset.type()); + Assertions.assertEquals(storageLocation, fileset.storageLocation()); + Assertions.assertEquals(3, fileset.properties().size()); + Assertions.assertEquals("v1", fileset.properties().get("k1")); + Assertions.assertEquals("中文测试test", fileset.properties().get("test")); + Assertions.assertEquals("test1", fileset.properties().get("中文key")); + } + @Test public void testExternalFileset() throws IOException { // create fileset diff --git a/clients/client-java/src/main/java/com/datastrato/gravitino/client/HTTPClient.java b/clients/client-java/src/main/java/com/datastrato/gravitino/client/HTTPClient.java index 71d10f4aea9..74ed72cafe4 100644 --- a/clients/client-java/src/main/java/com/datastrato/gravitino/client/HTTPClient.java +++ b/clients/client-java/src/main/java/com/datastrato/gravitino/client/HTTPClient.java @@ -785,7 +785,7 @@ public HTTPClient build() { private StringEntity toJson(Object requestBody) { try { - return new StringEntity(mapper.writeValueAsString(requestBody)); + return new StringEntity(mapper.writeValueAsString(requestBody), StandardCharsets.UTF_8); } catch (JsonProcessingException e) { throw new RESTException(e, "Failed to write request body: %s", requestBody); } diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/client/CatalogIT.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/client/CatalogIT.java index bc86ef25a9d..451986eeea3 100644 --- a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/client/CatalogIT.java +++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/client/CatalogIT.java @@ -112,6 +112,27 @@ public void testCreateCatalogWithoutProperties() { metalake.dropCatalog(catalogIdent); } + @Test + public void testCreateCatalogWithChinese() { + String catalogName = GravitinoITUtils.genRandomName("catalogz"); + NameIdentifier catalogIdent = NameIdentifier.of(metalakeName, catalogName); + Assertions.assertFalse(metalake.catalogExists(catalogIdent)); + + Map properties = Maps.newHashMap(); + properties.put("metastore.uris", hmsUri); + metalake.createCatalog( + catalogIdent, Catalog.Type.RELATIONAL, "hive", "这是中文comment", properties); + Assertions.assertTrue(metalake.catalogExists(catalogIdent)); + Catalog catalog = metalake.loadCatalog(catalogIdent); + Assertions.assertEquals(catalogName, catalog.name()); + Assertions.assertEquals(Catalog.Type.RELATIONAL, catalog.type()); + Assertions.assertEquals("hive", catalog.provider()); + Assertions.assertEquals("这是中文comment", catalog.comment()); + Assertions.assertTrue(catalog.properties().containsKey("metastore.uris")); + + metalake.dropCatalog(catalogIdent); + } + @Test public void testListCatalogsInfo() { String relCatalogName = GravitinoITUtils.genRandomName("rel_catalog_"); diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/client/MetalakeIT.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/client/MetalakeIT.java index 11abdd53f3b..d51f53029b6 100644 --- a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/client/MetalakeIT.java +++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/client/MetalakeIT.java @@ -157,6 +157,25 @@ public void testCreateMetalake() { }); } + @Test + public void testCreateMetalakeWithChinese() { + client.createMetalake( + NameIdentifier.parse(metalakeNameA), "这是中文comment", Collections.emptyMap()); + GravitinoMetalake metalake = client.loadMetalake(NameIdentifier.of(metalakeNameA)); + assertEquals(metalakeNameA, metalake.name()); + assertEquals("这是中文comment", metalake.comment()); + assertEquals(AuthConstants.ANONYMOUS_USER, metalake.auditInfo().creator()); + + // Test metalake name already exists + Map emptyMap = Collections.emptyMap(); + NameIdentifier exists = NameIdentifier.parse(metalakeNameA); + assertThrows( + MetalakeAlreadyExistsException.class, + () -> { + client.createMetalake(exists, "metalake A comment", emptyMap); + }); + } + @Test public void testDropMetalakes() { GravitinoMetalake metalakeA =