From 80fa6b1cc527c6df6a9b0cb8b1cd2d542f0dfcd6 Mon Sep 17 00:00:00 2001 From: zgq <203083679@qq.com> Date: Sat, 20 Apr 2024 23:29:08 +0800 Subject: [PATCH 01/11] table data export --- .../api/param/user/TableExportDataParam.java | 16 ++++ .../controller/rdb/DatabaseController.java | 1 + .../api/controller/rdb/TableController.java | 21 ++++- .../rdb/converter/RdbWebConverter.java | 4 + .../strategy/ExportDBData2CsvStrategy.java | 76 ++++++++++++------- .../strategy/ExportDBData2ExcelStrategy.java | 69 +++++++++++------ .../strategy/ExportDBData2JsonStrategy.java | 74 ++++++++++-------- .../strategy/ExportDBData2SqlStrategy.java | 21 ++++- .../export/strategy/ExportDBDataStrategy.java | 67 ++++++++++++---- .../request/DatabaseExportDataRequest.java | 4 +- .../rdb/request/TableExportDataRequest.java | 17 +++++ 11 files changed, 269 insertions(+), 101 deletions(-) create mode 100644 chat2db-server/chat2db-server-domain/chat2db-server-domain-api/src/main/java/ai/chat2db/server/domain/api/param/user/TableExportDataParam.java create mode 100644 chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/request/TableExportDataRequest.java diff --git a/chat2db-server/chat2db-server-domain/chat2db-server-domain-api/src/main/java/ai/chat2db/server/domain/api/param/user/TableExportDataParam.java b/chat2db-server/chat2db-server-domain/chat2db-server-domain-api/src/main/java/ai/chat2db/server/domain/api/param/user/TableExportDataParam.java new file mode 100644 index 000000000..020c42fd8 --- /dev/null +++ b/chat2db-server/chat2db-server-domain/chat2db-server-domain-api/src/main/java/ai/chat2db/server/domain/api/param/user/TableExportDataParam.java @@ -0,0 +1,16 @@ +package ai.chat2db.server.domain.api.param.user; + +import ai.chat2db.server.domain.api.param.datasource.DatabaseExportDataParam; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * @author: zgq + */ +@Data +@AllArgsConstructor +@NoArgsConstructor +public class TableExportDataParam extends DatabaseExportDataParam { + private String tableName; +} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/DatabaseController.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/DatabaseController.java index 6348db871..0ca354a21 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/DatabaseController.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/DatabaseController.java @@ -139,6 +139,7 @@ public void exportData(@Valid @RequestBody DatabaseExportDataRequest request, Ht ExportDBDataStrategy service = (ExportDBDataStrategy) constructor.newInstance(); service.doExport(param, response); } catch (Exception e) { + response.reset(); throw new RuntimeException(e); } diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/TableController.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/TableController.java index 23a9f32ec..cf2df2618 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/TableController.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/TableController.java @@ -1,6 +1,7 @@ package ai.chat2db.server.web.api.controller.rdb; import ai.chat2db.server.domain.api.param.*; +import ai.chat2db.server.domain.api.param.user.TableExportDataParam; import ai.chat2db.server.domain.api.service.DatabaseService; import ai.chat2db.server.domain.api.service.DlTemplateService; import ai.chat2db.server.domain.api.service.TableService; @@ -12,20 +13,22 @@ import ai.chat2db.server.web.api.aspect.ConnectionInfoAspect; import ai.chat2db.server.web.api.controller.ai.EmbeddingController; import ai.chat2db.server.web.api.controller.rdb.converter.RdbWebConverter; +import ai.chat2db.server.web.api.controller.rdb.data.export.strategy.ExportDBDataStrategy; +import ai.chat2db.server.web.api.controller.rdb.factory.ExportDBDataStrategyFactory; import ai.chat2db.server.web.api.controller.rdb.request.*; import ai.chat2db.server.web.api.controller.rdb.vo.ColumnVO; import ai.chat2db.server.web.api.controller.rdb.vo.IndexVO; import ai.chat2db.server.web.api.controller.rdb.vo.SqlVO; import ai.chat2db.server.web.api.controller.rdb.vo.TableVO; import ai.chat2db.spi.model.*; -import ai.chat2db.spi.sql.Chat2DBContext; -import ai.chat2db.spi.sql.ConnectInfo; import com.google.common.collect.Lists; +import jakarta.servlet.http.HttpServletResponse; import jakarta.validation.Valid; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.*; +import java.lang.reflect.Constructor; import java.util.List; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -255,4 +258,18 @@ public ActionResult delete(@Valid @RequestBody TableDeleteRequest request) { DropParam dropParam = rdbWebConverter.tableDelete2dropParam(request); return tableService.drop(dropParam); } + @PostMapping("/export_data") + public void exportData(@Valid @RequestBody TableExportDataRequest request, HttpServletResponse response) { + Class targetClass = ExportDBDataStrategyFactory.get(request.getExportType()); + response.setCharacterEncoding("utf-8"); + TableExportDataParam param = rdbWebConverter.request2param(request); + try { + Constructor constructor = targetClass.getDeclaredConstructor(); + ExportDBDataStrategy service = (ExportDBDataStrategy) constructor.newInstance(); + service.doExport(param, response); + } catch (Exception e) { + throw new RuntimeException(e); + } + + } } diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/converter/RdbWebConverter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/converter/RdbWebConverter.java index f9c529a5b..eea80ea4b 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/converter/RdbWebConverter.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/converter/RdbWebConverter.java @@ -3,6 +3,7 @@ import java.util.List; import ai.chat2db.server.domain.api.param.*; +import ai.chat2db.server.domain.api.param.user.TableExportDataParam; import ai.chat2db.server.web.api.controller.data.source.vo.DatabaseVO; import ai.chat2db.server.web.api.controller.rdb.request.*; import ai.chat2db.server.web.api.controller.rdb.vo.ColumnVO; @@ -256,4 +257,7 @@ public abstract class RdbWebConverter { public abstract TablePageQueryParam schemaReq2page(EsTableSchemaRequest request); public abstract DmlSqlCopyParam dmlRequest2param(DmlSqlCopyRequest request) ; + + public abstract TableExportDataParam request2param(TableExportDataRequest request); + } diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2CsvStrategy.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2CsvStrategy.java index d091bf0ca..c626512dd 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2CsvStrategy.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2CsvStrategy.java @@ -4,8 +4,12 @@ import ai.chat2db.spi.util.ResultSetUtils; import com.alibaba.excel.EasyExcel; import com.alibaba.excel.support.ExcelTypeEnum; +import jakarta.servlet.http.HttpServletResponse; +import org.jetbrains.annotations.NotNull; import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStream; import java.sql.Connection; import java.sql.ResultSet; import java.sql.ResultSetMetaData; @@ -13,42 +17,62 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; -import java.util.Objects; import java.util.stream.Collectors; -public class ExportDBData2CsvStrategy extends ExportDBDataStrategy { +public class ExportDBData2CsvStrategy extends ExportDBDataStrategy { public ExportDBData2CsvStrategy() { suffix = ExportFileSuffix.CSV.getSuffix(); - contentType = "application/zip"; + contentType = "text/csv"; } @Override - protected ByteArrayOutputStream exportData(Connection connection, String databaseName, String schemaName, String tableName) throws SQLException { - String sql; - if (Objects.isNull(schemaName)) { - sql = String.format("select * from %s", tableName); - } else { - sql = String.format("select * from %s.%s", schemaName, tableName); - } + protected ByteArrayOutputStream exportData(Connection connection, String databaseName, + String schemaName, String tableName) throws SQLException { ByteArrayOutputStream byteOut = new ByteArrayOutputStream(); - try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { - ResultSetMetaData metaData = resultSet.getMetaData(); - List> headList = ResultSetUtils.getRsHeader(resultSet) - .stream() - .map(Collections::singletonList) - .collect(Collectors.toList()); - int columnCount = metaData.getColumnCount(); - List> dataList = new ArrayList<>(); - while (resultSet.next()) { - List row = new ArrayList<>(); - for (int i = 1; i <= columnCount; i++) { - row.add(resultSet.getString(i)); - } - dataList.add(row); + export2CSV(connection, schemaName, tableName, byteOut); + return byteOut; + } + + @Override + protected void exportData(HttpServletResponse response, Connection connection, + String databaseName, String schemaName, String tableName) throws SQLException { + try { + export2CSV(connection, schemaName, tableName, response.getOutputStream()); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + private void export2CSV(Connection connection, String schemaName, String tableName, OutputStream byteOut) throws SQLException { + try (ResultSet resultSet = connection.createStatement().executeQuery(buildQuerySql(schemaName, tableName))) { + EasyExcel.write(byteOut) + .excelType(ExcelTypeEnum.CSV) + .sheet(tableName) + .head(getHeadList(resultSet)) + .doWrite(getDataList(resultSet)); + } + } + + @NotNull + private List> getDataList(ResultSet resultSet) throws SQLException { + ResultSetMetaData metaData = resultSet.getMetaData(); + int columnCount = metaData.getColumnCount(); + List> dataList = new ArrayList<>(); + while (resultSet.next()) { + List row = new ArrayList<>(); + for (int i = 1; i <= columnCount; i++) { + row.add(resultSet.getString(i)); } - EasyExcel.write(byteOut).excelType(ExcelTypeEnum.CSV).sheet(tableName).head(headList).doWrite(dataList); + dataList.add(row); } - return byteOut; + return dataList; + } + + @NotNull + private List> getHeadList(ResultSet resultSet) { + return ResultSetUtils.getRsHeader(resultSet) + .stream() + .map(Collections::singletonList) + .collect(Collectors.toList()); } } \ No newline at end of file diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2ExcelStrategy.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2ExcelStrategy.java index b8fc38ee8..c5c0780c1 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2ExcelStrategy.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2ExcelStrategy.java @@ -3,8 +3,12 @@ import ai.chat2db.server.domain.api.enums.ExportFileSuffix; import ai.chat2db.spi.util.ResultSetUtils; import com.alibaba.excel.EasyExcel; +import jakarta.servlet.http.HttpServletResponse; +import org.jetbrains.annotations.NotNull; import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStream; import java.sql.Connection; import java.sql.ResultSet; import java.sql.ResultSetMetaData; @@ -12,42 +16,61 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; -import java.util.Objects; import java.util.stream.Collectors; public class ExportDBData2ExcelStrategy extends ExportDBDataStrategy { public ExportDBData2ExcelStrategy() { suffix = ExportFileSuffix.EXCEL.getSuffix(); - contentType = "application/zip"; + contentType = "application/vnd.ms-excel"; } @Override - protected ByteArrayOutputStream exportData(Connection connection, String databaseName, String schemaName, String tableName) throws SQLException { - String sql; - if (Objects.isNull(schemaName)) { - sql = String.format("select * from %s", tableName); - } else { - sql = String.format("select * from %s.%s", schemaName, tableName); - } + protected ByteArrayOutputStream exportData(Connection connection, String databaseName, + String schemaName, String tableName) throws SQLException { ByteArrayOutputStream byteOut = new ByteArrayOutputStream(); - try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { + export2EXCEL(connection, schemaName, tableName, byteOut); + return byteOut; + } + + @Override + protected void exportData(HttpServletResponse response, Connection connection, String databaseName, + String schemaName, String tableName) throws SQLException { + try { + export2EXCEL(connection, schemaName, tableName, response.getOutputStream()); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private void export2EXCEL(Connection connection, String schemaName, String tableName, OutputStream byteOut) throws SQLException { + try (ResultSet resultSet = connection.createStatement().executeQuery(buildQuerySql(schemaName, tableName))) { ResultSetMetaData metaData = resultSet.getMetaData(); int columnCount = metaData.getColumnCount(); - List> headList = ResultSetUtils.getRsHeader(resultSet) - .stream() - .map(Collections::singletonList) - .collect(Collectors.toList()); - List> dataList = new ArrayList<>(); - while (resultSet.next()) { - List row = new ArrayList<>(); - for (int i = 1; i <= columnCount; i++) { - row.add(resultSet.getString(i)); - } - dataList.add(row); - } + List> headList = getHeadList(resultSet); + List> dataList = getDataList(resultSet, columnCount); EasyExcel.write(byteOut).sheet(tableName).head(headList).doWrite(dataList); } - return byteOut; + } + + @NotNull + private List> getDataList(ResultSet resultSet, int columnCount) throws SQLException { + List> dataList = new ArrayList<>(); + while (resultSet.next()) { + List row = new ArrayList<>(); + for (int i = 1; i <= columnCount; i++) { + row.add(resultSet.getString(i)); + } + dataList.add(row); + } + return dataList; + } + + @NotNull + private List> getHeadList(ResultSet resultSet) { + return ResultSetUtils.getRsHeader(resultSet) + .stream() + .map(Collections::singletonList) + .collect(Collectors.toList()); } } \ No newline at end of file diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2JsonStrategy.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2JsonStrategy.java index 47c8a59d8..325657043 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2JsonStrategy.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2JsonStrategy.java @@ -3,6 +3,8 @@ import ai.chat2db.server.domain.api.enums.ExportFileSuffix; import ai.chat2db.server.tools.base.excption.BusinessException; import com.fasterxml.jackson.databind.ObjectMapper; +import jakarta.servlet.http.HttpServletResponse; +import org.jetbrains.annotations.NotNull; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -13,49 +15,61 @@ import java.sql.ResultSet; import java.sql.ResultSetMetaData; import java.sql.SQLException; -import java.util.*; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; public class ExportDBData2JsonStrategy extends ExportDBDataStrategy { public ExportDBData2JsonStrategy() { suffix = ExportFileSuffix.JSON.getSuffix(); - contentType = "application/zip"; + contentType = "application/json"; } @Override protected ByteArrayOutputStream exportData(Connection connection, String databaseName, String schemaName, String tableName) throws SQLException { ByteArrayOutputStream byteOut = new ByteArrayOutputStream(); - try (PrintWriter writer = new PrintWriter(new OutputStreamWriter(byteOut, StandardCharsets.UTF_8))) { - String sql; - if (Objects.isNull(schemaName)) { - sql = String.format("SELECT * FROM %s", tableName); - } else { - sql = String.format("SELECT * FROM %s.%s", schemaName, tableName); - } - try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { - ResultSetMetaData metaData = resultSet.getMetaData(); - int columnCount = metaData.getColumnCount(); - List> data = new ArrayList<>(); - - while (resultSet.next()) { - Map row = new LinkedHashMap<>(); - for (int i = 1; i <= columnCount; i++) { - row.put(metaData.getColumnName(i), resultSet.getObject(i)); - } - data.add(row); - } - - ObjectMapper objectMapper = new ObjectMapper(); - try { - String jsonString = objectMapper.writeValueAsString(data); - writer.println(jsonString); - } catch (IOException e) { - throw new BusinessException("data.export2Json.error",data.toArray(),e); - } - } + try (ResultSet resultSet = connection.createStatement().executeQuery(buildQuerySql(schemaName, tableName)); + PrintWriter writer = new PrintWriter(new OutputStreamWriter(byteOut, StandardCharsets.UTF_8))) { + export2JSON(resultSet, writer); } return byteOut; } + @Override + protected void exportData(HttpServletResponse response, Connection connection, String databaseName, + String schemaName, String tableName) throws SQLException { + try (ResultSet resultSet = connection.createStatement().executeQuery(buildQuerySql(schemaName, tableName))) { + export2JSON(resultSet, response.getWriter()); + }catch (IOException e){ + throw new RuntimeException(e); + } + } + private void export2JSON(ResultSet resultSet, PrintWriter writer) throws SQLException { + List> data = getDataMap(resultSet); + ObjectMapper objectMapper = new ObjectMapper(); + try { + String jsonString = objectMapper.writeValueAsString(data); + writer.println(jsonString); + } catch (IOException e) { + throw new BusinessException("data.export2Json.error", data.toArray(), e); + } + } + + @NotNull + private List> getDataMap(ResultSet resultSet) throws SQLException { + ResultSetMetaData metaData = resultSet.getMetaData(); + int columnCount = metaData.getColumnCount(); + List> data = new ArrayList<>(); + while (resultSet.next()) { + Map row = new LinkedHashMap<>(); + for (int i = 1; i <= columnCount; i++) { + row.put(metaData.getColumnName(i), resultSet.getObject(i)); + } + data.add(row); + } + return data; + } } \ No newline at end of file diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2SqlStrategy.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2SqlStrategy.java index 1e0577474..dcd81e72a 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2SqlStrategy.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2SqlStrategy.java @@ -2,8 +2,10 @@ import ai.chat2db.server.domain.api.enums.ExportFileSuffix; import ai.chat2db.spi.sql.Chat2DBContext; +import jakarta.servlet.http.HttpServletResponse; import java.io.ByteArrayOutputStream; +import java.io.IOException; import java.io.OutputStreamWriter; import java.io.PrintWriter; import java.nio.charset.StandardCharsets; @@ -18,16 +20,29 @@ public class ExportDBData2SqlStrategy extends ExportDBDataStrategy { public ExportDBData2SqlStrategy() { suffix = ExportFileSuffix.SQL.getSuffix(); - contentType = "application/zip"; + contentType = "text/sql"; } @Override protected ByteArrayOutputStream exportData(Connection connection, String databaseName, String schemaName, String tableName) throws SQLException { ByteArrayOutputStream byteOut = new ByteArrayOutputStream(); try (PrintWriter writer = new PrintWriter(new OutputStreamWriter(byteOut, StandardCharsets.UTF_8))) { - String sql = Chat2DBContext.getDBManage().exportDatabaseData(connection, databaseName, schemaName, tableName); - writer.println(sql); + writer.println(export2SQL(connection, databaseName, schemaName, tableName)); } return byteOut; } + @Override + protected void exportData(HttpServletResponse response, Connection connection, + String databaseName, String schemaName, String tableName) throws SQLException { + try { + response.getWriter().print(export2SQL(connection, databaseName, schemaName, tableName)); + } catch (IOException e) { + throw new RuntimeException(e); + } + + } + + private String export2SQL(Connection connection, String databaseName, String schemaName, String tableName) throws SQLException { + return Chat2DBContext.getDBManage().exportDatabaseData(connection, databaseName, schemaName, tableName); + } } \ No newline at end of file diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBDataStrategy.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBDataStrategy.java index cb4a18209..d1b75c518 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBDataStrategy.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBDataStrategy.java @@ -2,14 +2,17 @@ import ai.chat2db.server.domain.api.enums.ExportFileSuffix; import ai.chat2db.server.domain.api.param.datasource.DatabaseExportDataParam; +import ai.chat2db.server.domain.api.param.user.TableExportDataParam; import ai.chat2db.spi.sql.Chat2DBContext; import jakarta.servlet.ServletOutputStream; import jakarta.servlet.http.HttpServletResponse; import lombok.AllArgsConstructor; import lombok.Getter; import lombok.NoArgsConstructor; +import org.springframework.http.MediaType; import java.io.ByteArrayOutputStream; +import java.io.IOException; import java.sql.Connection; import java.sql.SQLException; import java.util.List; @@ -27,39 +30,73 @@ public abstract class ExportDBDataStrategy { public String suffix; - public String contentType; + public String contentType = "application/zip"; + public void doExport(DatabaseExportDataParam param, HttpServletResponse response) { String databaseName = param.getDatabaseName(); String schemaName = param.getSchemaName(); setResponseHeaders(param, response); - try (ServletOutputStream outputStream = response.getOutputStream(); - ZipOutputStream zipOut = new ZipOutputStream(outputStream); - Connection connection = Chat2DBContext.getConnection();) { - List tableNames = Chat2DBContext.getMetaData().tableNames(connection, databaseName, schemaName, null); - tableNames.addAll(Chat2DBContext.getMetaData().viewNames(connection, databaseName, schemaName)); - for (String tableName : tableNames) { - String fileName = tableName + getSuffix(); - zipOut.putNextEntry(new ZipEntry(fileName)); - ByteArrayOutputStream byteOut = exportData(connection, databaseName, schemaName, tableName); - byteOut.writeTo(zipOut); - zipOut.closeEntry(); - byteOut.close(); + try (Connection connection = Chat2DBContext.getConnection()) { + if (param instanceof TableExportDataParam){ + doTableDataExport(((TableExportDataParam) param).getTableName(),response,databaseName,schemaName,connection); + }else { + doDbDataExport(response, databaseName, schemaName, connection); } } catch (Exception e) { throw new RuntimeException(e); } } + private void doDbDataExport(HttpServletResponse response, String databaseName, + String schemaName, Connection connection) throws IOException, SQLException { + ServletOutputStream outputStream = response.getOutputStream(); + ZipOutputStream zipOut = new ZipOutputStream(outputStream); + List tableNames = Chat2DBContext.getMetaData().tableNames(connection, databaseName, schemaName, null); + tableNames.addAll(Chat2DBContext.getMetaData().viewNames(connection, databaseName, schemaName)); + for (String tableName : tableNames) { + String fileName = tableName + getSuffix(); + zipOut.putNextEntry(new ZipEntry(fileName)); + ByteArrayOutputStream byteOut = exportData(connection, databaseName, schemaName, tableName); + byteOut.writeTo(zipOut); + zipOut.closeEntry(); + byteOut.close(); + } + } + + private void doTableDataExport(String tableName, HttpServletResponse response, String databaseName, + String schemaName, Connection connection) throws SQLException { + exportData(response, connection, databaseName, schemaName, tableName); + } + private void setResponseHeaders(DatabaseExportDataParam param, HttpServletResponse response) { - response.setContentType(contentType); - response.setHeader("Content-disposition", "attachment;filename*=utf-8''" + getFileName(param) + ExportFileSuffix.ZIP.getSuffix()); + if (param instanceof TableExportDataParam) { + String tableName = ((TableExportDataParam) param).getTableName(); + response.setContentType(contentType); + response.setHeader("Content-disposition", "attachment;filename*=utf-8''" + tableName + suffix); + } else { + response.setContentType("application/zip"); + response.setHeader("Content-disposition", "attachment;filename*=utf-8''" + getFileName(param) + ExportFileSuffix.ZIP.getSuffix()); + } } + protected String getFileName(DatabaseExportDataParam param) { return Objects.isNull(param.getSchemaName()) ? param.getDatabaseName() : param.getSchemaName(); } + public String buildQuerySql(String schemaName, String tableName) { + String sql; + if (Objects.isNull(schemaName)) { + sql = String.format("select * from %s", tableName); + } else { + sql = String.format("select * from %s.%s", schemaName, tableName); + } + return sql; + } + protected abstract ByteArrayOutputStream exportData(Connection connection, String databaseName, String schemaName, String tableName) throws SQLException; + protected abstract void exportData(HttpServletResponse response, Connection connection, String databaseName, String schemaName, String tableName) throws SQLException; + } \ No newline at end of file diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/request/DatabaseExportDataRequest.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/request/DatabaseExportDataRequest.java index 41686b503..0faaac022 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/request/DatabaseExportDataRequest.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/request/DatabaseExportDataRequest.java @@ -1,7 +1,7 @@ package ai.chat2db.server.web.api.controller.rdb.request; import ai.chat2db.server.web.api.controller.data.source.request.DataSourceBaseRequest; -import jakarta.validation.constraints.NotNull; +import jakarta.validation.constraints.NotBlank; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; @@ -14,6 +14,6 @@ @AllArgsConstructor @NoArgsConstructor public class DatabaseExportDataRequest extends DataSourceBaseRequest { - @NotNull + @NotBlank private String exportType; } \ No newline at end of file diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/request/TableExportDataRequest.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/request/TableExportDataRequest.java new file mode 100644 index 000000000..b5f7f4655 --- /dev/null +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/request/TableExportDataRequest.java @@ -0,0 +1,17 @@ +package ai.chat2db.server.web.api.controller.rdb.request; + +import jakarta.validation.constraints.NotBlank; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * @author: zgq + */ +@Data +@AllArgsConstructor +@NoArgsConstructor +public class TableExportDataRequest extends DatabaseExportDataRequest { + @NotBlank + private String tableName; +} From 0116b9d92dde152c473c7d5bf4dacd4447e74358 Mon Sep 17 00:00:00 2001 From: zgq <203083679@qq.com> Date: Tue, 23 Apr 2024 13:17:04 +0800 Subject: [PATCH 02/11] support export options SQL: 1.export to update statements 2.export to multipleRow insert statements 3.containsHeader 4.support specified fields 5.support replace a new table name JSON: 1.support specified fields 2.support dataTimeFormat CSV&EXCEL: 1.containsHeader 2.support specified fields --- .../datasource/DatabaseExportDataParam.java | 7 +- .../api/param/user/TableExportDataParam.java | 16 -- .../export/data/option/ExportDataOption.java | 33 ++++ .../data/option/JSONExportDataOption.java | 23 +++ .../data/option/SQLExportDataOption.java | 17 ++ .../model/export/data/option/TableOption.java | 21 +++ .../controller/rdb/DatabaseController.java | 2 +- .../api/controller/rdb/TableController.java | 14 -- .../rdb/converter/RdbWebConverter.java | 3 - .../strategy/ExportDBData2CsvStrategy.java | 55 ++---- .../strategy/ExportDBData2ExcelStrategy.java | 53 ++---- .../strategy/ExportDBData2JsonStrategy.java | 36 +++- .../strategy/ExportDBData2SqlStrategy.java | 174 +++++++++++++++++- .../export/strategy/ExportDBDataStrategy.java | 103 ++++++++--- .../request/DatabaseExportDataRequest.java | 12 +- .../rdb/request/TableExportDataRequest.java | 17 -- 16 files changed, 413 insertions(+), 173 deletions(-) delete mode 100644 chat2db-server/chat2db-server-domain/chat2db-server-domain-api/src/main/java/ai/chat2db/server/domain/api/param/user/TableExportDataParam.java create mode 100644 chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/export/data/option/ExportDataOption.java create mode 100644 chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/export/data/option/JSONExportDataOption.java create mode 100644 chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/export/data/option/SQLExportDataOption.java create mode 100644 chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/export/data/option/TableOption.java delete mode 100644 chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/request/TableExportDataRequest.java diff --git a/chat2db-server/chat2db-server-domain/chat2db-server-domain-api/src/main/java/ai/chat2db/server/domain/api/param/datasource/DatabaseExportDataParam.java b/chat2db-server/chat2db-server-domain/chat2db-server-domain-api/src/main/java/ai/chat2db/server/domain/api/param/datasource/DatabaseExportDataParam.java index 4e562cedf..242603408 100644 --- a/chat2db-server/chat2db-server-domain/chat2db-server-domain-api/src/main/java/ai/chat2db/server/domain/api/param/datasource/DatabaseExportDataParam.java +++ b/chat2db-server/chat2db-server-domain/chat2db-server-domain-api/src/main/java/ai/chat2db/server/domain/api/param/datasource/DatabaseExportDataParam.java @@ -1,9 +1,13 @@ package ai.chat2db.server.domain.api.param.datasource; +import ai.chat2db.server.tools.common.model.export.data.option.ExportDataOption; +import ai.chat2db.server.tools.common.model.export.data.option.TableOption; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; +import java.util.List; + /** * @author: zgq * @date: 2024年03月24日 13:17 @@ -14,5 +18,6 @@ public class DatabaseExportDataParam { private String databaseName; private String schemaName; - private String exportType; + private ExportDataOption exportDataOption; + private List tableOptions; } \ No newline at end of file diff --git a/chat2db-server/chat2db-server-domain/chat2db-server-domain-api/src/main/java/ai/chat2db/server/domain/api/param/user/TableExportDataParam.java b/chat2db-server/chat2db-server-domain/chat2db-server-domain-api/src/main/java/ai/chat2db/server/domain/api/param/user/TableExportDataParam.java deleted file mode 100644 index 020c42fd8..000000000 --- a/chat2db-server/chat2db-server-domain/chat2db-server-domain-api/src/main/java/ai/chat2db/server/domain/api/param/user/TableExportDataParam.java +++ /dev/null @@ -1,16 +0,0 @@ -package ai.chat2db.server.domain.api.param.user; - -import ai.chat2db.server.domain.api.param.datasource.DatabaseExportDataParam; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; - -/** - * @author: zgq - */ -@Data -@AllArgsConstructor -@NoArgsConstructor -public class TableExportDataParam extends DatabaseExportDataParam { - private String tableName; -} diff --git a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/export/data/option/ExportDataOption.java b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/export/data/option/ExportDataOption.java new file mode 100644 index 000000000..027aad38f --- /dev/null +++ b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/export/data/option/ExportDataOption.java @@ -0,0 +1,33 @@ +package ai.chat2db.server.tools.common.model.export.data.option; + + +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotNull; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@AllArgsConstructor +@NoArgsConstructor +@JsonTypeInfo( + use = JsonTypeInfo.Id.NAME, + property = "exportType", + visible = true) +@JsonSubTypes({ + @JsonSubTypes.Type(value = ExportDataOption.class, name = "CSV"), + @JsonSubTypes.Type(value = SQLExportDataOption.class, name = "SQL"), + @JsonSubTypes.Type(value = ExportDataOption.class, name = "EXCEL"), + @JsonSubTypes.Type(value = JSONExportDataOption.class, name = "JSON") +}) +public class ExportDataOption { + @NotBlank + private String exportType; + @NotNull + public Boolean containsHeader; + +} + + diff --git a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/export/data/option/JSONExportDataOption.java b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/export/data/option/JSONExportDataOption.java new file mode 100644 index 000000000..bd52278ea --- /dev/null +++ b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/export/data/option/JSONExportDataOption.java @@ -0,0 +1,23 @@ +package ai.chat2db.server.tools.common.model.export.data.option; + +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotNull; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * @author: zgq + * @date: 2024年04月22日 18:28 + */ +@Data +@AllArgsConstructor +@NoArgsConstructor +public class JSONExportDataOption extends ExportDataOption { + + @NotBlank + private String dataTimeFormat; + @NotNull + private Boolean isTimestamps; + +} diff --git a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/export/data/option/SQLExportDataOption.java b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/export/data/option/SQLExportDataOption.java new file mode 100644 index 000000000..18e475ff0 --- /dev/null +++ b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/export/data/option/SQLExportDataOption.java @@ -0,0 +1,17 @@ +package ai.chat2db.server.tools.common.model.export.data.option; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * @author: zgq + */ +@Data +@AllArgsConstructor +@NoArgsConstructor +public class SQLExportDataOption extends ExportDataOption { + private Boolean multipleRow; + private String newTableName; + private Boolean toUpdate; +} diff --git a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/export/data/option/TableOption.java b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/export/data/option/TableOption.java new file mode 100644 index 000000000..96ca59f5c --- /dev/null +++ b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/export/data/option/TableOption.java @@ -0,0 +1,21 @@ +package ai.chat2db.server.tools.common.model.export.data.option; + +import jakarta.validation.constraints.NotEmpty; +import jakarta.validation.constraints.NotNull; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.List; + +@Data +@AllArgsConstructor +@NoArgsConstructor +public class TableOption { + @NotNull + private String tableName; + @NotEmpty + @NotNull + private List filedNames; + +} \ No newline at end of file diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/DatabaseController.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/DatabaseController.java index 0ca354a21..77b63016a 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/DatabaseController.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/DatabaseController.java @@ -131,7 +131,7 @@ public void exportDatabase(@Valid @RequestBody DatabaseExportRequest request, Ht @PostMapping("/export_data") public void exportData(@Valid @RequestBody DatabaseExportDataRequest request, HttpServletResponse response) { - Class targetClass = ExportDBDataStrategyFactory.get(request.getExportType()); + Class targetClass = ExportDBDataStrategyFactory.get(request.getExportDataOption().getExportType()); response.setCharacterEncoding("utf-8"); DatabaseExportDataParam param = databaseConverter.request2param(request); try { diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/TableController.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/TableController.java index cf2df2618..8550ac0a7 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/TableController.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/TableController.java @@ -1,7 +1,6 @@ package ai.chat2db.server.web.api.controller.rdb; import ai.chat2db.server.domain.api.param.*; -import ai.chat2db.server.domain.api.param.user.TableExportDataParam; import ai.chat2db.server.domain.api.service.DatabaseService; import ai.chat2db.server.domain.api.service.DlTemplateService; import ai.chat2db.server.domain.api.service.TableService; @@ -258,18 +257,5 @@ public ActionResult delete(@Valid @RequestBody TableDeleteRequest request) { DropParam dropParam = rdbWebConverter.tableDelete2dropParam(request); return tableService.drop(dropParam); } - @PostMapping("/export_data") - public void exportData(@Valid @RequestBody TableExportDataRequest request, HttpServletResponse response) { - Class targetClass = ExportDBDataStrategyFactory.get(request.getExportType()); - response.setCharacterEncoding("utf-8"); - TableExportDataParam param = rdbWebConverter.request2param(request); - try { - Constructor constructor = targetClass.getDeclaredConstructor(); - ExportDBDataStrategy service = (ExportDBDataStrategy) constructor.newInstance(); - service.doExport(param, response); - } catch (Exception e) { - throw new RuntimeException(e); - } - } } diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/converter/RdbWebConverter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/converter/RdbWebConverter.java index eea80ea4b..bfa069a4d 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/converter/RdbWebConverter.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/converter/RdbWebConverter.java @@ -3,7 +3,6 @@ import java.util.List; import ai.chat2db.server.domain.api.param.*; -import ai.chat2db.server.domain.api.param.user.TableExportDataParam; import ai.chat2db.server.web.api.controller.data.source.vo.DatabaseVO; import ai.chat2db.server.web.api.controller.rdb.request.*; import ai.chat2db.server.web.api.controller.rdb.vo.ColumnVO; @@ -258,6 +257,4 @@ public abstract class RdbWebConverter { public abstract DmlSqlCopyParam dmlRequest2param(DmlSqlCopyRequest request) ; - public abstract TableExportDataParam request2param(TableExportDataRequest request); - } diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2CsvStrategy.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2CsvStrategy.java index c626512dd..08fcdcda1 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2CsvStrategy.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2CsvStrategy.java @@ -1,25 +1,20 @@ package ai.chat2db.server.web.api.controller.rdb.data.export.strategy; import ai.chat2db.server.domain.api.enums.ExportFileSuffix; -import ai.chat2db.spi.util.ResultSetUtils; -import com.alibaba.excel.EasyExcel; +import ai.chat2db.server.tools.common.model.export.data.option.ExportDataOption; import com.alibaba.excel.support.ExcelTypeEnum; import jakarta.servlet.http.HttpServletResponse; -import org.jetbrains.annotations.NotNull; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.OutputStream; +import java.nio.charset.StandardCharsets; import java.sql.Connection; import java.sql.ResultSet; -import java.sql.ResultSetMetaData; import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Collections; import java.util.List; -import java.util.stream.Collectors; -public class ExportDBData2CsvStrategy extends ExportDBDataStrategy { +public class ExportDBData2CsvStrategy extends ExportDBDataStrategy { public ExportDBData2CsvStrategy() { suffix = ExportFileSuffix.CSV.getSuffix(); @@ -27,52 +22,30 @@ public ExportDBData2CsvStrategy() { } @Override - protected ByteArrayOutputStream exportData(Connection connection, String databaseName, - String schemaName, String tableName) throws SQLException { + protected ByteArrayOutputStream doTableDataExport(Connection connection, String databaseName, + String schemaName, String tableName, + List filedNames, ExportDataOption options) throws SQLException { ByteArrayOutputStream byteOut = new ByteArrayOutputStream(); - export2CSV(connection, schemaName, tableName, byteOut); + export2CSV(connection, schemaName, tableName, byteOut, filedNames, options); return byteOut; } @Override - protected void exportData(HttpServletResponse response, Connection connection, - String databaseName, String schemaName, String tableName) throws SQLException { + protected void doTableDataExport(HttpServletResponse response, Connection connection, String databaseName, + String schemaName, String tableName, + List filedNames, ExportDataOption options) throws SQLException { try { - export2CSV(connection, schemaName, tableName, response.getOutputStream()); + export2CSV(connection, schemaName, tableName, response.getOutputStream(), filedNames, options); } catch (IOException e) { throw new RuntimeException(e); } } - private void export2CSV(Connection connection, String schemaName, String tableName, OutputStream byteOut) throws SQLException { + + private void export2CSV(Connection connection, String schemaName, String tableName, OutputStream out, List fileNames, ExportDataOption options) throws SQLException { try (ResultSet resultSet = connection.createStatement().executeQuery(buildQuerySql(schemaName, tableName))) { - EasyExcel.write(byteOut) - .excelType(ExcelTypeEnum.CSV) - .sheet(tableName) - .head(getHeadList(resultSet)) - .doWrite(getDataList(resultSet)); + write(tableName, out, fileNames, options, resultSet, StandardCharsets.UTF_8, ExcelTypeEnum.CSV); } } - @NotNull - private List> getDataList(ResultSet resultSet) throws SQLException { - ResultSetMetaData metaData = resultSet.getMetaData(); - int columnCount = metaData.getColumnCount(); - List> dataList = new ArrayList<>(); - while (resultSet.next()) { - List row = new ArrayList<>(); - for (int i = 1; i <= columnCount; i++) { - row.add(resultSet.getString(i)); - } - dataList.add(row); - } - return dataList; - } - @NotNull - private List> getHeadList(ResultSet resultSet) { - return ResultSetUtils.getRsHeader(resultSet) - .stream() - .map(Collections::singletonList) - .collect(Collectors.toList()); - } } \ No newline at end of file diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2ExcelStrategy.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2ExcelStrategy.java index c5c0780c1..e9215e339 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2ExcelStrategy.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2ExcelStrategy.java @@ -1,22 +1,18 @@ package ai.chat2db.server.web.api.controller.rdb.data.export.strategy; import ai.chat2db.server.domain.api.enums.ExportFileSuffix; -import ai.chat2db.spi.util.ResultSetUtils; -import com.alibaba.excel.EasyExcel; +import ai.chat2db.server.tools.common.model.export.data.option.ExportDataOption; +import com.alibaba.excel.support.ExcelTypeEnum; import jakarta.servlet.http.HttpServletResponse; -import org.jetbrains.annotations.NotNull; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.OutputStream; +import java.nio.charset.StandardCharsets; import java.sql.Connection; import java.sql.ResultSet; -import java.sql.ResultSetMetaData; import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Collections; import java.util.List; -import java.util.stream.Collectors; public class ExportDBData2ExcelStrategy extends ExportDBDataStrategy { @@ -26,51 +22,30 @@ public ExportDBData2ExcelStrategy() { } @Override - protected ByteArrayOutputStream exportData(Connection connection, String databaseName, - String schemaName, String tableName) throws SQLException { + protected ByteArrayOutputStream doTableDataExport(Connection connection, String databaseName, + String schemaName, String tableName, + List filedNames, ExportDataOption options) throws SQLException { ByteArrayOutputStream byteOut = new ByteArrayOutputStream(); - export2EXCEL(connection, schemaName, tableName, byteOut); + export2EXCEL(connection, schemaName, tableName, byteOut, filedNames, options); return byteOut; } @Override - protected void exportData(HttpServletResponse response, Connection connection, String databaseName, - String schemaName, String tableName) throws SQLException { - try { - export2EXCEL(connection, schemaName, tableName, response.getOutputStream()); + protected void doTableDataExport(HttpServletResponse response, Connection connection, String databaseName, + String schemaName, String tableName, + List filedNames, ExportDataOption options) throws SQLException { + try { + export2EXCEL(connection, schemaName, tableName, response.getOutputStream(), filedNames, options); } catch (IOException e) { throw new RuntimeException(e); } } - private void export2EXCEL(Connection connection, String schemaName, String tableName, OutputStream byteOut) throws SQLException { + private void export2EXCEL(Connection connection, String schemaName, String tableName, OutputStream out, List fileNames, ExportDataOption options) throws SQLException { try (ResultSet resultSet = connection.createStatement().executeQuery(buildQuerySql(schemaName, tableName))) { - ResultSetMetaData metaData = resultSet.getMetaData(); - int columnCount = metaData.getColumnCount(); - List> headList = getHeadList(resultSet); - List> dataList = getDataList(resultSet, columnCount); - EasyExcel.write(byteOut).sheet(tableName).head(headList).doWrite(dataList); + write(tableName, out, fileNames, options, resultSet, StandardCharsets.UTF_8, ExcelTypeEnum.XLSX); } } - @NotNull - private List> getDataList(ResultSet resultSet, int columnCount) throws SQLException { - List> dataList = new ArrayList<>(); - while (resultSet.next()) { - List row = new ArrayList<>(); - for (int i = 1; i <= columnCount; i++) { - row.add(resultSet.getString(i)); - } - dataList.add(row); - } - return dataList; - } - @NotNull - private List> getHeadList(ResultSet resultSet) { - return ResultSetUtils.getRsHeader(resultSet) - .stream() - .map(Collections::singletonList) - .collect(Collectors.toList()); - } } \ No newline at end of file diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2JsonStrategy.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2JsonStrategy.java index 325657043..46e115120 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2JsonStrategy.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2JsonStrategy.java @@ -2,7 +2,10 @@ import ai.chat2db.server.domain.api.enums.ExportFileSuffix; import ai.chat2db.server.tools.base.excption.BusinessException; +import ai.chat2db.server.tools.common.model.export.data.option.ExportDataOption; +import ai.chat2db.server.tools.common.model.export.data.option.JSONExportDataOption; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; import jakarta.servlet.http.HttpServletResponse; import org.jetbrains.annotations.NotNull; @@ -15,6 +18,7 @@ import java.sql.ResultSet; import java.sql.ResultSetMetaData; import java.sql.SQLException; +import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; @@ -28,28 +32,39 @@ public ExportDBData2JsonStrategy() { } @Override - protected ByteArrayOutputStream exportData(Connection connection, String databaseName, String schemaName, String tableName) throws SQLException { + protected ByteArrayOutputStream doTableDataExport(Connection connection, String databaseName, + String schemaName, String tableName, + List filedNames, ExportDataOption options) throws SQLException { ByteArrayOutputStream byteOut = new ByteArrayOutputStream(); try (ResultSet resultSet = connection.createStatement().executeQuery(buildQuerySql(schemaName, tableName)); PrintWriter writer = new PrintWriter(new OutputStreamWriter(byteOut, StandardCharsets.UTF_8))) { - export2JSON(resultSet, writer); + export2JSON(resultSet, writer, filedNames, options); } return byteOut; } @Override - protected void exportData(HttpServletResponse response, Connection connection, String databaseName, - String schemaName, String tableName) throws SQLException { + protected void doTableDataExport(HttpServletResponse response, Connection connection, String databaseName, + String schemaName, String tableName, + List filedNames, ExportDataOption options) throws SQLException { try (ResultSet resultSet = connection.createStatement().executeQuery(buildQuerySql(schemaName, tableName))) { - export2JSON(resultSet, response.getWriter()); - }catch (IOException e){ + export2JSON(resultSet, response.getWriter(), filedNames, options); + } catch (IOException e) { throw new RuntimeException(e); } } - private void export2JSON(ResultSet resultSet, PrintWriter writer) throws SQLException { - List> data = getDataMap(resultSet); + + private void export2JSON(ResultSet resultSet, PrintWriter writer, List filedNames, ExportDataOption options) throws SQLException { + List> data = getDataMap(resultSet, filedNames); ObjectMapper objectMapper = new ObjectMapper(); + JSONExportDataOption jsonExportDataOption = (JSONExportDataOption) options; + if (!jsonExportDataOption.getIsTimestamps()) { + objectMapper.configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, true); + SimpleDateFormat dateFormat = new SimpleDateFormat(jsonExportDataOption.getDataTimeFormat()); + objectMapper.setDateFormat(dateFormat); + objectMapper.configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, false); + } try { String jsonString = objectMapper.writeValueAsString(data); writer.println(jsonString); @@ -59,13 +74,16 @@ private void export2JSON(ResultSet resultSet, PrintWriter writer) throws SQLExce } @NotNull - private List> getDataMap(ResultSet resultSet) throws SQLException { + private List> getDataMap(ResultSet resultSet, List filedNames) throws SQLException { ResultSetMetaData metaData = resultSet.getMetaData(); int columnCount = metaData.getColumnCount(); List> data = new ArrayList<>(); while (resultSet.next()) { Map row = new LinkedHashMap<>(); for (int i = 1; i <= columnCount; i++) { + if (filedNames != null && !filedNames.contains(metaData.getColumnName(i))) { + continue; + } row.put(metaData.getColumnName(i), resultSet.getObject(i)); } data.add(row); diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2SqlStrategy.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2SqlStrategy.java index dcd81e72a..4e8a61c41 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2SqlStrategy.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2SqlStrategy.java @@ -1,8 +1,11 @@ package ai.chat2db.server.web.api.controller.rdb.data.export.strategy; import ai.chat2db.server.domain.api.enums.ExportFileSuffix; -import ai.chat2db.spi.sql.Chat2DBContext; +import ai.chat2db.server.tools.common.model.export.data.option.ExportDataOption; +import ai.chat2db.server.tools.common.model.export.data.option.SQLExportDataOption; +import ai.chat2db.spi.util.ResultSetUtils; import jakarta.servlet.http.HttpServletResponse; +import org.apache.commons.lang3.StringUtils; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -10,7 +13,12 @@ import java.io.PrintWriter; import java.nio.charset.StandardCharsets; import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; import java.sql.SQLException; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; /** * @author: zgq @@ -24,25 +32,175 @@ public ExportDBData2SqlStrategy() { } @Override - protected ByteArrayOutputStream exportData(Connection connection, String databaseName, String schemaName, String tableName) throws SQLException { + protected ByteArrayOutputStream doTableDataExport(Connection connection, String databaseName, + String schemaName, String tableName, List filedNames, ExportDataOption options) throws SQLException { ByteArrayOutputStream byteOut = new ByteArrayOutputStream(); try (PrintWriter writer = new PrintWriter(new OutputStreamWriter(byteOut, StandardCharsets.UTF_8))) { - writer.println(export2SQL(connection, databaseName, schemaName, tableName)); + writer.println(export2InsertSQL(connection, schemaName, tableName, filedNames, options)); } return byteOut; } + @Override - protected void exportData(HttpServletResponse response, Connection connection, - String databaseName, String schemaName, String tableName) throws SQLException { + protected void doTableDataExport(HttpServletResponse response, Connection connection, String databaseName, + String schemaName, String tableName, List filedNames, + ExportDataOption options) throws SQLException { try { - response.getWriter().print(export2SQL(connection, databaseName, schemaName, tableName)); + response.getWriter().println(export2InsertSQL(connection, schemaName, tableName, filedNames, options)); } catch (IOException e) { throw new RuntimeException(e); } } - private String export2SQL(Connection connection, String databaseName, String schemaName, String tableName) throws SQLException { - return Chat2DBContext.getDBManage().exportDatabaseData(connection, databaseName, schemaName, tableName); + private String export2InsertSQL(Connection connection, String schemaName, + String tableName, List filedNames, ExportDataOption options) throws SQLException { + + StringBuilder sqlBuilder = new StringBuilder(); + try (ResultSet resultSet = connection.createStatement().executeQuery(buildQuerySql(schemaName, tableName))) { + ResultSetMetaData metaData = resultSet.getMetaData(); + List headList = ResultSetUtils.getRsHeader(resultSet); + if (((SQLExportDataOption) options).getToUpdate()) { + buildUpdateSQL(tableName, filedNames, options, sqlBuilder, resultSet, metaData, headList); + } else { + buildInsertSQL(tableName, filedNames, options, sqlBuilder, resultSet, metaData, headList); + } + sqlBuilder.append("\n"); + } + return sqlBuilder.toString(); } + + private void buildInsertSQL(String tableName, List filedNames, ExportDataOption options, + StringBuilder sqlBuilder, ResultSet resultSet, ResultSetMetaData metaData, + List headList) throws SQLException { + boolean containsHeader = options.getContainsHeader(); + Boolean multipleRow = ((SQLExportDataOption) options).getMultipleRow(); + String newTableName = ((SQLExportDataOption) options).getNewTableName(); + if (multipleRow) { + if (headList.size() != filedNames.size() && StringUtils.isNotBlank(newTableName)) { + buildInsert(newTableName, containsHeader, filedNames, sqlBuilder); + } else if (StringUtils.isNotBlank(newTableName)) { + buildInsert(newTableName, containsHeader, headList, sqlBuilder); + } else if (headList.size() != filedNames.size()) { + buildInsert(tableName, containsHeader, filedNames, sqlBuilder); + } else { + buildInsert(tableName, containsHeader, headList, sqlBuilder); + } + } + + + while (resultSet.next()) { + int filedSeparatorCount = 0; + if (!multipleRow) { + if (headList.size() != filedNames.size() && StringUtils.isNotBlank(newTableName)) { + buildInsert(newTableName, containsHeader, filedNames, sqlBuilder); + } else if (StringUtils.isNotBlank(newTableName)) { + buildInsert(newTableName, containsHeader, headList, sqlBuilder); + } else if (headList.size() != filedNames.size()) { + buildInsert(tableName, containsHeader, filedNames, sqlBuilder); + } else { + buildInsert(tableName, containsHeader, headList, sqlBuilder); + } + } + + for (int i = 1; i <= metaData.getColumnCount(); i++) { + + if (headList.size() != filedNames.size() && !filedNames.contains(metaData.getColumnName(i))) { + continue; + } + String value = resultSet.getString(i); + if (i == 1) { + sqlBuilder.append("("); + } + if (Objects.isNull(value)) { + sqlBuilder.append("NULL"); + } else { + sqlBuilder.append("'").append(value).append("'"); + } + filedSeparatorCount++; + if (headList.size() == filedNames.size()) { + if (i < metaData.getColumnCount()) { + sqlBuilder.append(","); + } + } else { + if (filedSeparatorCount < filedNames.size()) { + sqlBuilder.append(","); + } + } + } + if (multipleRow) { + if (resultSet.isLast()) { + sqlBuilder.append(");\n"); + break; + } else { + sqlBuilder.append("),\n"); + } + } else { + sqlBuilder.append(");\n"); + } + } + } + + private void buildUpdateSQL(String tableName, List fieldNames, ExportDataOption options, + StringBuilder sqlBuilder, ResultSet resultSet, ResultSetMetaData metaData, + List headList) throws SQLException { + + String newTableName = ((SQLExportDataOption) options).getNewTableName(); + while (resultSet.next()) { + + if (StringUtils.isNotBlank(newTableName)) { + buildUpdate(newTableName, sqlBuilder); + } else { + buildUpdate(tableName, sqlBuilder); + } + int fieldSeparatorCount = 0; + for (int i = 1; i <= metaData.getColumnCount(); i++) { + String columnName = metaData.getColumnName(i); + + if (headList.size() != fieldNames.size() && !fieldNames.contains(columnName)) { + continue; + } + String value = resultSet.getString(i); + if (Objects.isNull(value)) { + sqlBuilder.append(columnName).append(" = NULL"); + } else { + sqlBuilder.append(columnName).append(" = '").append(value).append("'"); + } + fieldSeparatorCount++; + if (headList.size() == fieldNames.size()) { + if (i < metaData.getColumnCount()) { + sqlBuilder.append(","); + } + } else { + if (fieldSeparatorCount < fieldNames.size()) { + sqlBuilder.append(","); + } + } + } + sqlBuilder.append(" WHERE id = ").append("'").append(resultSet.getString("id")).append("'").append(";\n"); + } + } + + + private void buildUpdate(String newTableName, StringBuilder sqlBuilder) { + sqlBuilder.append("UPDATE ").append(newTableName).append(" SET "); + } + + + private void buildInsert(String tableName, boolean containsHeader, List filedNames, StringBuilder sqlBuilder) { + sqlBuilder.append("INSERT INTO ").append(tableName); + if (containsHeader) { + sqlBuilder.append(" ").append(convertList2String(filedNames)); + } + sqlBuilder.append(" VALUES "); + } + + + private String convertList2String(List list) { + return list.stream() + .map(s -> "'" + s + "'") + .collect(Collectors.joining(",", "(", ")")); + } + + } \ No newline at end of file diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBDataStrategy.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBDataStrategy.java index d1b75c518..1f97bab9c 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBDataStrategy.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBDataStrategy.java @@ -2,21 +2,35 @@ import ai.chat2db.server.domain.api.enums.ExportFileSuffix; import ai.chat2db.server.domain.api.param.datasource.DatabaseExportDataParam; -import ai.chat2db.server.domain.api.param.user.TableExportDataParam; +import ai.chat2db.server.tools.common.model.export.data.option.ExportDataOption; +import ai.chat2db.server.tools.common.model.export.data.option.TableOption; +import ai.chat2db.server.web.api.controller.rdb.RdbDmlExportController; import ai.chat2db.spi.sql.Chat2DBContext; +import ai.chat2db.spi.util.ResultSetUtils; +import com.alibaba.excel.EasyExcel; +import com.alibaba.excel.support.ExcelTypeEnum; +import com.alibaba.excel.write.builder.ExcelWriterBuilder; +import com.alibaba.excel.write.builder.ExcelWriterSheetBuilder; import jakarta.servlet.ServletOutputStream; import jakarta.servlet.http.HttpServletResponse; import lombok.AllArgsConstructor; import lombok.Getter; import lombok.NoArgsConstructor; -import org.springframework.http.MediaType; +import org.jetbrains.annotations.NotNull; import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.io.OutputStream; +import java.nio.charset.Charset; import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Objects; +import java.util.stream.Collectors; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; @@ -30,7 +44,7 @@ public abstract class ExportDBDataStrategy { public String suffix; - public String contentType = "application/zip"; + public String contentType; public void doExport(DatabaseExportDataParam param, HttpServletResponse response) { @@ -38,40 +52,39 @@ public void doExport(DatabaseExportDataParam param, HttpServletResponse response String schemaName = param.getSchemaName(); setResponseHeaders(param, response); try (Connection connection = Chat2DBContext.getConnection()) { - if (param instanceof TableExportDataParam){ - doTableDataExport(((TableExportDataParam) param).getTableName(),response,databaseName,schemaName,connection); - }else { - doDbDataExport(response, databaseName, schemaName, connection); + if (param.getTableOptions().size() == 1) { + String tableName = param.getTableOptions().get(0).getTableName(); + List filedNames = param.getTableOptions().get(0).getFiledNames(); + doTableDataExport(response, connection, databaseName, schemaName, tableName, filedNames, param.getExportDataOption()); + } else { + doDbDataExport(response, connection, databaseName, schemaName, param.getTableOptions(), param.getExportDataOption()); } } catch (Exception e) { throw new RuntimeException(e); } } - private void doDbDataExport(HttpServletResponse response, String databaseName, - String schemaName, Connection connection) throws IOException, SQLException { + private void doDbDataExport(HttpServletResponse response, Connection connection, String databaseName, + String schemaName, List tableOptions, + ExportDataOption options) throws IOException, SQLException { ServletOutputStream outputStream = response.getOutputStream(); ZipOutputStream zipOut = new ZipOutputStream(outputStream); - List tableNames = Chat2DBContext.getMetaData().tableNames(connection, databaseName, schemaName, null); - tableNames.addAll(Chat2DBContext.getMetaData().viewNames(connection, databaseName, schemaName)); - for (String tableName : tableNames) { + for (TableOption tableOption : tableOptions) { + String tableName = tableOption.getTableName(); String fileName = tableName + getSuffix(); + List filedNames = tableOption.getFiledNames(); zipOut.putNextEntry(new ZipEntry(fileName)); - ByteArrayOutputStream byteOut = exportData(connection, databaseName, schemaName, tableName); + ByteArrayOutputStream byteOut = doTableDataExport(connection, databaseName, schemaName, tableName, filedNames, options); byteOut.writeTo(zipOut); zipOut.closeEntry(); byteOut.close(); } } - private void doTableDataExport(String tableName, HttpServletResponse response, String databaseName, - String schemaName, Connection connection) throws SQLException { - exportData(response, connection, databaseName, schemaName, tableName); - } private void setResponseHeaders(DatabaseExportDataParam param, HttpServletResponse response) { - if (param instanceof TableExportDataParam) { - String tableName = ((TableExportDataParam) param).getTableName(); + if (param.getTableOptions().size() == 1) { + String tableName = param.getTableOptions().get(0).getTableName(); response.setContentType(contentType); response.setHeader("Content-disposition", "attachment;filename*=utf-8''" + tableName + suffix); } else { @@ -95,8 +108,56 @@ public String buildQuerySql(String schemaName, String tableName) { return sql; } - protected abstract ByteArrayOutputStream exportData(Connection connection, String databaseName, String schemaName, String tableName) throws SQLException; + protected abstract ByteArrayOutputStream doTableDataExport(Connection connection, String databaseName, + String schemaName, String tableName, + List filedNames, ExportDataOption options) throws SQLException; + + protected abstract void doTableDataExport(HttpServletResponse response, Connection connection, String databaseName, + String schemaName, String tableName, + List filedNames, ExportDataOption options) throws SQLException; + + public void write(String tableName, OutputStream out, List fileNames, ExportDataOption options, + ResultSet resultSet, Charset charset, ExcelTypeEnum type) throws SQLException { + Boolean containsHeader = options.getContainsHeader(); + ExcelWriterSheetBuilder excelWriterSheetBuilder = EasyExcel.write(out) + .charset(charset) + .excelType(type).sheet(tableName); + if (containsHeader) { + List header = ResultSetUtils.getRsHeader(resultSet); + if (fileNames.size() != header.size()) { + excelWriterSheetBuilder.head(getListHeadList(fileNames)); + } else { + excelWriterSheetBuilder.head(getListHeadList(header)); + } + } + excelWriterSheetBuilder.doWrite(getDataList(resultSet, fileNames)); + } + + + @NotNull + public List> getDataList(ResultSet resultSet, List fileNames) throws SQLException { + ResultSetMetaData metaData = resultSet.getMetaData(); + int columnCount = metaData.getColumnCount(); + List> dataList = new ArrayList<>(); + while (resultSet.next()) { + List row = new ArrayList<>(); + for (int i = 1; i <= columnCount; i++) { + if (fileNames.size() != columnCount && !fileNames.contains(metaData.getColumnName(i))) { + continue; + } + row.add(resultSet.getString(i)); + } + dataList.add(row); + } + return dataList; + } - protected abstract void exportData(HttpServletResponse response, Connection connection, String databaseName, String schemaName, String tableName) throws SQLException; + @NotNull + public List> getListHeadList(List headers) { + return headers + .stream() + .map(Collections::singletonList) + .collect(Collectors.toList()); + } } \ No newline at end of file diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/request/DatabaseExportDataRequest.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/request/DatabaseExportDataRequest.java index 0faaac022..9b9e518f3 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/request/DatabaseExportDataRequest.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/request/DatabaseExportDataRequest.java @@ -1,11 +1,15 @@ package ai.chat2db.server.web.api.controller.rdb.request; +import ai.chat2db.server.tools.common.model.export.data.option.ExportDataOption; +import ai.chat2db.server.tools.common.model.export.data.option.TableOption; import ai.chat2db.server.web.api.controller.data.source.request.DataSourceBaseRequest; -import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotNull; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; +import java.util.List; + /** * @author: zgq * @date: 2024年03月24日 12:36 @@ -14,6 +18,8 @@ @AllArgsConstructor @NoArgsConstructor public class DatabaseExportDataRequest extends DataSourceBaseRequest { - @NotBlank - private String exportType; + @NotNull + private ExportDataOption exportDataOption; + private List tableOptions; + } \ No newline at end of file diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/request/TableExportDataRequest.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/request/TableExportDataRequest.java deleted file mode 100644 index b5f7f4655..000000000 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/request/TableExportDataRequest.java +++ /dev/null @@ -1,17 +0,0 @@ -package ai.chat2db.server.web.api.controller.rdb.request; - -import jakarta.validation.constraints.NotBlank; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; - -/** - * @author: zgq - */ -@Data -@AllArgsConstructor -@NoArgsConstructor -public class TableExportDataRequest extends DatabaseExportDataRequest { - @NotBlank - private String tableName; -} From 999c2b338e61945f87891ea5d3374234e5197149 Mon Sep 17 00:00:00 2001 From: zgq <203083679@qq.com> Date: Tue, 23 Apr 2024 14:03:43 +0800 Subject: [PATCH 03/11] Basic work on importing data --- .../datasource/DatabaseExportDataParam.java | 4 +-- .../datasource/DatabaseImportDataParam.java | 21 +++++++++++++ .../data/option/ExportDataOption.java | 2 +- .../model/data/option/ImportDataOption.java | 30 +++++++++++++++++++ .../data/option/JSONExportDataOption.java | 2 +- .../data/option/SQLExportDataOption.java | 2 +- .../{export => }/data/option/TableOption.java | 2 +- .../controller/rdb/DatabaseController.java | 24 ++++++++++----- .../rdb/converter/DatabaseConverter.java | 6 ++++ .../strategy/ExportDBData2CsvStrategy.java | 2 +- .../strategy/ExportDBData2ExcelStrategy.java | 2 +- .../strategy/ExportDBData2JsonStrategy.java | 4 +-- .../strategy/ExportDBData2SqlStrategy.java | 4 +-- .../export/strategy/ExportDBDataStrategy.java | 10 ++++--- .../request/DatabaseExportDataRequest.java | 4 +-- .../request/DatabaseImportDataRequest.java | 20 +++++++++++++ 16 files changed, 113 insertions(+), 26 deletions(-) create mode 100644 chat2db-server/chat2db-server-domain/chat2db-server-domain-api/src/main/java/ai/chat2db/server/domain/api/param/datasource/DatabaseImportDataParam.java rename chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/{export => }/data/option/ExportDataOption.java (93%) create mode 100644 chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/ImportDataOption.java rename chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/{export => }/data/option/JSONExportDataOption.java (87%) rename chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/{export => }/data/option/SQLExportDataOption.java (82%) rename chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/{export => }/data/option/TableOption.java (85%) create mode 100644 chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/request/DatabaseImportDataRequest.java diff --git a/chat2db-server/chat2db-server-domain/chat2db-server-domain-api/src/main/java/ai/chat2db/server/domain/api/param/datasource/DatabaseExportDataParam.java b/chat2db-server/chat2db-server-domain/chat2db-server-domain-api/src/main/java/ai/chat2db/server/domain/api/param/datasource/DatabaseExportDataParam.java index 242603408..00ff0973b 100644 --- a/chat2db-server/chat2db-server-domain/chat2db-server-domain-api/src/main/java/ai/chat2db/server/domain/api/param/datasource/DatabaseExportDataParam.java +++ b/chat2db-server/chat2db-server-domain/chat2db-server-domain-api/src/main/java/ai/chat2db/server/domain/api/param/datasource/DatabaseExportDataParam.java @@ -1,7 +1,7 @@ package ai.chat2db.server.domain.api.param.datasource; -import ai.chat2db.server.tools.common.model.export.data.option.ExportDataOption; -import ai.chat2db.server.tools.common.model.export.data.option.TableOption; +import ai.chat2db.server.tools.common.model.data.option.ExportDataOption; +import ai.chat2db.server.tools.common.model.data.option.TableOption; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; diff --git a/chat2db-server/chat2db-server-domain/chat2db-server-domain-api/src/main/java/ai/chat2db/server/domain/api/param/datasource/DatabaseImportDataParam.java b/chat2db-server/chat2db-server-domain/chat2db-server-domain-api/src/main/java/ai/chat2db/server/domain/api/param/datasource/DatabaseImportDataParam.java new file mode 100644 index 000000000..c2bf5d082 --- /dev/null +++ b/chat2db-server/chat2db-server-domain/chat2db-server-domain-api/src/main/java/ai/chat2db/server/domain/api/param/datasource/DatabaseImportDataParam.java @@ -0,0 +1,21 @@ +package ai.chat2db.server.domain.api.param.datasource; + +import ai.chat2db.server.tools.common.model.data.option.ImportDataOption; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * 功能描述 + * + * @author: zgq + * @date: 2024年04月23日 13:52 + */ +@Data +@AllArgsConstructor +@NoArgsConstructor +public class DatabaseImportDataParam { + private String databaseName; + private String schemaName; + private ImportDataOption importDataOption; +} diff --git a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/export/data/option/ExportDataOption.java b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/ExportDataOption.java similarity index 93% rename from chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/export/data/option/ExportDataOption.java rename to chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/ExportDataOption.java index 027aad38f..b1600e7c0 100644 --- a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/export/data/option/ExportDataOption.java +++ b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/ExportDataOption.java @@ -1,4 +1,4 @@ -package ai.chat2db.server.tools.common.model.export.data.option; +package ai.chat2db.server.tools.common.model.data.option; import com.fasterxml.jackson.annotation.JsonSubTypes; diff --git a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/ImportDataOption.java b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/ImportDataOption.java new file mode 100644 index 000000000..ba1e3e6a4 --- /dev/null +++ b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/ImportDataOption.java @@ -0,0 +1,30 @@ +package ai.chat2db.server.tools.common.model.data.option; + +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import jakarta.validation.constraints.NotBlank; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * @author: zgq + * @date: 2024年04月23日 13:58 + */ +@Data +@AllArgsConstructor +@NoArgsConstructor +@JsonTypeInfo( + use = JsonTypeInfo.Id.NAME, + property = "exportType", + visible = true) +@JsonSubTypes({ + @JsonSubTypes.Type(value = ImportDataOption.class, name = "CSV"), + @JsonSubTypes.Type(value = ImportDataOption.class, name = "SQL"), + @JsonSubTypes.Type(value = ImportDataOption.class, name = "EXCEL"), + @JsonSubTypes.Type(value = ImportDataOption.class, name = "JSON") +}) +public class ImportDataOption { + @NotBlank + private String importType; +} diff --git a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/export/data/option/JSONExportDataOption.java b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/JSONExportDataOption.java similarity index 87% rename from chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/export/data/option/JSONExportDataOption.java rename to chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/JSONExportDataOption.java index bd52278ea..4b2f5ef77 100644 --- a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/export/data/option/JSONExportDataOption.java +++ b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/JSONExportDataOption.java @@ -1,4 +1,4 @@ -package ai.chat2db.server.tools.common.model.export.data.option; +package ai.chat2db.server.tools.common.model.data.option; import jakarta.validation.constraints.NotBlank; import jakarta.validation.constraints.NotNull; diff --git a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/export/data/option/SQLExportDataOption.java b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/SQLExportDataOption.java similarity index 82% rename from chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/export/data/option/SQLExportDataOption.java rename to chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/SQLExportDataOption.java index 18e475ff0..73845b799 100644 --- a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/export/data/option/SQLExportDataOption.java +++ b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/SQLExportDataOption.java @@ -1,4 +1,4 @@ -package ai.chat2db.server.tools.common.model.export.data.option; +package ai.chat2db.server.tools.common.model.data.option; import lombok.AllArgsConstructor; import lombok.Data; diff --git a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/export/data/option/TableOption.java b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/TableOption.java similarity index 85% rename from chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/export/data/option/TableOption.java rename to chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/TableOption.java index 96ca59f5c..6fdc68cf0 100644 --- a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/export/data/option/TableOption.java +++ b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/TableOption.java @@ -1,4 +1,4 @@ -package ai.chat2db.server.tools.common.model.export.data.option; +package ai.chat2db.server.tools.common.model.data.option; import jakarta.validation.constraints.NotEmpty; import jakarta.validation.constraints.NotNull; diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/DatabaseController.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/DatabaseController.java index 77b63016a..f3308aaa4 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/DatabaseController.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/DatabaseController.java @@ -1,10 +1,7 @@ package ai.chat2db.server.web.api.controller.rdb; import ai.chat2db.server.domain.api.param.MetaDataQueryParam; -import ai.chat2db.server.domain.api.param.datasource.DatabaseCreateParam; -import ai.chat2db.server.domain.api.param.datasource.DatabaseExportDataParam; -import ai.chat2db.server.domain.api.param.datasource.DatabaseExportParam; -import ai.chat2db.server.domain.api.param.datasource.DatabaseQueryAllParam; +import ai.chat2db.server.domain.api.param.datasource.*; import ai.chat2db.server.domain.api.service.DatabaseService; import ai.chat2db.server.tools.base.wrapper.result.ActionResult; import ai.chat2db.server.tools.base.wrapper.result.DataResult; @@ -16,10 +13,7 @@ import ai.chat2db.server.web.api.controller.rdb.converter.RdbWebConverter; import ai.chat2db.server.web.api.controller.rdb.data.export.strategy.ExportDBDataStrategy; import ai.chat2db.server.web.api.controller.rdb.factory.ExportDBDataStrategyFactory; -import ai.chat2db.server.web.api.controller.rdb.request.DatabaseCreateRequest; -import ai.chat2db.server.web.api.controller.rdb.request.DatabaseExportDataRequest; -import ai.chat2db.server.web.api.controller.rdb.request.DatabaseExportRequest; -import ai.chat2db.server.web.api.controller.rdb.request.UpdateDatabaseRequest; +import ai.chat2db.server.web.api.controller.rdb.request.*; import ai.chat2db.server.web.api.controller.rdb.vo.MetaSchemaVO; import ai.chat2db.spi.model.Database; import ai.chat2db.spi.model.MetaSchema; @@ -144,4 +138,18 @@ public void exportData(@Valid @RequestBody DatabaseExportDataRequest request, Ht } } + + @PostMapping("/import_data") + public void importData(@Valid @RequestBody DatabaseImportDataRequest request) { + Class targetClass = ExportDBDataStrategyFactory.get(request.getImportDataOption().getImportType()); + DatabaseImportDataParam param = databaseConverter.request2param(request); + try { + Constructor constructor = targetClass.getDeclaredConstructor(); + ExportDBDataStrategy service = (ExportDBDataStrategy) constructor.newInstance(); + service.doImport(param); + } catch (Exception e) { + throw new RuntimeException(e); + } + + } } diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/converter/DatabaseConverter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/converter/DatabaseConverter.java index bb9bc4a9a..9f576005b 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/converter/DatabaseConverter.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/converter/DatabaseConverter.java @@ -2,9 +2,11 @@ import ai.chat2db.server.domain.api.param.datasource.DatabaseExportDataParam; import ai.chat2db.server.domain.api.param.datasource.DatabaseExportParam; +import ai.chat2db.server.domain.api.param.datasource.DatabaseImportDataParam; import ai.chat2db.server.web.api.controller.rdb.request.DatabaseCreateRequest; import ai.chat2db.server.web.api.controller.rdb.request.DatabaseExportDataRequest; import ai.chat2db.server.web.api.controller.rdb.request.DatabaseExportRequest; +import ai.chat2db.server.web.api.controller.rdb.request.DatabaseImportDataRequest; import ai.chat2db.spi.model.Database; import org.mapstruct.Mapper; @@ -16,4 +18,8 @@ public abstract class DatabaseConverter { public abstract DatabaseExportParam request2param(DatabaseExportRequest request); public abstract DatabaseExportDataParam request2param(DatabaseExportDataRequest request); + + public abstract DatabaseImportDataParam request2param(DatabaseImportDataRequest request); + + } diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2CsvStrategy.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2CsvStrategy.java index 08fcdcda1..240086a0d 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2CsvStrategy.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2CsvStrategy.java @@ -1,7 +1,7 @@ package ai.chat2db.server.web.api.controller.rdb.data.export.strategy; import ai.chat2db.server.domain.api.enums.ExportFileSuffix; -import ai.chat2db.server.tools.common.model.export.data.option.ExportDataOption; +import ai.chat2db.server.tools.common.model.data.option.ExportDataOption; import com.alibaba.excel.support.ExcelTypeEnum; import jakarta.servlet.http.HttpServletResponse; diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2ExcelStrategy.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2ExcelStrategy.java index e9215e339..c87855743 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2ExcelStrategy.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2ExcelStrategy.java @@ -1,7 +1,7 @@ package ai.chat2db.server.web.api.controller.rdb.data.export.strategy; import ai.chat2db.server.domain.api.enums.ExportFileSuffix; -import ai.chat2db.server.tools.common.model.export.data.option.ExportDataOption; +import ai.chat2db.server.tools.common.model.data.option.ExportDataOption; import com.alibaba.excel.support.ExcelTypeEnum; import jakarta.servlet.http.HttpServletResponse; diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2JsonStrategy.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2JsonStrategy.java index 46e115120..d483f8772 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2JsonStrategy.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2JsonStrategy.java @@ -2,8 +2,8 @@ import ai.chat2db.server.domain.api.enums.ExportFileSuffix; import ai.chat2db.server.tools.base.excption.BusinessException; -import ai.chat2db.server.tools.common.model.export.data.option.ExportDataOption; -import ai.chat2db.server.tools.common.model.export.data.option.JSONExportDataOption; +import ai.chat2db.server.tools.common.model.data.option.ExportDataOption; +import ai.chat2db.server.tools.common.model.data.option.JSONExportDataOption; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.SerializationFeature; import jakarta.servlet.http.HttpServletResponse; diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2SqlStrategy.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2SqlStrategy.java index 4e8a61c41..9e1120f5e 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2SqlStrategy.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2SqlStrategy.java @@ -1,8 +1,8 @@ package ai.chat2db.server.web.api.controller.rdb.data.export.strategy; import ai.chat2db.server.domain.api.enums.ExportFileSuffix; -import ai.chat2db.server.tools.common.model.export.data.option.ExportDataOption; -import ai.chat2db.server.tools.common.model.export.data.option.SQLExportDataOption; +import ai.chat2db.server.tools.common.model.data.option.ExportDataOption; +import ai.chat2db.server.tools.common.model.data.option.SQLExportDataOption; import ai.chat2db.spi.util.ResultSetUtils; import jakarta.servlet.http.HttpServletResponse; import org.apache.commons.lang3.StringUtils; diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBDataStrategy.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBDataStrategy.java index 1f97bab9c..6d5b24458 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBDataStrategy.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBDataStrategy.java @@ -2,14 +2,13 @@ import ai.chat2db.server.domain.api.enums.ExportFileSuffix; import ai.chat2db.server.domain.api.param.datasource.DatabaseExportDataParam; -import ai.chat2db.server.tools.common.model.export.data.option.ExportDataOption; -import ai.chat2db.server.tools.common.model.export.data.option.TableOption; -import ai.chat2db.server.web.api.controller.rdb.RdbDmlExportController; +import ai.chat2db.server.domain.api.param.datasource.DatabaseImportDataParam; +import ai.chat2db.server.tools.common.model.data.option.ExportDataOption; +import ai.chat2db.server.tools.common.model.data.option.TableOption; import ai.chat2db.spi.sql.Chat2DBContext; import ai.chat2db.spi.util.ResultSetUtils; import com.alibaba.excel.EasyExcel; import com.alibaba.excel.support.ExcelTypeEnum; -import com.alibaba.excel.write.builder.ExcelWriterBuilder; import com.alibaba.excel.write.builder.ExcelWriterSheetBuilder; import jakarta.servlet.ServletOutputStream; import jakarta.servlet.http.HttpServletResponse; @@ -160,4 +159,7 @@ public List> getListHeadList(List headers) { .collect(Collectors.toList()); } + public void doImport(DatabaseImportDataParam param) { + + } } \ No newline at end of file diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/request/DatabaseExportDataRequest.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/request/DatabaseExportDataRequest.java index 9b9e518f3..5e6b95fd4 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/request/DatabaseExportDataRequest.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/request/DatabaseExportDataRequest.java @@ -1,7 +1,7 @@ package ai.chat2db.server.web.api.controller.rdb.request; -import ai.chat2db.server.tools.common.model.export.data.option.ExportDataOption; -import ai.chat2db.server.tools.common.model.export.data.option.TableOption; +import ai.chat2db.server.tools.common.model.data.option.ExportDataOption; +import ai.chat2db.server.tools.common.model.data.option.TableOption; import ai.chat2db.server.web.api.controller.data.source.request.DataSourceBaseRequest; import jakarta.validation.constraints.NotNull; import lombok.AllArgsConstructor; diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/request/DatabaseImportDataRequest.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/request/DatabaseImportDataRequest.java new file mode 100644 index 000000000..06ba8ce6c --- /dev/null +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/request/DatabaseImportDataRequest.java @@ -0,0 +1,20 @@ +package ai.chat2db.server.web.api.controller.rdb.request; + +import ai.chat2db.server.tools.common.model.data.option.ImportDataOption; +import ai.chat2db.server.web.api.controller.data.source.request.DataSourceBaseRequest; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * @author: zgq + * @date: 2024年04月23日 13:50 + */ +@Data +@AllArgsConstructor +@NoArgsConstructor +public class DatabaseImportDataRequest extends DataSourceBaseRequest { + + private ImportDataOption importDataOption; + +} From dbe81e14b64562b66a9efb3f11fb366a34b711cd Mon Sep 17 00:00:00 2001 From: zgq <203083679@qq.com> Date: Thu, 25 Apr 2024 20:59:55 +0800 Subject: [PATCH 04/11] =?UTF-8?q?support=20import=20CSV=EF=BC=8CEXCEL?= =?UTF-8?q?=EF=BC=8CJSON?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../datasource/DatabaseExportDataParam.java | 4 +- .../datasource/DatabaseImportDataParam.java | 3 + .../data/option/CSVImportDataOption.java | 18 +++ ...ableOption.java => ExportTableOption.java} | 4 +- .../model/data/option/ImportDataOption.java | 11 +- .../model/data/option/ImportTableOption.java | 20 +++ .../data/option/JSONImportDataOption.java | 16 +++ .../web/api/aspect/ConnectionInfoHandler.java | 7 + .../controller/rdb/DatabaseController.java | 33 +++-- .../strategy/ExportDBData2CsvStrategy.java | 35 +++++ .../strategy/ExportDBData2ExcelStrategy.java | 20 +++ .../strategy/ExportDBData2JsonStrategy.java | 81 +++++++++-- .../strategy/ExportDBData2SqlStrategy.java | 11 ++ .../export/strategy/ExportDBDataStrategy.java | 43 +++--- .../export/strategy/NoModelDataListener.java | 131 ++++++++++++++++++ .../request/DatabaseExportDataRequest.java | 4 +- .../request/DatabaseImportDataRequest.java | 4 +- 17 files changed, 391 insertions(+), 54 deletions(-) create mode 100644 chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/CSVImportDataOption.java rename chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/{TableOption.java => ExportTableOption.java} (83%) create mode 100644 chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/ImportTableOption.java create mode 100644 chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/JSONImportDataOption.java create mode 100644 chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/NoModelDataListener.java diff --git a/chat2db-server/chat2db-server-domain/chat2db-server-domain-api/src/main/java/ai/chat2db/server/domain/api/param/datasource/DatabaseExportDataParam.java b/chat2db-server/chat2db-server-domain/chat2db-server-domain-api/src/main/java/ai/chat2db/server/domain/api/param/datasource/DatabaseExportDataParam.java index 00ff0973b..bd28b9b57 100644 --- a/chat2db-server/chat2db-server-domain/chat2db-server-domain-api/src/main/java/ai/chat2db/server/domain/api/param/datasource/DatabaseExportDataParam.java +++ b/chat2db-server/chat2db-server-domain/chat2db-server-domain-api/src/main/java/ai/chat2db/server/domain/api/param/datasource/DatabaseExportDataParam.java @@ -1,7 +1,7 @@ package ai.chat2db.server.domain.api.param.datasource; import ai.chat2db.server.tools.common.model.data.option.ExportDataOption; -import ai.chat2db.server.tools.common.model.data.option.TableOption; +import ai.chat2db.server.tools.common.model.data.option.ExportTableOption; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; @@ -19,5 +19,5 @@ public class DatabaseExportDataParam { private String databaseName; private String schemaName; private ExportDataOption exportDataOption; - private List tableOptions; + private List exportTableOptions; } \ No newline at end of file diff --git a/chat2db-server/chat2db-server-domain/chat2db-server-domain-api/src/main/java/ai/chat2db/server/domain/api/param/datasource/DatabaseImportDataParam.java b/chat2db-server/chat2db-server-domain/chat2db-server-domain-api/src/main/java/ai/chat2db/server/domain/api/param/datasource/DatabaseImportDataParam.java index c2bf5d082..48ceeb90a 100644 --- a/chat2db-server/chat2db-server-domain/chat2db-server-domain-api/src/main/java/ai/chat2db/server/domain/api/param/datasource/DatabaseImportDataParam.java +++ b/chat2db-server/chat2db-server-domain/chat2db-server-domain-api/src/main/java/ai/chat2db/server/domain/api/param/datasource/DatabaseImportDataParam.java @@ -1,6 +1,8 @@ package ai.chat2db.server.domain.api.param.datasource; import ai.chat2db.server.tools.common.model.data.option.ImportDataOption; +import ai.chat2db.server.tools.common.model.data.option.ExportTableOption; +import ai.chat2db.server.tools.common.model.data.option.ImportTableOption; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; @@ -17,5 +19,6 @@ public class DatabaseImportDataParam { private String databaseName; private String schemaName; + private ImportTableOption importTableOption; private ImportDataOption importDataOption; } diff --git a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/CSVImportDataOption.java b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/CSVImportDataOption.java new file mode 100644 index 000000000..c01eaaa35 --- /dev/null +++ b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/CSVImportDataOption.java @@ -0,0 +1,18 @@ +package ai.chat2db.server.tools.common.model.data.option; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * @author: zgq + * @date: 2024年04月24日 20:49 + */ +@Data +@AllArgsConstructor +@NoArgsConstructor +public class CSVImportDataOption extends ImportDataOption { + private Integer headerRowNum; + private Integer dataStartRowNum; + private Integer dataEndRowNum; +} diff --git a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/TableOption.java b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/ExportTableOption.java similarity index 83% rename from chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/TableOption.java rename to chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/ExportTableOption.java index 6fdc68cf0..7b2f72f60 100644 --- a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/TableOption.java +++ b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/ExportTableOption.java @@ -11,11 +11,11 @@ @Data @AllArgsConstructor @NoArgsConstructor -public class TableOption { +public class ExportTableOption { @NotNull private String tableName; @NotEmpty @NotNull - private List filedNames; + private List exportColumnNames; } \ No newline at end of file diff --git a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/ImportDataOption.java b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/ImportDataOption.java index ba1e3e6a4..3134d3daa 100644 --- a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/ImportDataOption.java +++ b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/ImportDataOption.java @@ -16,13 +16,14 @@ @NoArgsConstructor @JsonTypeInfo( use = JsonTypeInfo.Id.NAME, - property = "exportType", - visible = true) + property = "importType", + visible = true +) @JsonSubTypes({ - @JsonSubTypes.Type(value = ImportDataOption.class, name = "CSV"), + @JsonSubTypes.Type(value = CSVImportDataOption.class, name = "CSV"), @JsonSubTypes.Type(value = ImportDataOption.class, name = "SQL"), - @JsonSubTypes.Type(value = ImportDataOption.class, name = "EXCEL"), - @JsonSubTypes.Type(value = ImportDataOption.class, name = "JSON") + @JsonSubTypes.Type(value = CSVImportDataOption.class, name = "EXCEL"), + @JsonSubTypes.Type(value = JSONImportDataOption.class, name = "JSON") }) public class ImportDataOption { @NotBlank diff --git a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/ImportTableOption.java b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/ImportTableOption.java new file mode 100644 index 000000000..2c45da366 --- /dev/null +++ b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/ImportTableOption.java @@ -0,0 +1,20 @@ +package ai.chat2db.server.tools.common.model.data.option; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.List; + +/** + * @author: zgq + * @date: 2024年04月24日 16:50 + */ +@Data +@AllArgsConstructor +@NoArgsConstructor +public class ImportTableOption { + private String tableName; + private List srcColumnNames; + private List targetColumnNames; +} diff --git a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/JSONImportDataOption.java b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/JSONImportDataOption.java new file mode 100644 index 000000000..87e0281e7 --- /dev/null +++ b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/JSONImportDataOption.java @@ -0,0 +1,16 @@ +package ai.chat2db.server.tools.common.model.data.option; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * @author: zgq + * @date: 2024年04月24日 16:55 + */ +@Data +@AllArgsConstructor +@NoArgsConstructor +public class JSONImportDataOption extends ImportDataOption{ + private String rootNodeName; +} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/aspect/ConnectionInfoHandler.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/aspect/ConnectionInfoHandler.java index ff1fe624a..804e14f95 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/aspect/ConnectionInfoHandler.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/aspect/ConnectionInfoHandler.java @@ -12,6 +12,8 @@ import ai.chat2db.spi.config.DriverConfig; import ai.chat2db.spi.sql.Chat2DBContext; import ai.chat2db.spi.sql.ConnectInfo; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; import jakarta.annotation.Resource; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; @@ -42,6 +44,11 @@ public Object connectionInfoHandler(ProceedingJoinPoint proceedingJoinPoint) thr if (params != null && params.length > 0) { for (int i = 0; i < params.length; i++) { Object param = params[i]; + if (param instanceof String && StringUtils.isNotBlank((String) param)) { + ObjectMapper objectMapper = new ObjectMapper(); + objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + param = objectMapper.readValue((String) param, DataSourceBaseRequest.class); + } if (param instanceof DataSourceBaseRequest) { Long dataSourceId = ((DataSourceBaseRequest) param).getDataSourceId(); String schemaName = ((DataSourceBaseRequest) param).getSchemaName(); diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/DatabaseController.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/DatabaseController.java index f3308aaa4..589590363 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/DatabaseController.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/DatabaseController.java @@ -18,11 +18,13 @@ import ai.chat2db.spi.model.Database; import ai.chat2db.spi.model.MetaSchema; import ai.chat2db.spi.model.Sql; +import com.fasterxml.jackson.databind.ObjectMapper; import jakarta.servlet.http.HttpServletResponse; import jakarta.validation.Valid; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.*; +import org.springframework.web.multipart.MultipartFile; import java.io.PrintWriter; import java.lang.reflect.Constructor; @@ -53,8 +55,8 @@ public class DatabaseController { @GetMapping("/database_schema_list") public DataResult databaseSchemaList(@Valid DataSourceBaseRequest request) { MetaDataQueryParam queryParam = MetaDataQueryParam.builder().dataSourceId(request.getDataSourceId()) - .refresh( - request.isRefresh()).build(); + .refresh( + request.isRefresh()).build(); DataResult result = databaseService.queryDatabaseSchema(queryParam); MetaSchemaVO schemaDto2vo = rdbWebConverter.metaSchemaDto2vo(result.getData()); return DataResult.of(schemaDto2vo); @@ -63,8 +65,8 @@ public DataResult databaseSchemaList(@Valid DataSourceBaseRequest @GetMapping("list") public ListResult databaseList(@Valid DataSourceBaseRequest request) { DatabaseQueryAllParam queryParam = DatabaseQueryAllParam.builder().dataSourceId(request.getDataSourceId()) - .refresh( - request.isRefresh()).build(); + .refresh( + request.isRefresh()).build(); ListResult result = databaseService.queryAll(queryParam); return ListResult.of(rdbWebConverter.databaseDto2vo(result.getData())); } @@ -89,7 +91,7 @@ public ActionResult deleteDatabase(@Valid @RequestBody DataSourceBaseRequest req */ @PostMapping("/create_database_sql") public DataResult createDatabase(@Valid @RequestBody DatabaseCreateRequest request) { - if(StringUtils.isBlank(request.getName())){ + if (StringUtils.isBlank(request.getName())) { request.setName(request.getDatabaseName()); } Database database = databaseConverter.request2param(request); @@ -105,12 +107,13 @@ public DataResult createDatabase(@Valid @RequestBody DatabaseCreateRequest @PostMapping("/modify_database") public ActionResult modifyDatabase(@Valid @RequestBody UpdateDatabaseRequest request) { DatabaseCreateParam param = DatabaseCreateParam.builder().name(request.getDatabaseName()) - .name(request.getNewDatabaseName()).build(); + .name(request.getNewDatabaseName()).build(); return databaseService.modifyDatabase(param); } + @PostMapping("/export") - public void exportDatabase(@Valid @RequestBody DatabaseExportRequest request, HttpServletResponse response){ - String fileName = Objects.isNull(request.getSchemaName())?request.getDatabaseName() : request.getSchemaName(); + public void exportDatabase(@Valid @RequestBody DatabaseExportRequest request, HttpServletResponse response) { + String fileName = Objects.isNull(request.getSchemaName()) ? request.getDatabaseName() : request.getSchemaName(); response.setContentType("text/sql"); response.setHeader("Content-disposition", "attachment;filename*=utf-8''" + fileName + ".sql"); response.setCharacterEncoding("utf-8"); @@ -124,7 +127,7 @@ public void exportDatabase(@Valid @RequestBody DatabaseExportRequest request, Ht } @PostMapping("/export_data") - public void exportData(@Valid @RequestBody DatabaseExportDataRequest request, HttpServletResponse response) { + public void exportData(@Valid @RequestBody DatabaseExportDataRequest request, HttpServletResponse response) { Class targetClass = ExportDBDataStrategyFactory.get(request.getExportDataOption().getExportType()); response.setCharacterEncoding("utf-8"); DatabaseExportDataParam param = databaseConverter.request2param(request); @@ -140,16 +143,18 @@ public void exportData(@Valid @RequestBody DatabaseExportDataRequest request, Ht } @PostMapping("/import_data") - public void importData(@Valid @RequestBody DatabaseImportDataRequest request) { - Class targetClass = ExportDBDataStrategyFactory.get(request.getImportDataOption().getImportType()); - DatabaseImportDataParam param = databaseConverter.request2param(request); + public ActionResult importData(@Valid @ModelAttribute(value = "file") MultipartFile file, @ModelAttribute("json") String json) { try { + ObjectMapper objectMapper = new ObjectMapper(); + DatabaseImportDataRequest request = objectMapper.readValue(json, DatabaseImportDataRequest.class); + Class targetClass = ExportDBDataStrategyFactory.get(request.getImportDataOption().getImportType()); + DatabaseImportDataParam param = databaseConverter.request2param(request); Constructor constructor = targetClass.getDeclaredConstructor(); ExportDBDataStrategy service = (ExportDBDataStrategy) constructor.newInstance(); - service.doImport(param); + service.doImport(param, file); + return ActionResult.isSuccess(); } catch (Exception e) { throw new RuntimeException(e); } - } } diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2CsvStrategy.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2CsvStrategy.java index 240086a0d..e42901b34 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2CsvStrategy.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2CsvStrategy.java @@ -1,9 +1,18 @@ package ai.chat2db.server.web.api.controller.rdb.data.export.strategy; import ai.chat2db.server.domain.api.enums.ExportFileSuffix; +import ai.chat2db.server.tools.base.excption.BusinessException; +import ai.chat2db.server.tools.common.model.data.option.CSVImportDataOption; import ai.chat2db.server.tools.common.model.data.option.ExportDataOption; +import ai.chat2db.server.tools.common.model.data.option.ImportDataOption; +import ai.chat2db.server.tools.common.model.data.option.ImportTableOption; +import ai.chat2db.spi.sql.Chat2DBContext; +import com.alibaba.excel.EasyExcel; +import com.alibaba.excel.exception.ExcelAnalysisException; import com.alibaba.excel.support.ExcelTypeEnum; import jakarta.servlet.http.HttpServletResponse; +import lombok.extern.slf4j.Slf4j; +import org.springframework.web.multipart.MultipartFile; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -13,7 +22,9 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.util.List; +import java.util.Objects; +@Slf4j public class ExportDBData2CsvStrategy extends ExportDBDataStrategy { public ExportDBData2CsvStrategy() { @@ -41,6 +52,30 @@ protected void doTableDataExport(HttpServletResponse response, Connection connec } } + + @Override + protected void doTableDataImport(Connection connection, String databaseName, String schemaName, + ImportTableOption importTableOption, + ImportDataOption importDataOption, MultipartFile file) { + NoModelDataListener noModelDataListener = new NoModelDataListener(schemaName, importTableOption, + importDataOption, Chat2DBContext.getConnection()); + try { + Integer headerRowNum = ((CSVImportDataOption) importDataOption).getHeaderRowNum(); + EasyExcel.read(file.getInputStream(), noModelDataListener) + .sheet() + .headRowNumber(headerRowNum) + .doRead(); + } catch (IOException e) { + throw new RuntimeException(e); + } catch (ExcelAnalysisException e) { + if (Objects.equals("超出指定数据行", e.getMessage())) { + log.info("数据读取完成"); + } else { + throw new BusinessException("parse.excel.error", null, e); + } + } + } + private void export2CSV(Connection connection, String schemaName, String tableName, OutputStream out, List fileNames, ExportDataOption options) throws SQLException { try (ResultSet resultSet = connection.createStatement().executeQuery(buildQuerySql(schemaName, tableName))) { write(tableName, out, fileNames, options, resultSet, StandardCharsets.UTF_8, ExcelTypeEnum.CSV); diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2ExcelStrategy.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2ExcelStrategy.java index c87855743..2c9b22ce5 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2ExcelStrategy.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2ExcelStrategy.java @@ -2,8 +2,11 @@ import ai.chat2db.server.domain.api.enums.ExportFileSuffix; import ai.chat2db.server.tools.common.model.data.option.ExportDataOption; +import ai.chat2db.server.tools.common.model.data.option.ImportDataOption; +import ai.chat2db.server.tools.common.model.data.option.ImportTableOption; import com.alibaba.excel.support.ExcelTypeEnum; import jakarta.servlet.http.HttpServletResponse; +import org.springframework.web.multipart.MultipartFile; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -41,6 +44,23 @@ protected void doTableDataExport(HttpServletResponse response, Connection connec } } + /** + * @param tableName + * @param srcColumnNames + * @param targetColumnNames + * @param connection + * @param databaseName + * @param schemaName + * @param importDataOption + * @param file + * @return + */ + @Override + protected void doTableDataImport(Connection connection, String databaseName, String schemaName, + ImportTableOption importTableOption, + ImportDataOption importDataOption, MultipartFile file) { + } + private void export2EXCEL(Connection connection, String schemaName, String tableName, OutputStream out, List fileNames, ExportDataOption options) throws SQLException { try (ResultSet resultSet = connection.createStatement().executeQuery(buildQuerySql(schemaName, tableName))) { write(tableName, out, fileNames, options, resultSet, StandardCharsets.UTF_8, ExcelTypeEnum.XLSX); diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2JsonStrategy.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2JsonStrategy.java index d483f8772..57bfdbbbd 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2JsonStrategy.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2JsonStrategy.java @@ -2,27 +2,23 @@ import ai.chat2db.server.domain.api.enums.ExportFileSuffix; import ai.chat2db.server.tools.base.excption.BusinessException; -import ai.chat2db.server.tools.common.model.data.option.ExportDataOption; -import ai.chat2db.server.tools.common.model.data.option.JSONExportDataOption; +import ai.chat2db.server.tools.common.model.data.option.*; +import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.SerializationFeature; import jakarta.servlet.http.HttpServletResponse; +import org.apache.commons.lang3.StringUtils; import org.jetbrains.annotations.NotNull; +import org.springframework.web.multipart.MultipartFile; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.OutputStreamWriter; import java.io.PrintWriter; import java.nio.charset.StandardCharsets; -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.ResultSetMetaData; -import java.sql.SQLException; +import java.sql.*; import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; +import java.util.*; public class ExportDBData2JsonStrategy extends ExportDBDataStrategy { @@ -55,6 +51,71 @@ protected void doTableDataExport(HttpServletResponse response, Connection connec } + + @Override + protected void doTableDataImport(Connection connection, String databaseName, String schemaName, + ImportTableOption importTableOption, + ImportDataOption importDataOption, MultipartFile file) { + String tableName = importTableOption.getTableName(); + List srcColumnNames = importTableOption.getSrcColumnNames(); + List targetColumnNames = importTableOption.getTargetColumnNames(); + List sqlList = new ArrayList<>(); + String rootNodeName = ((JSONImportDataOption) importDataOption).getRootNodeName(); + ObjectMapper objectMapper = new ObjectMapper(); + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss"); + objectMapper.setDateFormat(dateFormat); + objectMapper.configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, false); + try (Statement statement = connection.createStatement()) { + JsonNode jsonNode = objectMapper.readTree(file.getInputStream()); + if (StringUtils.isNotBlank(rootNodeName) && jsonNode.has(rootNodeName)) { + jsonNode = jsonNode.get(rootNodeName); + } + if (!jsonNode.isArray() || jsonNode.size() <= 0) { + throw new BusinessException("jsonFile.parse.error"); + } + Iterator records = jsonNode.elements(); + while (records.hasNext()) { + JsonNode recordNode = records.next(); + StringBuilder valuesBuilder = new StringBuilder(); + for (int i = 0; i < srcColumnNames.size(); i++) { + String columnName = srcColumnNames.get(i); + JsonNode columnValueNode = recordNode.get(columnName); + String columnValue = (columnValueNode != null) ? columnValueNode.asText() : "NULL"; + valuesBuilder.append((Objects.equals(columnValue, "NULL")) ? columnValue : ("'" + columnValue + "'")); + if (i < srcColumnNames.size() - 1) { + valuesBuilder.append(", "); + } + } + String columns = String.join(", ", targetColumnNames); + String values = valuesBuilder.toString(); + String sql = buildInsertSql(schemaName, tableName, columns, values); + sqlList.add(sql); + if (sqlList.size()>=1000) { + statement.addBatch(String.join("; ", sqlList)); + statement.executeBatch(); + statement.clearBatch(); + sqlList.clear(); + } + } + if (sqlList.size()>0) { + statement.addBatch(String.join("; ", sqlList)); + statement.executeBatch(); + statement.clearBatch(); + sqlList.clear(); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + private String buildInsertSql(String schemaName, String tableName, String columns, String values) { + if (StringUtils.isNotBlank(schemaName)) { + return String.format("INSERT INTO %s.%s (%s) VALUES (%s)", schemaName, tableName, columns, values); + } else { + return String.format("INSERT INTO %s (%s) VALUES (%s)", tableName, columns, values); + } + } + private void export2JSON(ResultSet resultSet, PrintWriter writer, List filedNames, ExportDataOption options) throws SQLException { List> data = getDataMap(resultSet, filedNames); ObjectMapper objectMapper = new ObjectMapper(); diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2SqlStrategy.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2SqlStrategy.java index 9e1120f5e..90fb3fb80 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2SqlStrategy.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2SqlStrategy.java @@ -2,10 +2,13 @@ import ai.chat2db.server.domain.api.enums.ExportFileSuffix; import ai.chat2db.server.tools.common.model.data.option.ExportDataOption; +import ai.chat2db.server.tools.common.model.data.option.ImportDataOption; +import ai.chat2db.server.tools.common.model.data.option.ImportTableOption; import ai.chat2db.server.tools.common.model.data.option.SQLExportDataOption; import ai.chat2db.spi.util.ResultSetUtils; import jakarta.servlet.http.HttpServletResponse; import org.apache.commons.lang3.StringUtils; +import org.springframework.web.multipart.MultipartFile; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -53,6 +56,14 @@ protected void doTableDataExport(HttpServletResponse response, Connection connec } + + @Override + protected void doTableDataImport(Connection connection, String databaseName, String schemaName, + ImportTableOption importTableOption, + ImportDataOption importDataOption, MultipartFile file) { + + } + private String export2InsertSQL(Connection connection, String schemaName, String tableName, List filedNames, ExportDataOption options) throws SQLException { diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBDataStrategy.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBDataStrategy.java index 6d5b24458..1344772a7 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBDataStrategy.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBDataStrategy.java @@ -4,7 +4,9 @@ import ai.chat2db.server.domain.api.param.datasource.DatabaseExportDataParam; import ai.chat2db.server.domain.api.param.datasource.DatabaseImportDataParam; import ai.chat2db.server.tools.common.model.data.option.ExportDataOption; -import ai.chat2db.server.tools.common.model.data.option.TableOption; +import ai.chat2db.server.tools.common.model.data.option.ExportTableOption; +import ai.chat2db.server.tools.common.model.data.option.ImportDataOption; +import ai.chat2db.server.tools.common.model.data.option.ImportTableOption; import ai.chat2db.spi.sql.Chat2DBContext; import ai.chat2db.spi.util.ResultSetUtils; import com.alibaba.excel.EasyExcel; @@ -16,6 +18,7 @@ import lombok.Getter; import lombok.NoArgsConstructor; import org.jetbrains.annotations.NotNull; +import org.springframework.web.multipart.MultipartFile; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -51,27 +54,33 @@ public void doExport(DatabaseExportDataParam param, HttpServletResponse response String schemaName = param.getSchemaName(); setResponseHeaders(param, response); try (Connection connection = Chat2DBContext.getConnection()) { - if (param.getTableOptions().size() == 1) { - String tableName = param.getTableOptions().get(0).getTableName(); - List filedNames = param.getTableOptions().get(0).getFiledNames(); + if (param.getExportTableOptions().size() == 1) { + String tableName = param.getExportTableOptions().get(0).getTableName(); + List filedNames = param.getExportTableOptions().get(0).getExportColumnNames(); doTableDataExport(response, connection, databaseName, schemaName, tableName, filedNames, param.getExportDataOption()); } else { - doDbDataExport(response, connection, databaseName, schemaName, param.getTableOptions(), param.getExportDataOption()); + doDbDataExport(response, connection, databaseName, schemaName, param.getExportTableOptions(), param.getExportDataOption()); } } catch (Exception e) { throw new RuntimeException(e); } } + public void doImport(DatabaseImportDataParam param, MultipartFile file) { + doTableDataImport(Chat2DBContext.getConnection(), param.getDatabaseName(), param.getSchemaName(), + param.getImportTableOption(), param.getImportDataOption(), file); + + } + private void doDbDataExport(HttpServletResponse response, Connection connection, String databaseName, - String schemaName, List tableOptions, + String schemaName, List exportTableOptions, ExportDataOption options) throws IOException, SQLException { ServletOutputStream outputStream = response.getOutputStream(); ZipOutputStream zipOut = new ZipOutputStream(outputStream); - for (TableOption tableOption : tableOptions) { - String tableName = tableOption.getTableName(); + for (ExportTableOption exportTableOption : exportTableOptions) { + String tableName = exportTableOption.getTableName(); String fileName = tableName + getSuffix(); - List filedNames = tableOption.getFiledNames(); + List filedNames = exportTableOption.getExportColumnNames(); zipOut.putNextEntry(new ZipEntry(fileName)); ByteArrayOutputStream byteOut = doTableDataExport(connection, databaseName, schemaName, tableName, filedNames, options); byteOut.writeTo(zipOut); @@ -82,8 +91,8 @@ private void doDbDataExport(HttpServletResponse response, Connection connection, private void setResponseHeaders(DatabaseExportDataParam param, HttpServletResponse response) { - if (param.getTableOptions().size() == 1) { - String tableName = param.getTableOptions().get(0).getTableName(); + if (param.getExportTableOptions().size() == 1) { + String tableName = param.getExportTableOptions().get(0).getTableName(); response.setContentType(contentType); response.setHeader("Content-disposition", "attachment;filename*=utf-8''" + tableName + suffix); } else { @@ -107,6 +116,10 @@ public String buildQuerySql(String schemaName, String tableName) { return sql; } + protected abstract void doTableDataImport(Connection connection, String databaseName, String schemaName, + ImportTableOption importTableOption, + ImportDataOption importDataOption, MultipartFile file); + protected abstract ByteArrayOutputStream doTableDataExport(Connection connection, String databaseName, String schemaName, String tableName, List filedNames, ExportDataOption options) throws SQLException; @@ -124,9 +137,9 @@ public void write(String tableName, OutputStream out, List fileNames, Ex if (containsHeader) { List header = ResultSetUtils.getRsHeader(resultSet); if (fileNames.size() != header.size()) { - excelWriterSheetBuilder.head(getListHeadList(fileNames)); + excelWriterSheetBuilder.head(getListHeadList(fileNames)); } else { - excelWriterSheetBuilder.head(getListHeadList(header)); + excelWriterSheetBuilder.head(getListHeadList(header)); } } excelWriterSheetBuilder.doWrite(getDataList(resultSet, fileNames)); @@ -158,8 +171,4 @@ public List> getListHeadList(List headers) { .map(Collections::singletonList) .collect(Collectors.toList()); } - - public void doImport(DatabaseImportDataParam param) { - - } } \ No newline at end of file diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/NoModelDataListener.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/NoModelDataListener.java new file mode 100644 index 000000000..2098b0391 --- /dev/null +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/NoModelDataListener.java @@ -0,0 +1,131 @@ +package ai.chat2db.server.web.api.controller.rdb.data.export.strategy; + +import ai.chat2db.server.tools.common.model.data.option.CSVImportDataOption; +import ai.chat2db.server.tools.common.model.data.option.ImportDataOption; +import ai.chat2db.server.tools.common.model.data.option.ImportTableOption; +import com.alibaba.excel.context.AnalysisContext; +import com.alibaba.excel.event.AnalysisEventListener; +import com.alibaba.excel.exception.ExcelAnalysisException; +import com.alibaba.excel.metadata.data.ReadCellData; +import com.alibaba.excel.util.ConverterUtils; +import com.alibaba.excel.util.ListUtils; +import com.alibaba.fastjson2.JSON; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; + +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.List; +import java.util.Map; + +@Slf4j +public class NoModelDataListener extends AnalysisEventListener> { + + private final String schemaName; + private final ImportTableOption importTableOption; + private final ImportDataOption importDataOption; + private final Connection connection; + private final int BATCH_SIZE = 1000; + private final List sqlCacheList = ListUtils.newArrayListWithExpectedSize(BATCH_SIZE); + private final List columnIndexList; + private Integer recordIndex = 0; + private final Integer limitRowSize ; + + + public NoModelDataListener(String schemaName, ImportTableOption importTableOption, ImportDataOption importDataOption, Connection connection) { + this.schemaName = schemaName; + this.importTableOption = importTableOption; + this.connection = connection; + this.importDataOption = importDataOption; + this.columnIndexList = ListUtils.newArrayListWithExpectedSize(importTableOption.getSrcColumnNames().size()); + this.limitRowSize= ((CSVImportDataOption) importDataOption).getDataEndRowNum()- ((CSVImportDataOption) importDataOption).getDataStartRowNum()+1; + + } + + + /** + * @param headMap + * @param context + */ + @Override + public void invokeHead(Map> headMap, AnalysisContext context) { + Map stringMap = ConverterUtils.convertToStringMap(headMap, context); + for (Map.Entry entry : stringMap.entrySet()) { + List srcColumnNames = importTableOption.getSrcColumnNames(); + if (srcColumnNames.contains(entry.getValue())) { + columnIndexList.add(entry.getKey()); + } + } + } + + + @Override + public void invoke(Map data, AnalysisContext context) { + try { + if (++recordIndex >= limitRowSize) { + if (sqlCacheList.size() > 0) { + executeBatchInsert(); + } + throw new ExcelAnalysisException("超出指定数据行"); + } + + log.info("解析到一条数据:{}", JSON.toJSONString(data)); + StringBuilder valueBuilder = new StringBuilder(); + for (int i = 0; i < data.size(); i++) { + if (!columnIndexList.contains(i)) { + continue; + } + String value = data.get(i); + if (StringUtils.isBlank(value)) { + value = "NULL"; + } else { + value = "'" + value + "'"; + } + valueBuilder.append(value); + if (i != data.size() - 1 && columnIndexList.contains(i + 1)) { + valueBuilder.append(","); + } + } + + String sql = String.format("INSERT INTO %s (%s) VALUES (%s)", importTableOption.getTableName(), getColumns(importTableOption.getTargetColumnNames()), valueBuilder); + sqlCacheList.add(sql); + + if (sqlCacheList.size() >= 1000) { + executeBatchInsert(); + } + } catch (SQLException e) { + throw new RuntimeException(e); + } + } + + private void executeBatchInsert() throws SQLException { + try (Statement stmt = connection.createStatement()) { + for (String sql : sqlCacheList) { + stmt.addBatch(sql); + } + stmt.executeBatch(); + stmt.clearBatch(); + sqlCacheList.clear(); + } + } + + + private String getColumns(List columnNames) { + return String.join(",", columnNames); + + } + + @Override + public void doAfterAllAnalysed(AnalysisContext context) { + try { + if (sqlCacheList.size() > 0) { + executeBatchInsert(); + } + } catch (SQLException e) { + throw new RuntimeException(e); + } + + } +} + diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/request/DatabaseExportDataRequest.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/request/DatabaseExportDataRequest.java index 5e6b95fd4..21ec60755 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/request/DatabaseExportDataRequest.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/request/DatabaseExportDataRequest.java @@ -1,7 +1,7 @@ package ai.chat2db.server.web.api.controller.rdb.request; import ai.chat2db.server.tools.common.model.data.option.ExportDataOption; -import ai.chat2db.server.tools.common.model.data.option.TableOption; +import ai.chat2db.server.tools.common.model.data.option.ExportTableOption; import ai.chat2db.server.web.api.controller.data.source.request.DataSourceBaseRequest; import jakarta.validation.constraints.NotNull; import lombok.AllArgsConstructor; @@ -20,6 +20,6 @@ public class DatabaseExportDataRequest extends DataSourceBaseRequest { @NotNull private ExportDataOption exportDataOption; - private List tableOptions; + private List exportTableOptions; } \ No newline at end of file diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/request/DatabaseImportDataRequest.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/request/DatabaseImportDataRequest.java index 06ba8ce6c..87329a688 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/request/DatabaseImportDataRequest.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/request/DatabaseImportDataRequest.java @@ -1,6 +1,7 @@ package ai.chat2db.server.web.api.controller.rdb.request; import ai.chat2db.server.tools.common.model.data.option.ImportDataOption; +import ai.chat2db.server.tools.common.model.data.option.ImportTableOption; import ai.chat2db.server.web.api.controller.data.source.request.DataSourceBaseRequest; import lombok.AllArgsConstructor; import lombok.Data; @@ -14,7 +15,6 @@ @AllArgsConstructor @NoArgsConstructor public class DatabaseImportDataRequest extends DataSourceBaseRequest { - + private ImportTableOption importTableOption; private ImportDataOption importDataOption; - } From bb773d8f4b84b56651799b856fc27a007d08de33 Mon Sep 17 00:00:00 2001 From: zgq <203083679@qq.com> Date: Sat, 27 Apr 2024 15:15:05 +0800 Subject: [PATCH 05/11] Refactor code --- .../datasource/DatabaseExportDataParam.java | 10 +- .../datasource/DatabaseImportDataParam.java | 9 +- .../model/data/option/AbstractDataOption.java | 15 ++ .../option/AbstractDataOptionInterface.java | 5 + .../option/AbstractExportDataOptions.java | 25 ++ .../option/AbstractImportDataOptions.java | 24 ++ .../data/option/BaseExportDataOptions.java | 22 ++ .../data/option/BaseImportDataOptions.java | 21 ++ .../option/BaseImportExcelDataOptions.java | 18 ++ .../data/option/CSVImportDataOption.java | 18 -- .../model/data/option/ExportDataOption.java | 33 --- .../model/data/option/ExportTableOption.java | 21 -- .../model/data/option/ImportDataOption.java | 31 --- .../model/data/option/ImportTableOption.java | 20 -- .../data/option/JSONExportDataOption.java | 23 -- .../data/option/JSONImportDataOption.java | 16 -- .../data/option/SQLExportDataOption.java | 17 -- .../option/json/ExportData2JsonOptions.java | 24 ++ .../option/json/ImportJsonDataOptions.java | 22 ++ .../option/sql/BaseExportData2SqlOptions.java | 20 ++ .../data/option/table/BaseTableOptions.java | 21 ++ .../option/table/ImportNewTableOptions.java | 18 ++ .../data/option/table/ImportTableOptions.java | 27 +++ .../option/table/TableOptionInterface.java | 15 ++ .../controller/rdb/DatabaseController.java | 31 +-- .../rdb/data/AbstractMultiFileExporter.java | 63 +++++ .../rdb/data/AbstractSingleFileExporter.java | 46 ++++ .../controller/rdb/data/BaseFileImporter.java | 33 +++ .../rdb/data/BaseMultiExcelExporter.java | 37 +++ .../rdb/data/BaseSingleExcelExporter.java | 44 ++++ .../controller/rdb/data/DataFileExporter.java | 14 ++ .../rdb/data/DataFileFactoryProducer.java | 30 +++ .../rdb/data/DataFileImportExportFactory.java | 6 + .../controller/rdb/data/DataFileImporter.java | 11 + .../rdb/data/MultiFileExporter.java | 12 + .../rdb/data/SingleFileExporter.java | 11 + .../controller/rdb/data/csv/CSVExporter.java | 29 +++ .../rdb/data/csv/CSVImportExportFactory.java | 27 +++ .../controller/rdb/data/csv/CSVImporter.java | 15 ++ .../rdb/data/csv/MultiCSVExporter.java | 23 ++ .../rdb/data/csv/SingleCSVExporter.java | 28 +++ .../strategy/ExportDBData2CsvStrategy.java | 86 ------- .../strategy/ExportDBData2ExcelStrategy.java | 71 ------ .../strategy/ExportDBData2JsonStrategy.java | 154 ------------- .../strategy/ExportDBData2SqlStrategy.java | 217 ------------------ .../export/strategy/ExportDBDataStrategy.java | 174 -------------- .../rdb/data/json/EasyJsonExportUtil.java | 63 +++++ .../rdb/data/json/JSONExporter.java | 25 ++ .../data/json/JSONImportExportFactory.java | 23 ++ .../rdb/data/json/JSONImporter.java | 125 ++++++++++ .../rdb/data/json/MultiJSONExporter.java | 42 ++++ .../rdb/data/json/SingleJSONExporter.java | 42 ++++ .../rdb/data/sql/EasySqlBuilder.java | 183 +++++++++++++++ .../rdb/data/sql/MultiSQLExporter.java | 41 ++++ .../controller/rdb/data/sql/SQLExporter.java | 27 +++ .../rdb/data/sql/SQLImportExportFactory.java | 29 +++ .../controller/rdb/data/sql/SQLImporter.java | 25 ++ .../rdb/data/sql/SingleSQLExporter.java | 43 ++++ .../rdb/data/xlsx/BaseExcelImporter.java | 31 +++ .../rdb/data/xlsx/EasyExcelExportUtil.java | 67 ++++++ .../rdb/data/xlsx/MultiXLSXExporter.java | 22 ++ .../NoModelDataListener.java | 99 ++++---- .../rdb/data/xlsx/SingleXLSXExporter.java | 23 ++ .../rdb/data/xlsx/XLSXExporter.java | 24 ++ .../data/xlsx/XLSXImportExportFactory.java | 23 ++ .../rdb/data/xlsx/XLSXImporter.java | 12 + .../request/DatabaseExportDataRequest.java | 8 +- .../request/DatabaseImportDataRequest.java | 8 +- 68 files changed, 1661 insertions(+), 961 deletions(-) create mode 100644 chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/AbstractDataOption.java create mode 100644 chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/AbstractDataOptionInterface.java create mode 100644 chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/AbstractExportDataOptions.java create mode 100644 chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/AbstractImportDataOptions.java create mode 100644 chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/BaseExportDataOptions.java create mode 100644 chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/BaseImportDataOptions.java create mode 100644 chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/BaseImportExcelDataOptions.java delete mode 100644 chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/CSVImportDataOption.java delete mode 100644 chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/ExportDataOption.java delete mode 100644 chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/ExportTableOption.java delete mode 100644 chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/ImportDataOption.java delete mode 100644 chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/ImportTableOption.java delete mode 100644 chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/JSONExportDataOption.java delete mode 100644 chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/JSONImportDataOption.java delete mode 100644 chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/SQLExportDataOption.java create mode 100644 chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/json/ExportData2JsonOptions.java create mode 100644 chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/json/ImportJsonDataOptions.java create mode 100644 chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/sql/BaseExportData2SqlOptions.java create mode 100644 chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/table/BaseTableOptions.java create mode 100644 chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/table/ImportNewTableOptions.java create mode 100644 chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/table/ImportTableOptions.java create mode 100644 chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/table/TableOptionInterface.java create mode 100644 chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/AbstractMultiFileExporter.java create mode 100644 chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/AbstractSingleFileExporter.java create mode 100644 chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/BaseFileImporter.java create mode 100644 chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/BaseMultiExcelExporter.java create mode 100644 chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/BaseSingleExcelExporter.java create mode 100644 chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/DataFileExporter.java create mode 100644 chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/DataFileFactoryProducer.java create mode 100644 chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/DataFileImportExportFactory.java create mode 100644 chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/DataFileImporter.java create mode 100644 chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/MultiFileExporter.java create mode 100644 chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/SingleFileExporter.java create mode 100644 chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/csv/CSVExporter.java create mode 100644 chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/csv/CSVImportExportFactory.java create mode 100644 chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/csv/CSVImporter.java create mode 100644 chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/csv/MultiCSVExporter.java create mode 100644 chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/csv/SingleCSVExporter.java delete mode 100644 chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2CsvStrategy.java delete mode 100644 chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2ExcelStrategy.java delete mode 100644 chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2JsonStrategy.java delete mode 100644 chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2SqlStrategy.java delete mode 100644 chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBDataStrategy.java create mode 100644 chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/json/EasyJsonExportUtil.java create mode 100644 chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/json/JSONExporter.java create mode 100644 chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/json/JSONImportExportFactory.java create mode 100644 chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/json/JSONImporter.java create mode 100644 chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/json/MultiJSONExporter.java create mode 100644 chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/json/SingleJSONExporter.java create mode 100644 chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/sql/EasySqlBuilder.java create mode 100644 chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/sql/MultiSQLExporter.java create mode 100644 chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/sql/SQLExporter.java create mode 100644 chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/sql/SQLImportExportFactory.java create mode 100644 chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/sql/SQLImporter.java create mode 100644 chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/sql/SingleSQLExporter.java create mode 100644 chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/BaseExcelImporter.java create mode 100644 chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/EasyExcelExportUtil.java create mode 100644 chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/MultiXLSXExporter.java rename chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/{export/strategy => xlsx}/NoModelDataListener.java (51%) create mode 100644 chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/SingleXLSXExporter.java create mode 100644 chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/XLSXExporter.java create mode 100644 chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/XLSXImportExportFactory.java create mode 100644 chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/XLSXImporter.java diff --git a/chat2db-server/chat2db-server-domain/chat2db-server-domain-api/src/main/java/ai/chat2db/server/domain/api/param/datasource/DatabaseExportDataParam.java b/chat2db-server/chat2db-server-domain/chat2db-server-domain-api/src/main/java/ai/chat2db/server/domain/api/param/datasource/DatabaseExportDataParam.java index bd28b9b57..0ec3ed792 100644 --- a/chat2db-server/chat2db-server-domain/chat2db-server-domain-api/src/main/java/ai/chat2db/server/domain/api/param/datasource/DatabaseExportDataParam.java +++ b/chat2db-server/chat2db-server-domain/chat2db-server-domain-api/src/main/java/ai/chat2db/server/domain/api/param/datasource/DatabaseExportDataParam.java @@ -1,7 +1,8 @@ package ai.chat2db.server.domain.api.param.datasource; -import ai.chat2db.server.tools.common.model.data.option.ExportDataOption; -import ai.chat2db.server.tools.common.model.data.option.ExportTableOption; +import ai.chat2db.server.tools.common.model.data.option.AbstractExportDataOptions; +import ai.chat2db.server.tools.common.model.data.option.table.BaseTableOptions; +import jakarta.validation.constraints.NotNull; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; @@ -18,6 +19,7 @@ public class DatabaseExportDataParam { private String databaseName; private String schemaName; - private ExportDataOption exportDataOption; - private List exportTableOptions; + @NotNull + private AbstractExportDataOptions exportDataOption; + private List exportTableOptions; } \ No newline at end of file diff --git a/chat2db-server/chat2db-server-domain/chat2db-server-domain-api/src/main/java/ai/chat2db/server/domain/api/param/datasource/DatabaseImportDataParam.java b/chat2db-server/chat2db-server-domain/chat2db-server-domain-api/src/main/java/ai/chat2db/server/domain/api/param/datasource/DatabaseImportDataParam.java index 48ceeb90a..69e6bb1b5 100644 --- a/chat2db-server/chat2db-server-domain/chat2db-server-domain-api/src/main/java/ai/chat2db/server/domain/api/param/datasource/DatabaseImportDataParam.java +++ b/chat2db-server/chat2db-server-domain/chat2db-server-domain-api/src/main/java/ai/chat2db/server/domain/api/param/datasource/DatabaseImportDataParam.java @@ -1,8 +1,7 @@ package ai.chat2db.server.domain.api.param.datasource; -import ai.chat2db.server.tools.common.model.data.option.ImportDataOption; -import ai.chat2db.server.tools.common.model.data.option.ExportTableOption; -import ai.chat2db.server.tools.common.model.data.option.ImportTableOption; +import ai.chat2db.server.tools.common.model.data.option.AbstractImportDataOptions; +import ai.chat2db.server.tools.common.model.data.option.table.ImportTableOptions; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; @@ -19,6 +18,6 @@ public class DatabaseImportDataParam { private String databaseName; private String schemaName; - private ImportTableOption importTableOption; - private ImportDataOption importDataOption; + private ImportTableOptions importTableOption; + private AbstractImportDataOptions importDataOption; } diff --git a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/AbstractDataOption.java b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/AbstractDataOption.java new file mode 100644 index 000000000..f3a94baf3 --- /dev/null +++ b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/AbstractDataOption.java @@ -0,0 +1,15 @@ +package ai.chat2db.server.tools.common.model.data.option; + +import jakarta.validation.constraints.NotNull; +import lombok.Data; + +/** + * @author: zgq + * @date: 2024年04月25日 22:59 + */ +@Data +public abstract class AbstractDataOption implements AbstractDataOptionInterface { + @NotNull + protected String fileType; + +} diff --git a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/AbstractDataOptionInterface.java b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/AbstractDataOptionInterface.java new file mode 100644 index 000000000..1b7467dbb --- /dev/null +++ b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/AbstractDataOptionInterface.java @@ -0,0 +1,5 @@ +package ai.chat2db.server.tools.common.model.data.option; + +public interface AbstractDataOptionInterface { + String getFileType(); +} diff --git a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/AbstractExportDataOptions.java b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/AbstractExportDataOptions.java new file mode 100644 index 000000000..c6400c4c8 --- /dev/null +++ b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/AbstractExportDataOptions.java @@ -0,0 +1,25 @@ +package ai.chat2db.server.tools.common.model.data.option; + +import ai.chat2db.server.tools.common.model.data.option.json.ExportData2JsonOptions; +import ai.chat2db.server.tools.common.model.data.option.sql.BaseExportData2SqlOptions; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import jakarta.validation.constraints.NotNull; + +/** + * @author: zgq + * @date: 2024年04月25日 23:31 + */ +@JsonTypeInfo( + use = JsonTypeInfo.Id.NAME, + property = "fileType", + visible = true) +@JsonSubTypes({ + @JsonSubTypes.Type(value = BaseExportDataOptions.class, name = "CSV"), + @JsonSubTypes.Type(value = BaseExportData2SqlOptions.class, name = "SQL"), + @JsonSubTypes.Type(value = BaseExportDataOptions.class, name = "XLSX"), + @JsonSubTypes.Type(value = ExportData2JsonOptions.class, name = "JSON") +}) +public abstract class AbstractExportDataOptions extends AbstractDataOption { + +} diff --git a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/AbstractImportDataOptions.java b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/AbstractImportDataOptions.java new file mode 100644 index 000000000..7fc0aef90 --- /dev/null +++ b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/AbstractImportDataOptions.java @@ -0,0 +1,24 @@ +package ai.chat2db.server.tools.common.model.data.option; + +import ai.chat2db.server.tools.common.model.data.option.json.ImportJsonDataOptions; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; + +/** + * @author: zgq + * @date: 2024年04月25日 23:04 + */ +@JsonTypeInfo( + use = JsonTypeInfo.Id.NAME, + property = "fileType", + visible = true +) +@JsonSubTypes({ + @JsonSubTypes.Type(value = BaseImportExcelDataOptions.class, name = "CSV"), + @JsonSubTypes.Type(value = BaseImportExcelDataOptions.class, name = "XLSX"), + @JsonSubTypes.Type(value = BaseImportExcelDataOptions.class, name = "XLS"), + @JsonSubTypes.Type(value = ImportJsonDataOptions.class, name = "JSON") +}) +public abstract class AbstractImportDataOptions extends AbstractDataOption { + +} diff --git a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/BaseExportDataOptions.java b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/BaseExportDataOptions.java new file mode 100644 index 000000000..435e272c8 --- /dev/null +++ b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/BaseExportDataOptions.java @@ -0,0 +1,22 @@ +package ai.chat2db.server.tools.common.model.data.option; + +import jakarta.validation.constraints.NotNull; +import lombok.Data; +import lombok.EqualsAndHashCode; + +/** + * 功能描述 + * + * @author: zgq + * @date: 2024年04月26日 0:02 + */ +@Data +@EqualsAndHashCode(callSuper =true) +public class BaseExportDataOptions extends AbstractExportDataOptions { + @NotNull + public Boolean containsHeader; + + public BaseExportDataOptions() { + this.containsHeader = true; + } +} diff --git a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/BaseImportDataOptions.java b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/BaseImportDataOptions.java new file mode 100644 index 000000000..81ca23958 --- /dev/null +++ b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/BaseImportDataOptions.java @@ -0,0 +1,21 @@ +package ai.chat2db.server.tools.common.model.data.option; + +import ai.chat2db.server.tools.common.model.data.option.AbstractImportDataOptions; +import lombok.Data; +import lombok.EqualsAndHashCode; + +/** + * @author: zgq + * @date: 2024年04月25日 23:40 + */ +@Data +@EqualsAndHashCode(callSuper = true) +public class BaseImportDataOptions extends AbstractImportDataOptions { + public Integer dataStartRowNum; + public Integer dataEndRowNum; + + public BaseImportDataOptions() { + dataStartRowNum = 1; + dataEndRowNum = Integer.MAX_VALUE; + } +} diff --git a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/BaseImportExcelDataOptions.java b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/BaseImportExcelDataOptions.java new file mode 100644 index 000000000..82925130d --- /dev/null +++ b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/BaseImportExcelDataOptions.java @@ -0,0 +1,18 @@ +package ai.chat2db.server.tools.common.model.data.option; + +import lombok.Data; +import lombok.EqualsAndHashCode; + +/** + * @author: zgq + * @date: 2024年04月26日 10:06 + */ +@Data +@EqualsAndHashCode(callSuper = true) +public class BaseImportExcelDataOptions extends BaseImportDataOptions { + public Integer headerRowNum; + + public BaseImportExcelDataOptions() { + this.headerRowNum = 1; + } +} diff --git a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/CSVImportDataOption.java b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/CSVImportDataOption.java deleted file mode 100644 index c01eaaa35..000000000 --- a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/CSVImportDataOption.java +++ /dev/null @@ -1,18 +0,0 @@ -package ai.chat2db.server.tools.common.model.data.option; - -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; - -/** - * @author: zgq - * @date: 2024年04月24日 20:49 - */ -@Data -@AllArgsConstructor -@NoArgsConstructor -public class CSVImportDataOption extends ImportDataOption { - private Integer headerRowNum; - private Integer dataStartRowNum; - private Integer dataEndRowNum; -} diff --git a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/ExportDataOption.java b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/ExportDataOption.java deleted file mode 100644 index b1600e7c0..000000000 --- a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/ExportDataOption.java +++ /dev/null @@ -1,33 +0,0 @@ -package ai.chat2db.server.tools.common.model.data.option; - - -import com.fasterxml.jackson.annotation.JsonSubTypes; -import com.fasterxml.jackson.annotation.JsonTypeInfo; -import jakarta.validation.constraints.NotBlank; -import jakarta.validation.constraints.NotNull; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; - -@Data -@AllArgsConstructor -@NoArgsConstructor -@JsonTypeInfo( - use = JsonTypeInfo.Id.NAME, - property = "exportType", - visible = true) -@JsonSubTypes({ - @JsonSubTypes.Type(value = ExportDataOption.class, name = "CSV"), - @JsonSubTypes.Type(value = SQLExportDataOption.class, name = "SQL"), - @JsonSubTypes.Type(value = ExportDataOption.class, name = "EXCEL"), - @JsonSubTypes.Type(value = JSONExportDataOption.class, name = "JSON") -}) -public class ExportDataOption { - @NotBlank - private String exportType; - @NotNull - public Boolean containsHeader; - -} - - diff --git a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/ExportTableOption.java b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/ExportTableOption.java deleted file mode 100644 index 7b2f72f60..000000000 --- a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/ExportTableOption.java +++ /dev/null @@ -1,21 +0,0 @@ -package ai.chat2db.server.tools.common.model.data.option; - -import jakarta.validation.constraints.NotEmpty; -import jakarta.validation.constraints.NotNull; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; - -import java.util.List; - -@Data -@AllArgsConstructor -@NoArgsConstructor -public class ExportTableOption { - @NotNull - private String tableName; - @NotEmpty - @NotNull - private List exportColumnNames; - -} \ No newline at end of file diff --git a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/ImportDataOption.java b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/ImportDataOption.java deleted file mode 100644 index 3134d3daa..000000000 --- a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/ImportDataOption.java +++ /dev/null @@ -1,31 +0,0 @@ -package ai.chat2db.server.tools.common.model.data.option; - -import com.fasterxml.jackson.annotation.JsonSubTypes; -import com.fasterxml.jackson.annotation.JsonTypeInfo; -import jakarta.validation.constraints.NotBlank; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; - -/** - * @author: zgq - * @date: 2024年04月23日 13:58 - */ -@Data -@AllArgsConstructor -@NoArgsConstructor -@JsonTypeInfo( - use = JsonTypeInfo.Id.NAME, - property = "importType", - visible = true -) -@JsonSubTypes({ - @JsonSubTypes.Type(value = CSVImportDataOption.class, name = "CSV"), - @JsonSubTypes.Type(value = ImportDataOption.class, name = "SQL"), - @JsonSubTypes.Type(value = CSVImportDataOption.class, name = "EXCEL"), - @JsonSubTypes.Type(value = JSONImportDataOption.class, name = "JSON") -}) -public class ImportDataOption { - @NotBlank - private String importType; -} diff --git a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/ImportTableOption.java b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/ImportTableOption.java deleted file mode 100644 index 2c45da366..000000000 --- a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/ImportTableOption.java +++ /dev/null @@ -1,20 +0,0 @@ -package ai.chat2db.server.tools.common.model.data.option; - -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; - -import java.util.List; - -/** - * @author: zgq - * @date: 2024年04月24日 16:50 - */ -@Data -@AllArgsConstructor -@NoArgsConstructor -public class ImportTableOption { - private String tableName; - private List srcColumnNames; - private List targetColumnNames; -} diff --git a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/JSONExportDataOption.java b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/JSONExportDataOption.java deleted file mode 100644 index 4b2f5ef77..000000000 --- a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/JSONExportDataOption.java +++ /dev/null @@ -1,23 +0,0 @@ -package ai.chat2db.server.tools.common.model.data.option; - -import jakarta.validation.constraints.NotBlank; -import jakarta.validation.constraints.NotNull; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; - -/** - * @author: zgq - * @date: 2024年04月22日 18:28 - */ -@Data -@AllArgsConstructor -@NoArgsConstructor -public class JSONExportDataOption extends ExportDataOption { - - @NotBlank - private String dataTimeFormat; - @NotNull - private Boolean isTimestamps; - -} diff --git a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/JSONImportDataOption.java b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/JSONImportDataOption.java deleted file mode 100644 index 87e0281e7..000000000 --- a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/JSONImportDataOption.java +++ /dev/null @@ -1,16 +0,0 @@ -package ai.chat2db.server.tools.common.model.data.option; - -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; - -/** - * @author: zgq - * @date: 2024年04月24日 16:55 - */ -@Data -@AllArgsConstructor -@NoArgsConstructor -public class JSONImportDataOption extends ImportDataOption{ - private String rootNodeName; -} diff --git a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/SQLExportDataOption.java b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/SQLExportDataOption.java deleted file mode 100644 index 73845b799..000000000 --- a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/SQLExportDataOption.java +++ /dev/null @@ -1,17 +0,0 @@ -package ai.chat2db.server.tools.common.model.data.option; - -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; - -/** - * @author: zgq - */ -@Data -@AllArgsConstructor -@NoArgsConstructor -public class SQLExportDataOption extends ExportDataOption { - private Boolean multipleRow; - private String newTableName; - private Boolean toUpdate; -} diff --git a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/json/ExportData2JsonOptions.java b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/json/ExportData2JsonOptions.java new file mode 100644 index 000000000..88a2620bb --- /dev/null +++ b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/json/ExportData2JsonOptions.java @@ -0,0 +1,24 @@ +package ai.chat2db.server.tools.common.model.data.option.json; + +import ai.chat2db.server.tools.common.model.data.option.AbstractExportDataOptions; +import cn.hutool.core.date.DatePattern; +import lombok.Data; +import lombok.EqualsAndHashCode; + +/** + * @author: zgq + * @date: 2024年04月26日 0:20 + */ +@Data +@EqualsAndHashCode(callSuper =true) +public class ExportData2JsonOptions extends AbstractExportDataOptions { + + private String dataTimeFormat; + + private Boolean isTimestamps; + + public ExportData2JsonOptions() { + dataTimeFormat = DatePattern.NORM_DATETIME_PATTERN; + isTimestamps = false; + } +} diff --git a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/json/ImportJsonDataOptions.java b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/json/ImportJsonDataOptions.java new file mode 100644 index 000000000..eeda4aa66 --- /dev/null +++ b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/json/ImportJsonDataOptions.java @@ -0,0 +1,22 @@ +package ai.chat2db.server.tools.common.model.data.option.json; + +import ai.chat2db.server.tools.common.model.data.option.BaseImportDataOptions; +import cn.hutool.core.date.DatePattern; +import lombok.Data; +import lombok.EqualsAndHashCode; + +/** + * @author: zgq + * @date: 2024年04月25日 23:08 + */ +@Data +@EqualsAndHashCode(callSuper = true) +public class ImportJsonDataOptions extends BaseImportDataOptions { + private String rootNodeName; + private String dataTimeFormat; + + public ImportJsonDataOptions() { + dataTimeFormat = DatePattern.NORM_DATETIME_PATTERN; + rootNodeName = ""; + } +} diff --git a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/sql/BaseExportData2SqlOptions.java b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/sql/BaseExportData2SqlOptions.java new file mode 100644 index 000000000..0ff86c371 --- /dev/null +++ b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/sql/BaseExportData2SqlOptions.java @@ -0,0 +1,20 @@ +package ai.chat2db.server.tools.common.model.data.option.sql; + +import ai.chat2db.server.tools.common.model.data.option.BaseExportDataOptions; +import jakarta.validation.constraints.NotBlank; +import lombok.Data; +import lombok.EqualsAndHashCode; + +/** + * @author: zgq + * @date: 2024年04月26日 0:36 + */ +@Data +@EqualsAndHashCode(callSuper =true) +public class BaseExportData2SqlOptions extends BaseExportDataOptions { + @NotBlank + public String sqlType; + public BaseExportData2SqlOptions() { + sqlType= "single"; + } +} diff --git a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/table/BaseTableOptions.java b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/table/BaseTableOptions.java new file mode 100644 index 000000000..d78c052a0 --- /dev/null +++ b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/table/BaseTableOptions.java @@ -0,0 +1,21 @@ +package ai.chat2db.server.tools.common.model.data.option.table; + +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotEmpty; +import lombok.Data; + +import java.util.List; + +/** + * @author: zgq + * @date: 2024年04月25日 23:53 + */ +@Data +public class BaseTableOptions implements TableOptionInterface { + @NotBlank + public String tableName; + @NotEmpty + public List tableColumns; + @NotEmpty + public List fileColumns; +} diff --git a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/table/ImportNewTableOptions.java b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/table/ImportNewTableOptions.java new file mode 100644 index 000000000..24915e6f5 --- /dev/null +++ b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/table/ImportNewTableOptions.java @@ -0,0 +1,18 @@ +package ai.chat2db.server.tools.common.model.data.option.table; + +import lombok.Data; +import lombok.EqualsAndHashCode; + +/** + * @author: zgq + * @date: 2024年04月26日 9:32 + */ +@Data +@EqualsAndHashCode(callSuper =true) +public class ImportNewTableOptions extends ImportTableOptions{ + + /* + * create table sql + * */ + private String sql; +} diff --git a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/table/ImportTableOptions.java b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/table/ImportTableOptions.java new file mode 100644 index 000000000..3eb0e896a --- /dev/null +++ b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/table/ImportTableOptions.java @@ -0,0 +1,27 @@ +package ai.chat2db.server.tools.common.model.data.option.table; + +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import lombok.Data; + +/** + * @author: zgq + * @date: 2024年04月26日 9:39 + */ + +@JsonTypeInfo( + use = JsonTypeInfo.Id.NAME, + property = "importType", + visible = true) +@JsonSubTypes({ + @JsonSubTypes.Type(value = ImportTableOptions.class, name = "OLD"), + @JsonSubTypes.Type(value = ImportNewTableOptions.class, name = "NEW"), +}) +@Data +public class ImportTableOptions extends BaseTableOptions { + private String importType; + + public ImportTableOptions() { + importType = "OLD"; + } +} diff --git a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/table/TableOptionInterface.java b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/table/TableOptionInterface.java new file mode 100644 index 000000000..031f97fd3 --- /dev/null +++ b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/table/TableOptionInterface.java @@ -0,0 +1,15 @@ +package ai.chat2db.server.tools.common.model.data.option.table; + +import java.util.List; + +/** + * @author: zgq + * @date: 2024年04月25日 23:49 + */ +public interface TableOptionInterface { + String getTableName(); + + List getTableColumns(); + + List getFileColumns(); +} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/DatabaseController.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/DatabaseController.java index 589590363..1ada93a4a 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/DatabaseController.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/DatabaseController.java @@ -11,8 +11,8 @@ import ai.chat2db.server.web.api.controller.data.source.vo.DatabaseVO; import ai.chat2db.server.web.api.controller.rdb.converter.DatabaseConverter; import ai.chat2db.server.web.api.controller.rdb.converter.RdbWebConverter; -import ai.chat2db.server.web.api.controller.rdb.data.export.strategy.ExportDBDataStrategy; -import ai.chat2db.server.web.api.controller.rdb.factory.ExportDBDataStrategyFactory; +import ai.chat2db.server.web.api.controller.rdb.data.DataFileExporter; +import ai.chat2db.server.web.api.controller.rdb.data.DataFileFactoryProducer; import ai.chat2db.server.web.api.controller.rdb.request.*; import ai.chat2db.server.web.api.controller.rdb.vo.MetaSchemaVO; import ai.chat2db.spi.model.Database; @@ -27,7 +27,6 @@ import org.springframework.web.multipart.MultipartFile; import java.io.PrintWriter; -import java.lang.reflect.Constructor; import java.util.Objects; /** @@ -128,13 +127,17 @@ public void exportDatabase(@Valid @RequestBody DatabaseExportRequest request, Ht @PostMapping("/export_data") public void exportData(@Valid @RequestBody DatabaseExportDataRequest request, HttpServletResponse response) { - Class targetClass = ExportDBDataStrategyFactory.get(request.getExportDataOption().getExportType()); - response.setCharacterEncoding("utf-8"); - DatabaseExportDataParam param = databaseConverter.request2param(request); try { - Constructor constructor = targetClass.getDeclaredConstructor(); - ExportDBDataStrategy service = (ExportDBDataStrategy) constructor.newInstance(); - service.doExport(param, response); + response.setCharacterEncoding("utf-8"); + DatabaseExportDataParam param = databaseConverter.request2param(request); + DataFileExporter exporter = DataFileFactoryProducer + .getFactory(request.getExportDataOption().getFileType()) + .createExporter(); + if (request.getExportTableOptions().size() > 1) { + exporter.createMultiFileExporter().doMultiFileExport(param, response); + } else { + exporter.createSingleFileExporter().doSingleFileExport(param, response); + } } catch (Exception e) { response.reset(); throw new RuntimeException(e); @@ -143,15 +146,13 @@ public void exportData(@Valid @RequestBody DatabaseExportDataRequest request, Ht } @PostMapping("/import_data") - public ActionResult importData(@Valid @ModelAttribute(value = "file") MultipartFile file, @ModelAttribute("json") String json) { + public ActionResult importData(@Valid @ModelAttribute(value = "file") MultipartFile file, @ModelAttribute("json") String jsonData) { try { ObjectMapper objectMapper = new ObjectMapper(); - DatabaseImportDataRequest request = objectMapper.readValue(json, DatabaseImportDataRequest.class); - Class targetClass = ExportDBDataStrategyFactory.get(request.getImportDataOption().getImportType()); + DatabaseImportDataRequest request = objectMapper.readValue(jsonData, DatabaseImportDataRequest.class); DatabaseImportDataParam param = databaseConverter.request2param(request); - Constructor constructor = targetClass.getDeclaredConstructor(); - ExportDBDataStrategy service = (ExportDBDataStrategy) constructor.newInstance(); - service.doImport(param, file); + DataFileFactoryProducer.getFactory(request.getImportDataOption().getFileType()) + .createImporter().importDataFile(param, file); return ActionResult.isSuccess(); } catch (Exception e) { throw new RuntimeException(e); diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/AbstractMultiFileExporter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/AbstractMultiFileExporter.java new file mode 100644 index 000000000..37e363ccb --- /dev/null +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/AbstractMultiFileExporter.java @@ -0,0 +1,63 @@ +package ai.chat2db.server.web.api.controller.rdb.data; + +import ai.chat2db.server.domain.api.enums.ExportFileSuffix; +import ai.chat2db.server.domain.api.param.datasource.DatabaseExportDataParam; +import ai.chat2db.server.tools.common.model.data.option.AbstractExportDataOptions; +import ai.chat2db.server.tools.common.model.data.option.table.BaseTableOptions; +import ai.chat2db.spi.sql.Chat2DBContext; +import jakarta.servlet.ServletOutputStream; +import jakarta.servlet.http.HttpServletResponse; +import lombok.Data; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.sql.Connection; +import java.sql.SQLException; +import java.util.List; +import java.util.Objects; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; + +/** + * @author: zgq + * @date: 2024年04月26日 11:41 + */ +@Data +public abstract class AbstractMultiFileExporter implements MultiFileExporter { + + public String suffix; + + @Override + public void doMultiFileExport(DatabaseExportDataParam param, HttpServletResponse response) throws IOException, SQLException { + String databaseName = param.getDatabaseName(); + String schemaName = param.getSchemaName(); + setResponseHeaders(databaseName, schemaName, response); + ServletOutputStream outputStream = response.getOutputStream(); + ZipOutputStream zipOutputStream = new ZipOutputStream(outputStream); + for (BaseTableOptions exportTableOption : param.getExportTableOptions()) { + String tableName = exportTableOption.getTableName(); + String fileName = tableName + getSuffix(); + List tableColumns = exportTableOption.getTableColumns(); + zipOutputStream.putNextEntry(new ZipEntry(fileName)); + ByteArrayOutputStream byteOut = doTableDataExport(Chat2DBContext.getConnection(), databaseName, schemaName, + tableName, tableColumns, param.getExportDataOption()); + byteOut.writeTo(zipOutputStream); + zipOutputStream.closeEntry(); + byteOut.close(); + } + } + + protected abstract ByteArrayOutputStream doTableDataExport(Connection connection, String databaseName, String schemaName, + String tableName, List tableColumns, + AbstractExportDataOptions exportDataOption) throws SQLException; + + private void setResponseHeaders(String databaseName, String schemaName, HttpServletResponse response) { + response.setContentType("application/zip"); + response.setHeader("Content-disposition", "attachment;filename*=utf-8''" + getFileName(databaseName, schemaName) + ExportFileSuffix.ZIP.getSuffix()); + } + + private String getFileName(String databaseName, String schemaName) { + return Objects.isNull(schemaName) ? databaseName : schemaName; + } + +} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/AbstractSingleFileExporter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/AbstractSingleFileExporter.java new file mode 100644 index 000000000..70b36517c --- /dev/null +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/AbstractSingleFileExporter.java @@ -0,0 +1,46 @@ +package ai.chat2db.server.web.api.controller.rdb.data; + +import ai.chat2db.server.domain.api.param.datasource.DatabaseExportDataParam; +import ai.chat2db.server.tools.common.model.data.option.AbstractExportDataOptions; +import ai.chat2db.server.tools.common.model.data.option.table.BaseTableOptions; +import ai.chat2db.spi.sql.Chat2DBContext; +import jakarta.servlet.http.HttpServletResponse; + +import java.sql.Connection; +import java.sql.SQLException; +import java.util.List; + +/** + * 功能描述 + * + * @author: zgq + * @date: 2024年04月26日 11:39 + */ +public abstract class AbstractSingleFileExporter implements SingleFileExporter { + + public String suffix; + public String contentType; + + + @Override + public void doSingleFileExport(DatabaseExportDataParam param, HttpServletResponse response) throws SQLException { + BaseTableOptions tableOptions = param.getExportTableOptions().get(0); + String tableName = tableOptions.getTableName(); + List tableColumns = tableOptions.getTableColumns(); + String schemaName = param.getSchemaName(); + setResponseHeaders(tableName, response); + doTableDataExport(response, Chat2DBContext.getConnection(), param.getDatabaseName(), + schemaName, tableName, tableColumns, param.getExportDataOption()); + } + + protected abstract void doTableDataExport(HttpServletResponse response, Connection connection, + String databaseName, String schemaName, String tableName, + List tableColumns, AbstractExportDataOptions exportDataOption) throws SQLException; + + private void setResponseHeaders(String tableName, HttpServletResponse response) { + response.setContentType(contentType); + response.setHeader("Content-Disposition", "attachment;filename=" + tableName + suffix); + } + + +} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/BaseFileImporter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/BaseFileImporter.java new file mode 100644 index 000000000..7f14e3172 --- /dev/null +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/BaseFileImporter.java @@ -0,0 +1,33 @@ +package ai.chat2db.server.web.api.controller.rdb.data; + +import ai.chat2db.server.domain.api.param.datasource.DatabaseImportDataParam; +import ai.chat2db.server.tools.common.model.data.option.AbstractImportDataOptions; +import ai.chat2db.server.tools.common.model.data.option.table.ImportTableOptions; +import ai.chat2db.spi.sql.Chat2DBContext; +import org.springframework.web.multipart.MultipartFile; + +import java.io.IOException; +import java.sql.Connection; +import java.util.List; + +/** + * @author: zgq + * @date: 2024年04月26日 22:30 + */ +public abstract class BaseFileImporter implements DataFileImporter { + + @Override + public void importDataFile(DatabaseImportDataParam param, MultipartFile file) throws IOException { + String databaseName = param.getDatabaseName(); + String schemaName = param.getSchemaName(); + ImportTableOptions importTableOption = param.getImportTableOption(); + String tableName = importTableOption.getTableName(); + List tableColumns = importTableOption.getTableColumns(); + List fileColumns = importTableOption.getFileColumns(); + doImportData(Chat2DBContext.getConnection(),databaseName, schemaName, tableName, tableColumns, fileColumns, param.getImportDataOption(), file); + } + + protected abstract void doImportData(Connection connection, String databaseName, String schemaName, String tableName, + List tableColumns, List fileColumns, + AbstractImportDataOptions importDataOption, MultipartFile file) throws IOException; +} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/BaseMultiExcelExporter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/BaseMultiExcelExporter.java new file mode 100644 index 000000000..3d01b053e --- /dev/null +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/BaseMultiExcelExporter.java @@ -0,0 +1,37 @@ +package ai.chat2db.server.web.api.controller.rdb.data; + +import ai.chat2db.server.tools.common.model.data.option.AbstractExportDataOptions; +import ai.chat2db.server.web.api.controller.rdb.data.sql.EasySqlBuilder; +import ai.chat2db.server.web.api.controller.rdb.data.xlsx.EasyExcelExportUtil; +import ai.chat2db.spi.util.ResultSetUtils; +import com.alibaba.excel.support.ExcelTypeEnum; + +import java.io.ByteArrayOutputStream; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.List; + +/** + * @author: zgq + * @date: 2024年04月26日 13:17 + */ +public abstract class BaseMultiExcelExporter extends AbstractMultiFileExporter { + + + @Override + protected ByteArrayOutputStream doTableDataExport(Connection connection, String databaseName, String schemaName, + String tableName, List tableColumns, + AbstractExportDataOptions exportDataOption) throws SQLException { + ByteArrayOutputStream byteOut = new ByteArrayOutputStream(); + String sql = EasySqlBuilder.buildQuerySql(databaseName, schemaName, tableName); + try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { + EasyExcelExportUtil.write(byteOut, EasyExcelExportUtil.getDataList(resultSet, tableColumns), + tableName, ResultSetUtils.getRsHeader(resultSet), + tableColumns, getExcelType(), exportDataOption); + return byteOut; + } + } + + protected abstract ExcelTypeEnum getExcelType(); +} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/BaseSingleExcelExporter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/BaseSingleExcelExporter.java new file mode 100644 index 000000000..1e1759033 --- /dev/null +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/BaseSingleExcelExporter.java @@ -0,0 +1,44 @@ +package ai.chat2db.server.web.api.controller.rdb.data; + +import ai.chat2db.server.tools.common.model.data.option.AbstractExportDataOptions; +import ai.chat2db.server.web.api.controller.rdb.data.sql.EasySqlBuilder; +import ai.chat2db.server.web.api.controller.rdb.data.xlsx.EasyExcelExportUtil; +import ai.chat2db.spi.util.ResultSetUtils; +import com.alibaba.excel.support.ExcelTypeEnum; +import jakarta.servlet.ServletOutputStream; +import jakarta.servlet.http.HttpServletResponse; + +import java.io.IOException; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.List; + +/** + * 功能描述 + * + * @author: zgq + * @date: 2024年04月26日 13:04 + */ +public abstract class BaseSingleExcelExporter extends AbstractSingleFileExporter { + + @Override + protected void doTableDataExport(HttpServletResponse response, Connection connection, String databaseName, + String schemaName, String tableName, List tableColumns, + AbstractExportDataOptions exportDataOption) throws SQLException { + String sql = EasySqlBuilder.buildQuerySql(databaseName, schemaName, tableName); + try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { + ServletOutputStream outputStream = response.getOutputStream(); + EasyExcelExportUtil.write(outputStream, EasyExcelExportUtil.getDataList(resultSet, tableColumns), + tableName, ResultSetUtils.getRsHeader(resultSet), + tableColumns, getExcelType(), exportDataOption); + } catch (IOException e) { + throw new RuntimeException(e); + } + + } + + + protected abstract ExcelTypeEnum getExcelType(); + +} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/DataFileExporter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/DataFileExporter.java new file mode 100644 index 000000000..831594d35 --- /dev/null +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/DataFileExporter.java @@ -0,0 +1,14 @@ +package ai.chat2db.server.web.api.controller.rdb.data; + +/** + * @author: zgq + * @date: 2024年04月26日 10:44 + */ +public interface DataFileExporter { + + + SingleFileExporter createSingleFileExporter(); + + MultiFileExporter createMultiFileExporter(); + +} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/DataFileFactoryProducer.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/DataFileFactoryProducer.java new file mode 100644 index 000000000..e9a08d96b --- /dev/null +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/DataFileFactoryProducer.java @@ -0,0 +1,30 @@ +package ai.chat2db.server.web.api.controller.rdb.data; + +import ai.chat2db.server.web.api.controller.rdb.data.csv.CSVImportExportFactory; +import ai.chat2db.server.web.api.controller.rdb.data.json.JSONImportExportFactory; +import ai.chat2db.server.web.api.controller.rdb.data.sql.SQLImportExportFactory; +import ai.chat2db.server.web.api.controller.rdb.data.xlsx.XLSXImportExportFactory; + +import java.util.HashMap; +import java.util.Map; + +/** + * @author: zgq + * @date: 2024年04月26日 11:51 + */ +public class DataFileFactoryProducer { + + public static final Map factories = new HashMap<>(); + + static { + factories.put("CSV", new CSVImportExportFactory()); + factories.put("XLSX", new XLSXImportExportFactory()); + factories.put("JSON", new JSONImportExportFactory()); + factories.put("SQL", new SQLImportExportFactory()); + } + + public static DataFileImportExportFactory getFactory(String type) { + return factories.get(type); + } +} + diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/DataFileImportExportFactory.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/DataFileImportExportFactory.java new file mode 100644 index 000000000..2379a1133 --- /dev/null +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/DataFileImportExportFactory.java @@ -0,0 +1,6 @@ +package ai.chat2db.server.web.api.controller.rdb.data; + +public interface DataFileImportExportFactory { + DataFileImporter createImporter(); + DataFileExporter createExporter(); +} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/DataFileImporter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/DataFileImporter.java new file mode 100644 index 000000000..2217e1be0 --- /dev/null +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/DataFileImporter.java @@ -0,0 +1,11 @@ +package ai.chat2db.server.web.api.controller.rdb.data; + +import ai.chat2db.server.domain.api.param.datasource.DatabaseImportDataParam; +import org.springframework.web.multipart.MultipartFile; + +import java.io.IOException; + +public interface DataFileImporter { + + void importDataFile(DatabaseImportDataParam param, MultipartFile file) throws IOException; +} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/MultiFileExporter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/MultiFileExporter.java new file mode 100644 index 000000000..cdd1b4026 --- /dev/null +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/MultiFileExporter.java @@ -0,0 +1,12 @@ +package ai.chat2db.server.web.api.controller.rdb.data; + +import ai.chat2db.server.domain.api.param.datasource.DatabaseExportDataParam; +import jakarta.servlet.http.HttpServletResponse; + +import java.io.IOException; +import java.sql.SQLException; + +public interface MultiFileExporter { + + void doMultiFileExport(DatabaseExportDataParam param, HttpServletResponse response) throws IOException, SQLException; +} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/SingleFileExporter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/SingleFileExporter.java new file mode 100644 index 000000000..ad0e2914e --- /dev/null +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/SingleFileExporter.java @@ -0,0 +1,11 @@ +package ai.chat2db.server.web.api.controller.rdb.data; + +import ai.chat2db.server.domain.api.param.datasource.DatabaseExportDataParam; +import jakarta.servlet.http.HttpServletResponse; + +import java.sql.SQLException; + +public interface SingleFileExporter { + + void doSingleFileExport(DatabaseExportDataParam param, HttpServletResponse response) throws SQLException; +} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/csv/CSVExporter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/csv/CSVExporter.java new file mode 100644 index 000000000..b8b238055 --- /dev/null +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/csv/CSVExporter.java @@ -0,0 +1,29 @@ +package ai.chat2db.server.web.api.controller.rdb.data.csv; + +import ai.chat2db.server.web.api.controller.rdb.data.DataFileExporter; +import ai.chat2db.server.web.api.controller.rdb.data.MultiFileExporter; +import ai.chat2db.server.web.api.controller.rdb.data.SingleFileExporter; + +/** + * @author: zgq + * @date: 2024年04月26日 11:27 + */ +public class CSVExporter implements DataFileExporter { + + + /** + * @return + */ + @Override + public SingleFileExporter createSingleFileExporter() { + return new SingleCSVExporter(); + } + + /** + * @return + */ + @Override + public MultiFileExporter createMultiFileExporter() { + return new MultiCSVExporter(); + } +} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/csv/CSVImportExportFactory.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/csv/CSVImportExportFactory.java new file mode 100644 index 000000000..c48f6f133 --- /dev/null +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/csv/CSVImportExportFactory.java @@ -0,0 +1,27 @@ +package ai.chat2db.server.web.api.controller.rdb.data.csv; + +import ai.chat2db.server.web.api.controller.rdb.data.DataFileExporter; +import ai.chat2db.server.web.api.controller.rdb.data.DataFileImportExportFactory; +import ai.chat2db.server.web.api.controller.rdb.data.DataFileImporter; + +/** + * @author: zgq + * @date: 2024年04月26日 11:21 + */ +public class CSVImportExportFactory implements DataFileImportExportFactory { + /** + * @return + */ + @Override + public DataFileImporter createImporter() { + return new CSVImporter(); + } + + /** + * @return + */ + @Override + public DataFileExporter createExporter() { + return new CSVExporter(); + } +} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/csv/CSVImporter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/csv/CSVImporter.java new file mode 100644 index 000000000..6139e2fac --- /dev/null +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/csv/CSVImporter.java @@ -0,0 +1,15 @@ +package ai.chat2db.server.web.api.controller.rdb.data.csv; + +import ai.chat2db.server.web.api.controller.rdb.data.DataFileImporter; +import ai.chat2db.server.web.api.controller.rdb.data.xlsx.BaseExcelImporter; + +/** + * 功能描述 + * + * @author: zgq + * @date: 2024年04月26日 11:26 + */ +public class CSVImporter extends BaseExcelImporter implements DataFileImporter { + + +} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/csv/MultiCSVExporter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/csv/MultiCSVExporter.java new file mode 100644 index 000000000..d0f879f64 --- /dev/null +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/csv/MultiCSVExporter.java @@ -0,0 +1,23 @@ +package ai.chat2db.server.web.api.controller.rdb.data.csv; + +import ai.chat2db.server.domain.api.enums.ExportFileSuffix; +import ai.chat2db.server.web.api.controller.rdb.data.BaseMultiExcelExporter; +import com.alibaba.excel.support.ExcelTypeEnum; + +/** + * @author: zgq + * @date: 2024年04月26日 11:29 + */ +public class MultiCSVExporter extends BaseMultiExcelExporter { + + public MultiCSVExporter() { + suffix = ExportFileSuffix.CSV.getSuffix(); + } + + + @Override + protected ExcelTypeEnum getExcelType() { + return ExcelTypeEnum.CSV; + + } +} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/csv/SingleCSVExporter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/csv/SingleCSVExporter.java new file mode 100644 index 000000000..374b8af0f --- /dev/null +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/csv/SingleCSVExporter.java @@ -0,0 +1,28 @@ +package ai.chat2db.server.web.api.controller.rdb.data.csv; + +import ai.chat2db.server.domain.api.enums.ExportFileSuffix; +import ai.chat2db.server.web.api.controller.rdb.data.BaseSingleExcelExporter; +import ai.chat2db.server.web.api.controller.rdb.data.SingleFileExporter; +import com.alibaba.excel.support.ExcelTypeEnum; + +/** + * 功能描述 + * + * @author: zgq + * @date: 2024年04月26日 11:28 + */ +public class SingleCSVExporter extends BaseSingleExcelExporter implements SingleFileExporter { + + public SingleCSVExporter() { + suffix = ExportFileSuffix.CSV.getSuffix(); + contentType = "text/csv"; + } + + /** + * @return + */ + @Override + protected ExcelTypeEnum getExcelType() { + return ExcelTypeEnum.CSV; + } +} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2CsvStrategy.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2CsvStrategy.java deleted file mode 100644 index e42901b34..000000000 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2CsvStrategy.java +++ /dev/null @@ -1,86 +0,0 @@ -package ai.chat2db.server.web.api.controller.rdb.data.export.strategy; - -import ai.chat2db.server.domain.api.enums.ExportFileSuffix; -import ai.chat2db.server.tools.base.excption.BusinessException; -import ai.chat2db.server.tools.common.model.data.option.CSVImportDataOption; -import ai.chat2db.server.tools.common.model.data.option.ExportDataOption; -import ai.chat2db.server.tools.common.model.data.option.ImportDataOption; -import ai.chat2db.server.tools.common.model.data.option.ImportTableOption; -import ai.chat2db.spi.sql.Chat2DBContext; -import com.alibaba.excel.EasyExcel; -import com.alibaba.excel.exception.ExcelAnalysisException; -import com.alibaba.excel.support.ExcelTypeEnum; -import jakarta.servlet.http.HttpServletResponse; -import lombok.extern.slf4j.Slf4j; -import org.springframework.web.multipart.MultipartFile; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.OutputStream; -import java.nio.charset.StandardCharsets; -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.List; -import java.util.Objects; - -@Slf4j -public class ExportDBData2CsvStrategy extends ExportDBDataStrategy { - - public ExportDBData2CsvStrategy() { - suffix = ExportFileSuffix.CSV.getSuffix(); - contentType = "text/csv"; - } - - @Override - protected ByteArrayOutputStream doTableDataExport(Connection connection, String databaseName, - String schemaName, String tableName, - List filedNames, ExportDataOption options) throws SQLException { - ByteArrayOutputStream byteOut = new ByteArrayOutputStream(); - export2CSV(connection, schemaName, tableName, byteOut, filedNames, options); - return byteOut; - } - - @Override - protected void doTableDataExport(HttpServletResponse response, Connection connection, String databaseName, - String schemaName, String tableName, - List filedNames, ExportDataOption options) throws SQLException { - try { - export2CSV(connection, schemaName, tableName, response.getOutputStream(), filedNames, options); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - - @Override - protected void doTableDataImport(Connection connection, String databaseName, String schemaName, - ImportTableOption importTableOption, - ImportDataOption importDataOption, MultipartFile file) { - NoModelDataListener noModelDataListener = new NoModelDataListener(schemaName, importTableOption, - importDataOption, Chat2DBContext.getConnection()); - try { - Integer headerRowNum = ((CSVImportDataOption) importDataOption).getHeaderRowNum(); - EasyExcel.read(file.getInputStream(), noModelDataListener) - .sheet() - .headRowNumber(headerRowNum) - .doRead(); - } catch (IOException e) { - throw new RuntimeException(e); - } catch (ExcelAnalysisException e) { - if (Objects.equals("超出指定数据行", e.getMessage())) { - log.info("数据读取完成"); - } else { - throw new BusinessException("parse.excel.error", null, e); - } - } - } - - private void export2CSV(Connection connection, String schemaName, String tableName, OutputStream out, List fileNames, ExportDataOption options) throws SQLException { - try (ResultSet resultSet = connection.createStatement().executeQuery(buildQuerySql(schemaName, tableName))) { - write(tableName, out, fileNames, options, resultSet, StandardCharsets.UTF_8, ExcelTypeEnum.CSV); - } - } - - -} \ No newline at end of file diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2ExcelStrategy.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2ExcelStrategy.java deleted file mode 100644 index 2c9b22ce5..000000000 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2ExcelStrategy.java +++ /dev/null @@ -1,71 +0,0 @@ -package ai.chat2db.server.web.api.controller.rdb.data.export.strategy; - -import ai.chat2db.server.domain.api.enums.ExportFileSuffix; -import ai.chat2db.server.tools.common.model.data.option.ExportDataOption; -import ai.chat2db.server.tools.common.model.data.option.ImportDataOption; -import ai.chat2db.server.tools.common.model.data.option.ImportTableOption; -import com.alibaba.excel.support.ExcelTypeEnum; -import jakarta.servlet.http.HttpServletResponse; -import org.springframework.web.multipart.MultipartFile; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.OutputStream; -import java.nio.charset.StandardCharsets; -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.List; - -public class ExportDBData2ExcelStrategy extends ExportDBDataStrategy { - - public ExportDBData2ExcelStrategy() { - suffix = ExportFileSuffix.EXCEL.getSuffix(); - contentType = "application/vnd.ms-excel"; - } - - @Override - protected ByteArrayOutputStream doTableDataExport(Connection connection, String databaseName, - String schemaName, String tableName, - List filedNames, ExportDataOption options) throws SQLException { - ByteArrayOutputStream byteOut = new ByteArrayOutputStream(); - export2EXCEL(connection, schemaName, tableName, byteOut, filedNames, options); - return byteOut; - } - - @Override - protected void doTableDataExport(HttpServletResponse response, Connection connection, String databaseName, - String schemaName, String tableName, - List filedNames, ExportDataOption options) throws SQLException { - try { - export2EXCEL(connection, schemaName, tableName, response.getOutputStream(), filedNames, options); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - /** - * @param tableName - * @param srcColumnNames - * @param targetColumnNames - * @param connection - * @param databaseName - * @param schemaName - * @param importDataOption - * @param file - * @return - */ - @Override - protected void doTableDataImport(Connection connection, String databaseName, String schemaName, - ImportTableOption importTableOption, - ImportDataOption importDataOption, MultipartFile file) { - } - - private void export2EXCEL(Connection connection, String schemaName, String tableName, OutputStream out, List fileNames, ExportDataOption options) throws SQLException { - try (ResultSet resultSet = connection.createStatement().executeQuery(buildQuerySql(schemaName, tableName))) { - write(tableName, out, fileNames, options, resultSet, StandardCharsets.UTF_8, ExcelTypeEnum.XLSX); - } - } - - -} \ No newline at end of file diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2JsonStrategy.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2JsonStrategy.java deleted file mode 100644 index 57bfdbbbd..000000000 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2JsonStrategy.java +++ /dev/null @@ -1,154 +0,0 @@ -package ai.chat2db.server.web.api.controller.rdb.data.export.strategy; - -import ai.chat2db.server.domain.api.enums.ExportFileSuffix; -import ai.chat2db.server.tools.base.excption.BusinessException; -import ai.chat2db.server.tools.common.model.data.option.*; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.SerializationFeature; -import jakarta.servlet.http.HttpServletResponse; -import org.apache.commons.lang3.StringUtils; -import org.jetbrains.annotations.NotNull; -import org.springframework.web.multipart.MultipartFile; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.OutputStreamWriter; -import java.io.PrintWriter; -import java.nio.charset.StandardCharsets; -import java.sql.*; -import java.text.SimpleDateFormat; -import java.util.*; - -public class ExportDBData2JsonStrategy extends ExportDBDataStrategy { - - public ExportDBData2JsonStrategy() { - suffix = ExportFileSuffix.JSON.getSuffix(); - contentType = "application/json"; - } - - @Override - protected ByteArrayOutputStream doTableDataExport(Connection connection, String databaseName, - String schemaName, String tableName, - List filedNames, ExportDataOption options) throws SQLException { - ByteArrayOutputStream byteOut = new ByteArrayOutputStream(); - try (ResultSet resultSet = connection.createStatement().executeQuery(buildQuerySql(schemaName, tableName)); - PrintWriter writer = new PrintWriter(new OutputStreamWriter(byteOut, StandardCharsets.UTF_8))) { - export2JSON(resultSet, writer, filedNames, options); - } - return byteOut; - } - - @Override - protected void doTableDataExport(HttpServletResponse response, Connection connection, String databaseName, - String schemaName, String tableName, - List filedNames, ExportDataOption options) throws SQLException { - try (ResultSet resultSet = connection.createStatement().executeQuery(buildQuerySql(schemaName, tableName))) { - export2JSON(resultSet, response.getWriter(), filedNames, options); - } catch (IOException e) { - throw new RuntimeException(e); - } - - } - - - @Override - protected void doTableDataImport(Connection connection, String databaseName, String schemaName, - ImportTableOption importTableOption, - ImportDataOption importDataOption, MultipartFile file) { - String tableName = importTableOption.getTableName(); - List srcColumnNames = importTableOption.getSrcColumnNames(); - List targetColumnNames = importTableOption.getTargetColumnNames(); - List sqlList = new ArrayList<>(); - String rootNodeName = ((JSONImportDataOption) importDataOption).getRootNodeName(); - ObjectMapper objectMapper = new ObjectMapper(); - SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss"); - objectMapper.setDateFormat(dateFormat); - objectMapper.configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, false); - try (Statement statement = connection.createStatement()) { - JsonNode jsonNode = objectMapper.readTree(file.getInputStream()); - if (StringUtils.isNotBlank(rootNodeName) && jsonNode.has(rootNodeName)) { - jsonNode = jsonNode.get(rootNodeName); - } - if (!jsonNode.isArray() || jsonNode.size() <= 0) { - throw new BusinessException("jsonFile.parse.error"); - } - Iterator records = jsonNode.elements(); - while (records.hasNext()) { - JsonNode recordNode = records.next(); - StringBuilder valuesBuilder = new StringBuilder(); - for (int i = 0; i < srcColumnNames.size(); i++) { - String columnName = srcColumnNames.get(i); - JsonNode columnValueNode = recordNode.get(columnName); - String columnValue = (columnValueNode != null) ? columnValueNode.asText() : "NULL"; - valuesBuilder.append((Objects.equals(columnValue, "NULL")) ? columnValue : ("'" + columnValue + "'")); - if (i < srcColumnNames.size() - 1) { - valuesBuilder.append(", "); - } - } - String columns = String.join(", ", targetColumnNames); - String values = valuesBuilder.toString(); - String sql = buildInsertSql(schemaName, tableName, columns, values); - sqlList.add(sql); - if (sqlList.size()>=1000) { - statement.addBatch(String.join("; ", sqlList)); - statement.executeBatch(); - statement.clearBatch(); - sqlList.clear(); - } - } - if (sqlList.size()>0) { - statement.addBatch(String.join("; ", sqlList)); - statement.executeBatch(); - statement.clearBatch(); - sqlList.clear(); - } - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - private String buildInsertSql(String schemaName, String tableName, String columns, String values) { - if (StringUtils.isNotBlank(schemaName)) { - return String.format("INSERT INTO %s.%s (%s) VALUES (%s)", schemaName, tableName, columns, values); - } else { - return String.format("INSERT INTO %s (%s) VALUES (%s)", tableName, columns, values); - } - } - - private void export2JSON(ResultSet resultSet, PrintWriter writer, List filedNames, ExportDataOption options) throws SQLException { - List> data = getDataMap(resultSet, filedNames); - ObjectMapper objectMapper = new ObjectMapper(); - JSONExportDataOption jsonExportDataOption = (JSONExportDataOption) options; - if (!jsonExportDataOption.getIsTimestamps()) { - objectMapper.configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, true); - SimpleDateFormat dateFormat = new SimpleDateFormat(jsonExportDataOption.getDataTimeFormat()); - objectMapper.setDateFormat(dateFormat); - objectMapper.configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, false); - } - try { - String jsonString = objectMapper.writeValueAsString(data); - writer.println(jsonString); - } catch (IOException e) { - throw new BusinessException("data.export2Json.error", data.toArray(), e); - } - } - - @NotNull - private List> getDataMap(ResultSet resultSet, List filedNames) throws SQLException { - ResultSetMetaData metaData = resultSet.getMetaData(); - int columnCount = metaData.getColumnCount(); - List> data = new ArrayList<>(); - while (resultSet.next()) { - Map row = new LinkedHashMap<>(); - for (int i = 1; i <= columnCount; i++) { - if (filedNames != null && !filedNames.contains(metaData.getColumnName(i))) { - continue; - } - row.put(metaData.getColumnName(i), resultSet.getObject(i)); - } - data.add(row); - } - return data; - } -} \ No newline at end of file diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2SqlStrategy.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2SqlStrategy.java deleted file mode 100644 index 90fb3fb80..000000000 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2SqlStrategy.java +++ /dev/null @@ -1,217 +0,0 @@ -package ai.chat2db.server.web.api.controller.rdb.data.export.strategy; - -import ai.chat2db.server.domain.api.enums.ExportFileSuffix; -import ai.chat2db.server.tools.common.model.data.option.ExportDataOption; -import ai.chat2db.server.tools.common.model.data.option.ImportDataOption; -import ai.chat2db.server.tools.common.model.data.option.ImportTableOption; -import ai.chat2db.server.tools.common.model.data.option.SQLExportDataOption; -import ai.chat2db.spi.util.ResultSetUtils; -import jakarta.servlet.http.HttpServletResponse; -import org.apache.commons.lang3.StringUtils; -import org.springframework.web.multipart.MultipartFile; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.OutputStreamWriter; -import java.io.PrintWriter; -import java.nio.charset.StandardCharsets; -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.ResultSetMetaData; -import java.sql.SQLException; -import java.util.List; -import java.util.Objects; -import java.util.stream.Collectors; - -/** - * @author: zgq - * @date: 2024年03月24日 12:50 - */ -public class ExportDBData2SqlStrategy extends ExportDBDataStrategy { - - public ExportDBData2SqlStrategy() { - suffix = ExportFileSuffix.SQL.getSuffix(); - contentType = "text/sql"; - } - - @Override - protected ByteArrayOutputStream doTableDataExport(Connection connection, String databaseName, - String schemaName, String tableName, List filedNames, ExportDataOption options) throws SQLException { - ByteArrayOutputStream byteOut = new ByteArrayOutputStream(); - try (PrintWriter writer = new PrintWriter(new OutputStreamWriter(byteOut, StandardCharsets.UTF_8))) { - writer.println(export2InsertSQL(connection, schemaName, tableName, filedNames, options)); - } - return byteOut; - } - - @Override - protected void doTableDataExport(HttpServletResponse response, Connection connection, String databaseName, - String schemaName, String tableName, List filedNames, - ExportDataOption options) throws SQLException { - try { - response.getWriter().println(export2InsertSQL(connection, schemaName, tableName, filedNames, options)); - } catch (IOException e) { - throw new RuntimeException(e); - } - - } - - - @Override - protected void doTableDataImport(Connection connection, String databaseName, String schemaName, - ImportTableOption importTableOption, - ImportDataOption importDataOption, MultipartFile file) { - - } - - private String export2InsertSQL(Connection connection, String schemaName, - String tableName, List filedNames, ExportDataOption options) throws SQLException { - - StringBuilder sqlBuilder = new StringBuilder(); - try (ResultSet resultSet = connection.createStatement().executeQuery(buildQuerySql(schemaName, tableName))) { - ResultSetMetaData metaData = resultSet.getMetaData(); - List headList = ResultSetUtils.getRsHeader(resultSet); - if (((SQLExportDataOption) options).getToUpdate()) { - buildUpdateSQL(tableName, filedNames, options, sqlBuilder, resultSet, metaData, headList); - } else { - buildInsertSQL(tableName, filedNames, options, sqlBuilder, resultSet, metaData, headList); - } - sqlBuilder.append("\n"); - } - return sqlBuilder.toString(); - } - - private void buildInsertSQL(String tableName, List filedNames, ExportDataOption options, - StringBuilder sqlBuilder, ResultSet resultSet, ResultSetMetaData metaData, - List headList) throws SQLException { - boolean containsHeader = options.getContainsHeader(); - Boolean multipleRow = ((SQLExportDataOption) options).getMultipleRow(); - String newTableName = ((SQLExportDataOption) options).getNewTableName(); - if (multipleRow) { - if (headList.size() != filedNames.size() && StringUtils.isNotBlank(newTableName)) { - buildInsert(newTableName, containsHeader, filedNames, sqlBuilder); - } else if (StringUtils.isNotBlank(newTableName)) { - buildInsert(newTableName, containsHeader, headList, sqlBuilder); - } else if (headList.size() != filedNames.size()) { - buildInsert(tableName, containsHeader, filedNames, sqlBuilder); - } else { - buildInsert(tableName, containsHeader, headList, sqlBuilder); - } - } - - - while (resultSet.next()) { - int filedSeparatorCount = 0; - if (!multipleRow) { - if (headList.size() != filedNames.size() && StringUtils.isNotBlank(newTableName)) { - buildInsert(newTableName, containsHeader, filedNames, sqlBuilder); - } else if (StringUtils.isNotBlank(newTableName)) { - buildInsert(newTableName, containsHeader, headList, sqlBuilder); - } else if (headList.size() != filedNames.size()) { - buildInsert(tableName, containsHeader, filedNames, sqlBuilder); - } else { - buildInsert(tableName, containsHeader, headList, sqlBuilder); - } - } - - for (int i = 1; i <= metaData.getColumnCount(); i++) { - - if (headList.size() != filedNames.size() && !filedNames.contains(metaData.getColumnName(i))) { - continue; - } - String value = resultSet.getString(i); - if (i == 1) { - sqlBuilder.append("("); - } - if (Objects.isNull(value)) { - sqlBuilder.append("NULL"); - } else { - sqlBuilder.append("'").append(value).append("'"); - } - filedSeparatorCount++; - if (headList.size() == filedNames.size()) { - if (i < metaData.getColumnCount()) { - sqlBuilder.append(","); - } - } else { - if (filedSeparatorCount < filedNames.size()) { - sqlBuilder.append(","); - } - } - } - if (multipleRow) { - if (resultSet.isLast()) { - sqlBuilder.append(");\n"); - break; - } else { - sqlBuilder.append("),\n"); - } - } else { - sqlBuilder.append(");\n"); - } - } - } - - private void buildUpdateSQL(String tableName, List fieldNames, ExportDataOption options, - StringBuilder sqlBuilder, ResultSet resultSet, ResultSetMetaData metaData, - List headList) throws SQLException { - - String newTableName = ((SQLExportDataOption) options).getNewTableName(); - while (resultSet.next()) { - - if (StringUtils.isNotBlank(newTableName)) { - buildUpdate(newTableName, sqlBuilder); - } else { - buildUpdate(tableName, sqlBuilder); - } - int fieldSeparatorCount = 0; - for (int i = 1; i <= metaData.getColumnCount(); i++) { - String columnName = metaData.getColumnName(i); - - if (headList.size() != fieldNames.size() && !fieldNames.contains(columnName)) { - continue; - } - String value = resultSet.getString(i); - if (Objects.isNull(value)) { - sqlBuilder.append(columnName).append(" = NULL"); - } else { - sqlBuilder.append(columnName).append(" = '").append(value).append("'"); - } - fieldSeparatorCount++; - if (headList.size() == fieldNames.size()) { - if (i < metaData.getColumnCount()) { - sqlBuilder.append(","); - } - } else { - if (fieldSeparatorCount < fieldNames.size()) { - sqlBuilder.append(","); - } - } - } - sqlBuilder.append(" WHERE id = ").append("'").append(resultSet.getString("id")).append("'").append(";\n"); - } - } - - - private void buildUpdate(String newTableName, StringBuilder sqlBuilder) { - sqlBuilder.append("UPDATE ").append(newTableName).append(" SET "); - } - - - private void buildInsert(String tableName, boolean containsHeader, List filedNames, StringBuilder sqlBuilder) { - sqlBuilder.append("INSERT INTO ").append(tableName); - if (containsHeader) { - sqlBuilder.append(" ").append(convertList2String(filedNames)); - } - sqlBuilder.append(" VALUES "); - } - - - private String convertList2String(List list) { - return list.stream() - .map(s -> "'" + s + "'") - .collect(Collectors.joining(",", "(", ")")); - } - - -} \ No newline at end of file diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBDataStrategy.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBDataStrategy.java deleted file mode 100644 index 1344772a7..000000000 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBDataStrategy.java +++ /dev/null @@ -1,174 +0,0 @@ -package ai.chat2db.server.web.api.controller.rdb.data.export.strategy; - -import ai.chat2db.server.domain.api.enums.ExportFileSuffix; -import ai.chat2db.server.domain.api.param.datasource.DatabaseExportDataParam; -import ai.chat2db.server.domain.api.param.datasource.DatabaseImportDataParam; -import ai.chat2db.server.tools.common.model.data.option.ExportDataOption; -import ai.chat2db.server.tools.common.model.data.option.ExportTableOption; -import ai.chat2db.server.tools.common.model.data.option.ImportDataOption; -import ai.chat2db.server.tools.common.model.data.option.ImportTableOption; -import ai.chat2db.spi.sql.Chat2DBContext; -import ai.chat2db.spi.util.ResultSetUtils; -import com.alibaba.excel.EasyExcel; -import com.alibaba.excel.support.ExcelTypeEnum; -import com.alibaba.excel.write.builder.ExcelWriterSheetBuilder; -import jakarta.servlet.ServletOutputStream; -import jakarta.servlet.http.HttpServletResponse; -import lombok.AllArgsConstructor; -import lombok.Getter; -import lombok.NoArgsConstructor; -import org.jetbrains.annotations.NotNull; -import org.springframework.web.multipart.MultipartFile; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.OutputStream; -import java.nio.charset.Charset; -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.ResultSetMetaData; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Objects; -import java.util.stream.Collectors; -import java.util.zip.ZipEntry; -import java.util.zip.ZipOutputStream; - -/** - * @author: zgq - * @date: 2024年03月24日 12:46 - */ -@Getter -@AllArgsConstructor -@NoArgsConstructor -public abstract class ExportDBDataStrategy { - - public String suffix; - public String contentType; - - - public void doExport(DatabaseExportDataParam param, HttpServletResponse response) { - String databaseName = param.getDatabaseName(); - String schemaName = param.getSchemaName(); - setResponseHeaders(param, response); - try (Connection connection = Chat2DBContext.getConnection()) { - if (param.getExportTableOptions().size() == 1) { - String tableName = param.getExportTableOptions().get(0).getTableName(); - List filedNames = param.getExportTableOptions().get(0).getExportColumnNames(); - doTableDataExport(response, connection, databaseName, schemaName, tableName, filedNames, param.getExportDataOption()); - } else { - doDbDataExport(response, connection, databaseName, schemaName, param.getExportTableOptions(), param.getExportDataOption()); - } - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - public void doImport(DatabaseImportDataParam param, MultipartFile file) { - doTableDataImport(Chat2DBContext.getConnection(), param.getDatabaseName(), param.getSchemaName(), - param.getImportTableOption(), param.getImportDataOption(), file); - - } - - private void doDbDataExport(HttpServletResponse response, Connection connection, String databaseName, - String schemaName, List exportTableOptions, - ExportDataOption options) throws IOException, SQLException { - ServletOutputStream outputStream = response.getOutputStream(); - ZipOutputStream zipOut = new ZipOutputStream(outputStream); - for (ExportTableOption exportTableOption : exportTableOptions) { - String tableName = exportTableOption.getTableName(); - String fileName = tableName + getSuffix(); - List filedNames = exportTableOption.getExportColumnNames(); - zipOut.putNextEntry(new ZipEntry(fileName)); - ByteArrayOutputStream byteOut = doTableDataExport(connection, databaseName, schemaName, tableName, filedNames, options); - byteOut.writeTo(zipOut); - zipOut.closeEntry(); - byteOut.close(); - } - } - - - private void setResponseHeaders(DatabaseExportDataParam param, HttpServletResponse response) { - if (param.getExportTableOptions().size() == 1) { - String tableName = param.getExportTableOptions().get(0).getTableName(); - response.setContentType(contentType); - response.setHeader("Content-disposition", "attachment;filename*=utf-8''" + tableName + suffix); - } else { - response.setContentType("application/zip"); - response.setHeader("Content-disposition", "attachment;filename*=utf-8''" + getFileName(param) + ExportFileSuffix.ZIP.getSuffix()); - } - } - - - protected String getFileName(DatabaseExportDataParam param) { - return Objects.isNull(param.getSchemaName()) ? param.getDatabaseName() : param.getSchemaName(); - } - - public String buildQuerySql(String schemaName, String tableName) { - String sql; - if (Objects.isNull(schemaName)) { - sql = String.format("select * from %s", tableName); - } else { - sql = String.format("select * from %s.%s", schemaName, tableName); - } - return sql; - } - - protected abstract void doTableDataImport(Connection connection, String databaseName, String schemaName, - ImportTableOption importTableOption, - ImportDataOption importDataOption, MultipartFile file); - - protected abstract ByteArrayOutputStream doTableDataExport(Connection connection, String databaseName, - String schemaName, String tableName, - List filedNames, ExportDataOption options) throws SQLException; - - protected abstract void doTableDataExport(HttpServletResponse response, Connection connection, String databaseName, - String schemaName, String tableName, - List filedNames, ExportDataOption options) throws SQLException; - - public void write(String tableName, OutputStream out, List fileNames, ExportDataOption options, - ResultSet resultSet, Charset charset, ExcelTypeEnum type) throws SQLException { - Boolean containsHeader = options.getContainsHeader(); - ExcelWriterSheetBuilder excelWriterSheetBuilder = EasyExcel.write(out) - .charset(charset) - .excelType(type).sheet(tableName); - if (containsHeader) { - List header = ResultSetUtils.getRsHeader(resultSet); - if (fileNames.size() != header.size()) { - excelWriterSheetBuilder.head(getListHeadList(fileNames)); - } else { - excelWriterSheetBuilder.head(getListHeadList(header)); - } - } - excelWriterSheetBuilder.doWrite(getDataList(resultSet, fileNames)); - } - - - @NotNull - public List> getDataList(ResultSet resultSet, List fileNames) throws SQLException { - ResultSetMetaData metaData = resultSet.getMetaData(); - int columnCount = metaData.getColumnCount(); - List> dataList = new ArrayList<>(); - while (resultSet.next()) { - List row = new ArrayList<>(); - for (int i = 1; i <= columnCount; i++) { - if (fileNames.size() != columnCount && !fileNames.contains(metaData.getColumnName(i))) { - continue; - } - row.add(resultSet.getString(i)); - } - dataList.add(row); - } - return dataList; - } - - @NotNull - public List> getListHeadList(List headers) { - return headers - .stream() - .map(Collections::singletonList) - .collect(Collectors.toList()); - } -} \ No newline at end of file diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/json/EasyJsonExportUtil.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/json/EasyJsonExportUtil.java new file mode 100644 index 000000000..fb307966c --- /dev/null +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/json/EasyJsonExportUtil.java @@ -0,0 +1,63 @@ +package ai.chat2db.server.web.api.controller.rdb.data.json; + +import ai.chat2db.server.tools.base.excption.BusinessException; +import ai.chat2db.server.tools.common.model.data.option.json.ExportData2JsonOptions; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; + +import java.io.IOException; +import java.io.PrintWriter; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * @author: zgq + * @date: 2024年04月26日 14:46 + */ +public class EasyJsonExportUtil { + + + public static void write(List tableColumns, ExportData2JsonOptions exportDataOption, ResultSet resultSet, PrintWriter writer) throws SQLException { + List> data = EasyJsonExportUtil.getDataMap(resultSet, tableColumns); + ObjectMapper objectMapper = EasyJsonExportUtil.getObjectMapper(exportDataOption); + try { + String jsonString = objectMapper.writeValueAsString(data); + writer.println(jsonString); + } catch (IOException e) { + throw new BusinessException("data.export2Json.error", data.toArray(), e); + } + } + public static List> getDataMap(ResultSet resultSet, List filedNames) throws SQLException { + ResultSetMetaData metaData = resultSet.getMetaData(); + int columnCount = metaData.getColumnCount(); + List> data = new ArrayList<>(); + while (resultSet.next()) { + Map row = new LinkedHashMap<>(); + for (int i = 1; i <= columnCount; i++) { + if (filedNames != null && !filedNames.contains(metaData.getColumnName(i))) { + continue; + } + row.put(metaData.getColumnName(i), resultSet.getObject(i)); + } + data.add(row); + } + return data; + } + + public static ObjectMapper getObjectMapper(ExportData2JsonOptions jsonExportDataOption) { + ObjectMapper objectMapper = new ObjectMapper(); + if (!jsonExportDataOption.getIsTimestamps()) { + objectMapper.configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, true); + SimpleDateFormat dateFormat = new SimpleDateFormat(jsonExportDataOption.getDataTimeFormat()); + objectMapper.setDateFormat(dateFormat); + objectMapper.configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, false); + } + return objectMapper; + } +} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/json/JSONExporter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/json/JSONExporter.java new file mode 100644 index 000000000..97cd46237 --- /dev/null +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/json/JSONExporter.java @@ -0,0 +1,25 @@ +package ai.chat2db.server.web.api.controller.rdb.data.json; + +import ai.chat2db.server.web.api.controller.rdb.data.DataFileExporter; +import ai.chat2db.server.web.api.controller.rdb.data.MultiFileExporter; +import ai.chat2db.server.web.api.controller.rdb.data.SingleFileExporter; + +/** + * 功能描述 + * + * @author: zgq + * @date: 2024年04月26日 14:20 + */ +public class JSONExporter implements DataFileExporter { + + @Override + public SingleFileExporter createSingleFileExporter() { + return new SingleJSONExporter(); + } + + + @Override + public MultiFileExporter createMultiFileExporter() { + return new MultiJSONExporter(); + } +} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/json/JSONImportExportFactory.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/json/JSONImportExportFactory.java new file mode 100644 index 000000000..db2a63782 --- /dev/null +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/json/JSONImportExportFactory.java @@ -0,0 +1,23 @@ +package ai.chat2db.server.web.api.controller.rdb.data.json; + +import ai.chat2db.server.web.api.controller.rdb.data.DataFileExporter; +import ai.chat2db.server.web.api.controller.rdb.data.DataFileImportExportFactory; +import ai.chat2db.server.web.api.controller.rdb.data.DataFileImporter; + +/** + * @author: zgq + * @date: 2024年04月26日 14:22 + */ +public class JSONImportExportFactory implements DataFileImportExportFactory { + + @Override + public DataFileImporter createImporter() { + return new JSONImporter(); + } + + + @Override + public DataFileExporter createExporter() { + return new JSONExporter(); + } +} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/json/JSONImporter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/json/JSONImporter.java new file mode 100644 index 000000000..336b9d3c8 --- /dev/null +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/json/JSONImporter.java @@ -0,0 +1,125 @@ +package ai.chat2db.server.web.api.controller.rdb.data.json; + +import ai.chat2db.server.tools.base.excption.BusinessException; +import ai.chat2db.server.tools.common.model.data.option.AbstractImportDataOptions; +import ai.chat2db.server.tools.common.model.data.option.json.ImportJsonDataOptions; +import ai.chat2db.server.web.api.controller.rdb.data.BaseFileImporter; +import ai.chat2db.server.web.api.controller.rdb.data.DataFileImporter; +import ai.chat2db.server.web.api.controller.rdb.data.sql.EasySqlBuilder; +import cn.hutool.core.date.DatePattern; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.jetbrains.annotations.NotNull; +import org.springframework.web.multipart.MultipartFile; + +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Objects; + +/** + * @author: zgq + * @date: 2024年04月26日 14:20 + */ +@Slf4j +public class JSONImporter extends BaseFileImporter implements DataFileImporter { + + @Override + protected void doImportData(Connection connection, String databaseName, String schemaName, String tableName, List tableColumns, + List fileColumns, AbstractImportDataOptions importDataOption, MultipartFile file) { + String rootNodeName = ((ImportJsonDataOptions) importDataOption).getRootNodeName(); + String dataTimeFormat = ((ImportJsonDataOptions) importDataOption).getDataTimeFormat(); + Integer dataStartRowNum = ((ImportJsonDataOptions) importDataOption).getDataStartRowNum(); + Integer dataEndRowNum = ((ImportJsonDataOptions) importDataOption).getDataEndRowNum(); + int limitRowSize = dataEndRowNum - dataStartRowNum + 1; + int sqlCount = 0; + int recordCount = 0; + ObjectMapper objectMapper = new ObjectMapper(); + StringBuilder sqlBuilder = new StringBuilder(); + try (Statement statement = connection.createStatement()) { + JsonNode jsonNode = objectMapper.readTree(file.getInputStream()); + jsonNode = getJsonNode(rootNodeName, jsonNode); + Iterator records = jsonNode.elements(); + List values = new ArrayList<>(); + while (records.hasNext() && recordCount++ < limitRowSize) { + JsonNode recordNode = records.next(); + iteratorValues(fileColumns, dataTimeFormat, values, recordNode); + addSql(tableName, fileColumns, sqlBuilder, statement, values); + sqlCount++; + if (sqlCount == 1000) { + executeBatch(statement); + sqlCount = 0; + } + } + if (sqlCount > 0) { + executeBatch(statement); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + private void iteratorValues(List fileColumns, String dataTimeFormat, + List values, JsonNode recordNode) throws Exception { + for (String columnName : fileColumns) { + JsonNode columnValueNode = recordNode.get(columnName); + if (Objects.isNull(columnValueNode)) { + values.add("NULL"); + } else { + String value = columnValueNode.asText(); + if (isValidDate(value, dataTimeFormat)) { + value = formatDate(value, dataTimeFormat); + } + values.add(value); + } + } + } + + @NotNull + private JsonNode getJsonNode(String rootNodeName, JsonNode jsonNode) { + if (StringUtils.isNotBlank(rootNodeName) && jsonNode.has(rootNodeName)) { + jsonNode = jsonNode.get(rootNodeName); + } + if (!jsonNode.isArray() || jsonNode.size() <= 0) { + throw new BusinessException("jsonFile.parse.error"); + } + return jsonNode; + } + + private static boolean isValidDate(String dateString, String dataTimeFormat) { + try { + SimpleDateFormat sdf = new SimpleDateFormat(dataTimeFormat); + sdf.parse(dateString); + return true; + } catch (Exception e) { + return false; + } + } + + private static String formatDate(String dateString, String dataTimeFormat) throws Exception { + SimpleDateFormat inputFormat = new SimpleDateFormat(dataTimeFormat); + SimpleDateFormat outputFormat = new SimpleDateFormat(DatePattern.NORM_DATETIME_PATTERN); + return outputFormat.format(inputFormat.parse(dateString)); + } + + private void executeBatch(Statement statement) throws SQLException { + statement.executeBatch(); + statement.clearBatch(); + } + + private void addSql(String tableName, List fileColumns, StringBuilder sqlBuilder, + Statement statement, List values) throws SQLException { + EasySqlBuilder.buildInsert(tableName, true, fileColumns, sqlBuilder); + EasySqlBuilder.buildInsertValues(values, sqlBuilder); + sqlBuilder.append(";"); + values.clear(); + statement.addBatch(sqlBuilder.toString()); + sqlBuilder.setLength(0); + } +} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/json/MultiJSONExporter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/json/MultiJSONExporter.java new file mode 100644 index 000000000..135ff1b7c --- /dev/null +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/json/MultiJSONExporter.java @@ -0,0 +1,42 @@ +package ai.chat2db.server.web.api.controller.rdb.data.json; + +import ai.chat2db.server.domain.api.enums.ExportFileSuffix; +import ai.chat2db.server.tools.common.model.data.option.AbstractExportDataOptions; +import ai.chat2db.server.tools.common.model.data.option.json.ExportData2JsonOptions; +import ai.chat2db.server.web.api.controller.rdb.data.AbstractMultiFileExporter; +import ai.chat2db.server.web.api.controller.rdb.data.MultiFileExporter; +import ai.chat2db.server.web.api.controller.rdb.data.sql.EasySqlBuilder; + +import java.io.ByteArrayOutputStream; +import java.io.OutputStreamWriter; +import java.io.PrintWriter; +import java.nio.charset.StandardCharsets; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.List; + +/** + * @author: zgq + * @date: 2024年04月26日 14:22 + */ +public class MultiJSONExporter extends AbstractMultiFileExporter implements MultiFileExporter { + + + public MultiJSONExporter() { + suffix = ExportFileSuffix.JSON.getSuffix(); + } + + @Override + protected ByteArrayOutputStream doTableDataExport(Connection connection, String databaseName, String schemaName, + String tableName, List tableColumns, + AbstractExportDataOptions exportDataOption) throws SQLException { + ByteArrayOutputStream byteOut = new ByteArrayOutputStream(); + String sql = EasySqlBuilder.buildQuerySql(databaseName, schemaName, tableName); + try (ResultSet resultSet = connection.createStatement().executeQuery(sql); + PrintWriter writer = new PrintWriter(new OutputStreamWriter(byteOut, StandardCharsets.UTF_8))) { + EasyJsonExportUtil.write(tableColumns, (ExportData2JsonOptions) exportDataOption, resultSet, writer); + } + return byteOut; + } +} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/json/SingleJSONExporter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/json/SingleJSONExporter.java new file mode 100644 index 000000000..f9df8f83f --- /dev/null +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/json/SingleJSONExporter.java @@ -0,0 +1,42 @@ +package ai.chat2db.server.web.api.controller.rdb.data.json; + +import ai.chat2db.server.domain.api.enums.ExportFileSuffix; +import ai.chat2db.server.tools.common.model.data.option.AbstractExportDataOptions; +import ai.chat2db.server.tools.common.model.data.option.json.ExportData2JsonOptions; +import ai.chat2db.server.web.api.controller.rdb.data.AbstractSingleFileExporter; +import ai.chat2db.server.web.api.controller.rdb.data.sql.EasySqlBuilder; +import ai.chat2db.server.web.api.controller.rdb.data.SingleFileExporter; +import jakarta.servlet.http.HttpServletResponse; + +import java.io.IOException; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.List; + +/** + * 功能描述 + * + * @author: zgq + * @date: 2024年04月26日 14:21 + */ +public class SingleJSONExporter extends AbstractSingleFileExporter implements SingleFileExporter { + + public SingleJSONExporter() { + suffix = ExportFileSuffix.JSON.getSuffix(); + contentType = "application/json"; + } + + @Override + protected void doTableDataExport(HttpServletResponse response, Connection connection, String databaseName, + String schemaName, String tableName, List tableColumns, + AbstractExportDataOptions exportDataOption) throws SQLException { + String sql = EasySqlBuilder.buildQuerySql(databaseName, schemaName, tableName); + try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { + EasyJsonExportUtil.write(tableColumns, (ExportData2JsonOptions) exportDataOption, resultSet, response.getWriter()); + } catch (IOException e) { + throw new RuntimeException(e); + } + + } +} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/sql/EasySqlBuilder.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/sql/EasySqlBuilder.java new file mode 100644 index 000000000..08a3b924d --- /dev/null +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/sql/EasySqlBuilder.java @@ -0,0 +1,183 @@ +package ai.chat2db.server.web.api.controller.rdb.data.sql; + +import ai.chat2db.server.tools.common.model.data.option.AbstractExportDataOptions; +import ai.chat2db.server.tools.common.model.data.option.sql.BaseExportData2SqlOptions; +import ai.chat2db.spi.util.ResultSetUtils; +import org.apache.commons.lang3.StringUtils; + +import java.io.PrintWriter; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; + +/** + * @author: zgq + * @date: 2024年04月26日 12:46 + */ +public class EasySqlBuilder { + + public static String buildQuerySql(String databaseName, String schemaName, String tableName) { + StringBuilder sqlBuilder = new StringBuilder("SELECT * FROM "); + buildTableName(databaseName, schemaName, tableName, sqlBuilder); + return sqlBuilder.toString(); + } + + private static void buildTableName(String databaseName, String schemaName, String tableName, StringBuilder sqlBuilder) { + if (StringUtils.isNotBlank(databaseName)) { + sqlBuilder.append(databaseName).append("."); + } + + if (StringUtils.isNotBlank(schemaName)) { + sqlBuilder.append(schemaName).append("."); + } + + if (StringUtils.isNotBlank(tableName)) { + sqlBuilder.append(tableName); + } + } + + public static void exportData2Sql(String tableName, List tableColumns, AbstractExportDataOptions exportDataOption, + ResultSet resultSet, PrintWriter writer) throws SQLException { + ResultSetMetaData metaData = resultSet.getMetaData(); + List headList = ResultSetUtils.getRsHeader(resultSet); + if (tableColumns.size() != headList.size()) { + headList = tableColumns; + } + String sqlType = ((BaseExportData2SqlOptions) exportDataOption).getSqlType(); + switch (sqlType) { + case "single" -> buildSingleInsert(writer, resultSet, metaData, tableName, headList, exportDataOption); + case "multi" -> buildMultiInsert(writer, resultSet, metaData, tableName, headList, exportDataOption); + case "update" -> buildUpdateStatement(writer, resultSet, metaData, tableName, headList); + default -> throw new IllegalStateException("Unexpected value: " + sqlType); + } + } + + public static void buildUpdateStatement(PrintWriter writer, ResultSet resultSet, ResultSetMetaData metaData, + String tableName, List headList) throws SQLException { + StringBuilder sqlBuilder = new StringBuilder(); + while (resultSet.next()) { + EasySqlBuilder.buildUpdate(tableName, sqlBuilder); + int columnCount = 0; + for (int i = 1; i <= metaData.getColumnCount(); i++) { + String columnName = metaData.getColumnName(i); + if (!headList.contains(columnName) || Objects.equals("id", columnName)) { + continue; + } + columnCount++; + String value = resultSet.getString(i); + if (Objects.isNull(value)) { + sqlBuilder.append(columnName).append(" = NULL"); + } else { + sqlBuilder.append(columnName).append(" = '").append(value).append("'"); + } + + if (columnCount < headList.size()-1) { + sqlBuilder.append(", "); + + } + } + EasySqlBuilder.buildUpdateConditions(resultSet.getString("id"), sqlBuilder); + } + writer.println(sqlBuilder); + } + + public static void buildMultiInsert(PrintWriter writer, ResultSet resultSet, ResultSetMetaData metaData, + String tableName, List headList, AbstractExportDataOptions exportDataOption) throws SQLException { + Boolean containsHeader = ((BaseExportData2SqlOptions) exportDataOption).getContainsHeader(); + StringBuilder sqlBuilder = new StringBuilder(); + List values = new ArrayList<>(); + EasySqlBuilder.buildInsert(tableName, containsHeader, headList, sqlBuilder); + while (resultSet.next()) { + int columnCount = 0; + for (int i = 1; i < metaData.getColumnCount(); i++) { + if (!headList.contains(metaData.getColumnName(i))) { + continue; + } + columnCount++; + String value = resultSet.getString(i); + if (Objects.isNull(value)) { + values.add(null); + } else { + values.add(value); + } + if (columnCount == headList.size()) { + break; + } + } + EasySqlBuilder.buildInsertValues(values, sqlBuilder); + if (resultSet.isLast()) { + sqlBuilder.append(";\n"); + break; + } else { + sqlBuilder.append(",\n"); + } + values.clear(); + } + writer.println(sqlBuilder); + } + + public static void buildSingleInsert(PrintWriter writer, ResultSet resultSet, ResultSetMetaData metaData, + String tableName, List headList, AbstractExportDataOptions exportDataOption) throws SQLException { + Boolean containsHeader = ((BaseExportData2SqlOptions) exportDataOption).getContainsHeader(); + StringBuilder sqlBuilder = new StringBuilder(); + List values = new ArrayList<>(); + while (resultSet.next()) { + int columnCount = 0; + for (int i = 1; i < metaData.getColumnCount(); i++) { + if (!headList.contains(metaData.getColumnName(i))) { + continue; + } + columnCount++; + String value = resultSet.getString(i); + if (Objects.isNull(value)) { + values.add(null); + } else { + values.add(value); + } + if (columnCount == headList.size()) { + break; + } + } + EasySqlBuilder.buildInsert(tableName, containsHeader, headList, sqlBuilder); + EasySqlBuilder.buildInsertValues(values, sqlBuilder); + sqlBuilder.append(";\n"); + values.clear(); + } + writer.println(sqlBuilder); + } + + public static void buildInsert(String tableName, boolean containsHeader, List filedNames, StringBuilder sqlBuilder) { + sqlBuilder.append("INSERT INTO ").append(tableName); + if (containsHeader) { + sqlBuilder.append(" ").append(buildColumns(filedNames)); + } + sqlBuilder.append(" VALUES "); + } + + public static void buildInsertValues(List values, StringBuilder sqlBuilder) { + sqlBuilder.append(buildValues(values)); + } + + public static void buildUpdate(String tableName, StringBuilder sqlBuilder) { + sqlBuilder.append("UPDATE ").append(tableName).append(" SET "); + } + + public static void buildUpdateConditions(String value, StringBuilder sqlBuilder) { + sqlBuilder.append(" WHERE id = ").append("'").append(value).append("'").append(";\n"); + } + + public static String buildValues(List list) { + return list.stream() + .map(s -> s == null ? null : "'" + s + "'") + .collect(Collectors.joining(",", "(", ")")); + } + + public static String buildColumns(List headers) { + return headers.stream() + .collect(Collectors.joining(",", "(", ")")); + } +} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/sql/MultiSQLExporter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/sql/MultiSQLExporter.java new file mode 100644 index 000000000..56755966e --- /dev/null +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/sql/MultiSQLExporter.java @@ -0,0 +1,41 @@ +package ai.chat2db.server.web.api.controller.rdb.data.sql; + +import ai.chat2db.server.domain.api.enums.ExportFileSuffix; +import ai.chat2db.server.domain.api.enums.ExportTypeEnum; +import ai.chat2db.server.tools.common.model.data.option.AbstractExportDataOptions; +import ai.chat2db.server.web.api.controller.rdb.data.AbstractMultiFileExporter; +import ai.chat2db.server.web.api.controller.rdb.data.MultiFileExporter; + +import java.io.ByteArrayOutputStream; +import java.io.OutputStreamWriter; +import java.io.PrintWriter; +import java.nio.charset.StandardCharsets; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.List; + +/** + * 功能描述 + * + * @author: zgq + * @date: 2024年04月26日 15:42 + */ +public class MultiSQLExporter extends AbstractMultiFileExporter implements MultiFileExporter { + + public MultiSQLExporter() { + suffix = ExportFileSuffix.SQL.getSuffix(); + } + @Override + protected ByteArrayOutputStream doTableDataExport(Connection connection, String databaseName, String schemaName, + String tableName, List tableColumns, + AbstractExportDataOptions exportDataOption) throws SQLException { + ByteArrayOutputStream byteOut = new ByteArrayOutputStream(); + String sql = EasySqlBuilder.buildQuerySql(databaseName, schemaName, tableName); + try (ResultSet resultSet = connection.createStatement().executeQuery(sql); + PrintWriter writer = new PrintWriter(new OutputStreamWriter(byteOut, StandardCharsets.UTF_8))) { + EasySqlBuilder.exportData2Sql(tableName, tableColumns, exportDataOption, resultSet, writer); + } + return byteOut; + } +} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/sql/SQLExporter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/sql/SQLExporter.java new file mode 100644 index 000000000..59f4330f4 --- /dev/null +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/sql/SQLExporter.java @@ -0,0 +1,27 @@ +package ai.chat2db.server.web.api.controller.rdb.data.sql; + +import ai.chat2db.server.web.api.controller.rdb.data.DataFileExporter; +import ai.chat2db.server.web.api.controller.rdb.data.MultiFileExporter; +import ai.chat2db.server.web.api.controller.rdb.data.SingleFileExporter; + +/** + * @author: zgq + * @date: 2024年04月26日 15:33 + */ +public class SQLExporter implements DataFileExporter { + /** + * @return + */ + @Override + public SingleFileExporter createSingleFileExporter() { + return new SingleSQLExporter(); + } + + /** + * @return + */ + @Override + public MultiFileExporter createMultiFileExporter() { + return new MultiSQLExporter(); + } +} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/sql/SQLImportExportFactory.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/sql/SQLImportExportFactory.java new file mode 100644 index 000000000..a7795ebcb --- /dev/null +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/sql/SQLImportExportFactory.java @@ -0,0 +1,29 @@ +package ai.chat2db.server.web.api.controller.rdb.data.sql; + +import ai.chat2db.server.web.api.controller.rdb.data.DataFileExporter; +import ai.chat2db.server.web.api.controller.rdb.data.DataFileImportExportFactory; +import ai.chat2db.server.web.api.controller.rdb.data.DataFileImporter; + +/** + * 功能描述 + * + * @author: zgq + * @date: 2024年04月26日 15:32 + */ +public class SQLImportExportFactory implements DataFileImportExportFactory { + /** + * @return + */ + @Override + public DataFileImporter createImporter() { + return new SQLImporter(); + } + + /** + * @return + */ + @Override + public DataFileExporter createExporter() { + return new SQLExporter(); + } +} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/sql/SQLImporter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/sql/SQLImporter.java new file mode 100644 index 000000000..6a1f8d613 --- /dev/null +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/sql/SQLImporter.java @@ -0,0 +1,25 @@ +package ai.chat2db.server.web.api.controller.rdb.data.sql; + +import ai.chat2db.server.tools.common.model.data.option.AbstractImportDataOptions; +import ai.chat2db.server.web.api.controller.rdb.data.BaseFileImporter; +import ai.chat2db.server.web.api.controller.rdb.data.DataFileImporter; +import org.springframework.web.multipart.MultipartFile; + +import java.sql.Connection; +import java.util.List; + +/** + * 功能描述 + * + * @author: zgq + * @date: 2024年04月26日 15:33 + */ +public class SQLImporter extends BaseFileImporter implements DataFileImporter { + + @Override + protected void doImportData(Connection connection, String databaseName, String schemaName, String tableName, + List tableColumns, List fileColumns, + AbstractImportDataOptions importDataOption, MultipartFile file) { + + } +} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/sql/SingleSQLExporter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/sql/SingleSQLExporter.java new file mode 100644 index 000000000..db998ec70 --- /dev/null +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/sql/SingleSQLExporter.java @@ -0,0 +1,43 @@ +package ai.chat2db.server.web.api.controller.rdb.data.sql; + +import ai.chat2db.server.domain.api.enums.ExportFileSuffix; +import ai.chat2db.server.tools.common.model.data.option.AbstractExportDataOptions; +import ai.chat2db.server.web.api.controller.rdb.data.AbstractSingleFileExporter; +import ai.chat2db.server.web.api.controller.rdb.data.SingleFileExporter; +import jakarta.servlet.http.HttpServletResponse; + +import java.io.IOException; +import java.io.PrintWriter; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.List; + +/** + * 功能描述 + * + * @author: zgq + * @date: 2024年04月26日 15:41 + */ +public class SingleSQLExporter extends AbstractSingleFileExporter implements SingleFileExporter { + + + public SingleSQLExporter() { + suffix = ExportFileSuffix.CSV.getSuffix(); + contentType = "text/sql"; + } + + @Override + protected void doTableDataExport(HttpServletResponse response, Connection connection, + String databaseName, String schemaName, + String tableName, List tableColumns, + AbstractExportDataOptions exportDataOption) throws SQLException { + String sql = EasySqlBuilder.buildQuerySql(databaseName, schemaName, tableName); + try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { + PrintWriter writer = response.getWriter(); + EasySqlBuilder.exportData2Sql(tableName, tableColumns, exportDataOption, resultSet, writer); + } catch (IOException e) { + throw new RuntimeException(e); + } + } +} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/BaseExcelImporter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/BaseExcelImporter.java new file mode 100644 index 000000000..aeed8b0cf --- /dev/null +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/BaseExcelImporter.java @@ -0,0 +1,31 @@ +package ai.chat2db.server.web.api.controller.rdb.data.xlsx; + +import ai.chat2db.server.tools.common.model.data.option.BaseImportExcelDataOptions; +import ai.chat2db.server.tools.common.model.data.option.AbstractImportDataOptions; +import ai.chat2db.server.web.api.controller.rdb.data.BaseFileImporter; +import com.alibaba.excel.EasyExcel; +import org.springframework.web.multipart.MultipartFile; + +import java.io.IOException; +import java.sql.Connection; +import java.util.List; + +/** + * @author: zgq + * @date: 2024年04月27日 11:16 + */ +public class BaseExcelImporter extends BaseFileImporter { + @Override + protected void doImportData(Connection connection, String databaseName, String schemaName, String tableName, + List tableColumns, List fileColumns, + AbstractImportDataOptions importDataOption, MultipartFile file) throws IOException { + NoModelDataListener noModelDataListener = new NoModelDataListener(schemaName, tableName, tableColumns, + fileColumns, importDataOption, connection); + Integer headerRowNum = ((BaseImportExcelDataOptions) importDataOption).getHeaderRowNum(); + EasyExcel.read(file.getInputStream(), noModelDataListener) + .sheet() + .headRowNumber(headerRowNum) + .doRead(); + + } +} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/EasyExcelExportUtil.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/EasyExcelExportUtil.java new file mode 100644 index 000000000..34884912e --- /dev/null +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/EasyExcelExportUtil.java @@ -0,0 +1,67 @@ +package ai.chat2db.server.web.api.controller.rdb.data.xlsx; + +import ai.chat2db.server.tools.common.model.data.option.AbstractExportDataOptions; +import ai.chat2db.server.tools.common.model.data.option.BaseExportDataOptions; +import ai.chat2db.spi.util.ResultSetUtils; +import com.alibaba.excel.EasyExcel; +import com.alibaba.excel.support.ExcelTypeEnum; +import com.alibaba.excel.write.builder.ExcelWriterSheetBuilder; +import org.jetbrains.annotations.NotNull; + +import java.io.OutputStream; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +/** + * @author: zgq + * @date: 2024年04月26日 13:31 + */ +public class EasyExcelExportUtil { + + public static void write(OutputStream out, List> dataList, String fileName, + List srcHeaders,List targetHeaders ,ExcelTypeEnum type, + AbstractExportDataOptions exportDataOption) { + Boolean containsHeader = ((BaseExportDataOptions) exportDataOption).getContainsHeader(); + ExcelWriterSheetBuilder excelWriterSheetBuilder = EasyExcel.write(out).excelType(type).sheet(fileName); + if (containsHeader) { + if (srcHeaders.size() != targetHeaders.size()) { + excelWriterSheetBuilder.head(getListHeadList(targetHeaders)); + } else { + excelWriterSheetBuilder.head(getListHeadList(srcHeaders)); + } + } + excelWriterSheetBuilder.doWrite(dataList); + + } + + @NotNull + public static List> getListHeadList(List headers) { + return headers + .stream() + .map(Collections::singletonList) + .collect(Collectors.toList()); + } + + @NotNull + public static List> getDataList(ResultSet resultSet, List fileNames) throws SQLException { + ResultSetMetaData metaData = resultSet.getMetaData(); + int columnCount = metaData.getColumnCount(); + List> dataList = new ArrayList<>(); + while (resultSet.next()) { + List row = new ArrayList<>(); + for (int i = 1; i <= columnCount; i++) { + if (fileNames.size() != columnCount && !fileNames.contains(metaData.getColumnName(i))) { + continue; + } + row.add(resultSet.getString(i)); + } + dataList.add(row); + } + return dataList; + } +} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/MultiXLSXExporter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/MultiXLSXExporter.java new file mode 100644 index 000000000..4a2626268 --- /dev/null +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/MultiXLSXExporter.java @@ -0,0 +1,22 @@ +package ai.chat2db.server.web.api.controller.rdb.data.xlsx; + +import ai.chat2db.server.domain.api.enums.ExportFileSuffix; +import ai.chat2db.server.web.api.controller.rdb.data.BaseMultiExcelExporter; +import com.alibaba.excel.support.ExcelTypeEnum; + +/** + * @author: zgq + * @date: 2024年04月26日 14:06 + */ +public class MultiXLSXExporter extends BaseMultiExcelExporter { + + + public MultiXLSXExporter() { + suffix = ExportFileSuffix.EXCEL.getSuffix(); + } + + @Override + protected ExcelTypeEnum getExcelType() { + return ExcelTypeEnum.XLSX; + } +} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/NoModelDataListener.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/NoModelDataListener.java similarity index 51% rename from chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/NoModelDataListener.java rename to chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/NoModelDataListener.java index 2098b0391..acc02e0a5 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/NoModelDataListener.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/NoModelDataListener.java @@ -1,60 +1,77 @@ -package ai.chat2db.server.web.api.controller.rdb.data.export.strategy; +package ai.chat2db.server.web.api.controller.rdb.data.xlsx; -import ai.chat2db.server.tools.common.model.data.option.CSVImportDataOption; -import ai.chat2db.server.tools.common.model.data.option.ImportDataOption; -import ai.chat2db.server.tools.common.model.data.option.ImportTableOption; +import ai.chat2db.server.tools.common.model.data.option.BaseImportExcelDataOptions; +import ai.chat2db.server.tools.common.model.data.option.AbstractImportDataOptions; +import ai.chat2db.server.web.api.controller.rdb.data.sql.EasySqlBuilder; import com.alibaba.excel.context.AnalysisContext; import com.alibaba.excel.event.AnalysisEventListener; -import com.alibaba.excel.exception.ExcelAnalysisException; import com.alibaba.excel.metadata.data.ReadCellData; import com.alibaba.excel.util.ConverterUtils; import com.alibaba.excel.util.ListUtils; -import com.alibaba.fastjson2.JSON; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; import java.sql.Connection; import java.sql.SQLException; import java.sql.Statement; +import java.time.Instant; +import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Objects; @Slf4j public class NoModelDataListener extends AnalysisEventListener> { private final String schemaName; - private final ImportTableOption importTableOption; - private final ImportDataOption importDataOption; + private final String tableName; + private final List tableColumns; + private final List fileColumns; private final Connection connection; private final int BATCH_SIZE = 1000; private final List sqlCacheList = ListUtils.newArrayListWithExpectedSize(BATCH_SIZE); private final List columnIndexList; - private Integer recordIndex = 0; - private final Integer limitRowSize ; + private int recordIndex = 0; + private final int limitRowSize; - public NoModelDataListener(String schemaName, ImportTableOption importTableOption, ImportDataOption importDataOption, Connection connection) { + public NoModelDataListener(String schemaName, String tableName, List tableColumns, + List fileColumns, AbstractImportDataOptions importDataOption, Connection connection) { this.schemaName = schemaName; - this.importTableOption = importTableOption; + this.tableName = tableName; + this.tableColumns = tableColumns; + this.fileColumns = fileColumns; this.connection = connection; - this.importDataOption = importDataOption; - this.columnIndexList = ListUtils.newArrayListWithExpectedSize(importTableOption.getSrcColumnNames().size()); - this.limitRowSize= ((CSVImportDataOption) importDataOption).getDataEndRowNum()- ((CSVImportDataOption) importDataOption).getDataStartRowNum()+1; + this.columnIndexList = ListUtils.newArrayListWithExpectedSize(fileColumns.size()); + this.limitRowSize = ((BaseImportExcelDataOptions) importDataOption).getDataEndRowNum() + - ((BaseImportExcelDataOptions) importDataOption).getDataStartRowNum() + 1; } + @Override + public boolean hasNext(AnalysisContext context) { + if (recordIndex++ >= limitRowSize) { + log.info("read {} records，stopping reading", limitRowSize); + try { + if (sqlCacheList.size() > 0) { + executeBatchInsert(); + } + } catch (SQLException e) { + throw new RuntimeException(e); + } + return false; + } + return super.hasNext(context); + } - /** - * @param headMap - * @param context - */ @Override public void invokeHead(Map> headMap, AnalysisContext context) { Map stringMap = ConverterUtils.convertToStringMap(headMap, context); for (Map.Entry entry : stringMap.entrySet()) { - List srcColumnNames = importTableOption.getSrcColumnNames(); - if (srcColumnNames.contains(entry.getValue())) { - columnIndexList.add(entry.getKey()); + String value = entry.getValue(); + if (fileColumns.contains(value)) { + Integer key = entry.getKey(); + log.info("head index: {} value: {}", key, value); + columnIndexList.add(key); } } } @@ -63,34 +80,23 @@ public void invokeHead(Map> headMap, AnalysisContext co @Override public void invoke(Map data, AnalysisContext context) { try { - if (++recordIndex >= limitRowSize) { - if (sqlCacheList.size() > 0) { - executeBatchInsert(); - } - throw new ExcelAnalysisException("超出指定数据行"); - } - - log.info("解析到一条数据:{}", JSON.toJSONString(data)); - StringBuilder valueBuilder = new StringBuilder(); + List values = new ArrayList<>(); for (int i = 0; i < data.size(); i++) { if (!columnIndexList.contains(i)) { continue; } String value = data.get(i); - if (StringUtils.isBlank(value)) { - value = "NULL"; + if (Objects.isNull(value)) { + values.add(null); } else { - value = "'" + value + "'"; - } - valueBuilder.append(value); - if (i != data.size() - 1 && columnIndexList.contains(i + 1)) { - valueBuilder.append(","); + values.add(value); } } - - String sql = String.format("INSERT INTO %s (%s) VALUES (%s)", importTableOption.getTableName(), getColumns(importTableOption.getTargetColumnNames()), valueBuilder); + String sql = String.format("INSERT INTO %s %s VALUES %s", tableName, + EasySqlBuilder.buildColumns(tableColumns), + EasySqlBuilder.buildValues(values)); + values.clear(); sqlCacheList.add(sql); - if (sqlCacheList.size() >= 1000) { executeBatchInsert(); } @@ -104,18 +110,15 @@ private void executeBatchInsert() throws SQLException { for (String sql : sqlCacheList) { stmt.addBatch(sql); } + log.info("execute batch {}/1000 sql start",sqlCacheList.size()); + Instant startTime = Instant.now(); stmt.executeBatch(); + log.info("execute batch sql success,cost time: {}ms", Instant.now().toEpochMilli() - startTime.toEpochMilli()); stmt.clearBatch(); sqlCacheList.clear(); } } - - private String getColumns(List columnNames) { - return String.join(",", columnNames); - - } - @Override public void doAfterAllAnalysed(AnalysisContext context) { try { diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/SingleXLSXExporter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/SingleXLSXExporter.java new file mode 100644 index 000000000..a5826e774 --- /dev/null +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/SingleXLSXExporter.java @@ -0,0 +1,23 @@ +package ai.chat2db.server.web.api.controller.rdb.data.xlsx; + +import ai.chat2db.server.domain.api.enums.ExportFileSuffix; +import ai.chat2db.server.web.api.controller.rdb.data.BaseSingleExcelExporter; +import com.alibaba.excel.support.ExcelTypeEnum; + +/** + * 功能描述 + * + * @author: zgq + * @date: 2024年04月26日 14:05 + */ +public class SingleXLSXExporter extends BaseSingleExcelExporter { + + public SingleXLSXExporter() { + suffix = ExportFileSuffix.EXCEL.getSuffix(); + contentType = "application/vnd.ms-excel"; + } + @Override + protected ExcelTypeEnum getExcelType() { + return ExcelTypeEnum.XLSX; + } +} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/XLSXExporter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/XLSXExporter.java new file mode 100644 index 000000000..c4fcccea6 --- /dev/null +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/XLSXExporter.java @@ -0,0 +1,24 @@ +package ai.chat2db.server.web.api.controller.rdb.data.xlsx; + +import ai.chat2db.server.web.api.controller.rdb.data.DataFileExporter; +import ai.chat2db.server.web.api.controller.rdb.data.MultiFileExporter; +import ai.chat2db.server.web.api.controller.rdb.data.SingleFileExporter; + +/** + * @author: zgq + * @date: 2024年04月26日 14:05 + */ +public class XLSXExporter implements DataFileExporter { + + + @Override + public SingleFileExporter createSingleFileExporter() { + return new SingleXLSXExporter(); + } + + + @Override + public MultiFileExporter createMultiFileExporter() { + return new MultiXLSXExporter(); + } +} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/XLSXImportExportFactory.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/XLSXImportExportFactory.java new file mode 100644 index 000000000..4031fad51 --- /dev/null +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/XLSXImportExportFactory.java @@ -0,0 +1,23 @@ +package ai.chat2db.server.web.api.controller.rdb.data.xlsx; + +import ai.chat2db.server.web.api.controller.rdb.data.DataFileExporter; +import ai.chat2db.server.web.api.controller.rdb.data.DataFileImportExportFactory; +import ai.chat2db.server.web.api.controller.rdb.data.DataFileImporter; + +/** + * @author: zgq + * @date: 2024年04月26日 14:17 + */ +public class XLSXImportExportFactory implements DataFileImportExportFactory { + + @Override + public DataFileImporter createImporter() { + return new XLSXImporter(); + } + + + @Override + public DataFileExporter createExporter() { + return new XLSXExporter(); + } +} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/XLSXImporter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/XLSXImporter.java new file mode 100644 index 000000000..07a39dd8c --- /dev/null +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/XLSXImporter.java @@ -0,0 +1,12 @@ +package ai.chat2db.server.web.api.controller.rdb.data.xlsx; + +import ai.chat2db.server.web.api.controller.rdb.data.DataFileImporter; + +/** + * @author: zgq + * @date: 2024年04月26日 14:19 + */ +public class XLSXImporter extends BaseExcelImporter implements DataFileImporter { + + +} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/request/DatabaseExportDataRequest.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/request/DatabaseExportDataRequest.java index 21ec60755..5d10cea49 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/request/DatabaseExportDataRequest.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/request/DatabaseExportDataRequest.java @@ -1,7 +1,7 @@ package ai.chat2db.server.web.api.controller.rdb.request; -import ai.chat2db.server.tools.common.model.data.option.ExportDataOption; -import ai.chat2db.server.tools.common.model.data.option.ExportTableOption; +import ai.chat2db.server.tools.common.model.data.option.AbstractExportDataOptions; +import ai.chat2db.server.tools.common.model.data.option.table.BaseTableOptions; import ai.chat2db.server.web.api.controller.data.source.request.DataSourceBaseRequest; import jakarta.validation.constraints.NotNull; import lombok.AllArgsConstructor; @@ -19,7 +19,7 @@ @NoArgsConstructor public class DatabaseExportDataRequest extends DataSourceBaseRequest { @NotNull - private ExportDataOption exportDataOption; - private List exportTableOptions; + private AbstractExportDataOptions exportDataOption; + private List exportTableOptions; } \ No newline at end of file diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/request/DatabaseImportDataRequest.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/request/DatabaseImportDataRequest.java index 87329a688..a9b66fab9 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/request/DatabaseImportDataRequest.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/request/DatabaseImportDataRequest.java @@ -1,7 +1,7 @@ package ai.chat2db.server.web.api.controller.rdb.request; -import ai.chat2db.server.tools.common.model.data.option.ImportDataOption; -import ai.chat2db.server.tools.common.model.data.option.ImportTableOption; +import ai.chat2db.server.tools.common.model.data.option.AbstractImportDataOptions; +import ai.chat2db.server.tools.common.model.data.option.table.ImportTableOptions; import ai.chat2db.server.web.api.controller.data.source.request.DataSourceBaseRequest; import lombok.AllArgsConstructor; import lombok.Data; @@ -15,6 +15,6 @@ @AllArgsConstructor @NoArgsConstructor public class DatabaseImportDataRequest extends DataSourceBaseRequest { - private ImportTableOption importTableOption; - private ImportDataOption importDataOption; + private ImportTableOptions importTableOption; + private AbstractImportDataOptions importDataOption; } From 05c971c4bc0d0a30aae1e659657aa350b504a764 Mon Sep 17 00:00:00 2001 From: zgq <203083679@qq.com> Date: Sat, 27 Apr 2024 16:37:02 +0800 Subject: [PATCH 06/11] Optimize code --- .../datasource/DatabaseExportDataParam.java | 4 +- .../datasource/DatabaseImportDataParam.java | 4 +- .../data/option/AbstractDataOption.java | 2 +- .../option/AbstractDataOptionInterface.java | 2 +- .../option/AbstractExportDataOptions.java | 7 +- .../option/AbstractImportDataOptions.java | 4 +- .../data/option/BaseExportDataOptions.java | 2 +- .../data/option/BaseImportDataOptions.java | 3 +- .../option/BaseImportExcelDataOptions.java | 2 +- .../option/json/ExportData2JsonOptions.java | 4 +- .../option/json/ImportJsonDataOptions.java | 4 +- .../option/sql/BaseExportData2SqlOptions.java | 4 +- .../data/option/table/BaseTableOptions.java | 2 +- .../option/table/ImportNewTableOptions.java | 2 +- .../data/option/table/ImportTableOptions.java | 2 +- .../option/table/TableOptionInterface.java | 2 +- .../controller/rdb/DatabaseController.java | 11 +-- .../api/controller/rdb/TableController.java | 14 +-- .../rdb/data/AbstractDataFileExporter.java | 92 +++++++++++++++++++ ...ter.java => AbstractDataFileImporter.java} | 6 +- .../rdb/data/AbstractMultiFileExporter.java | 63 ------------- .../rdb/data/AbstractSingleFileExporter.java | 46 ---------- .../rdb/data/BaseMultiExcelExporter.java | 37 -------- .../controller/rdb/data/DataFileExporter.java | 11 ++- .../rdb/data/MultiFileExporter.java | 12 --- .../rdb/data/SingleFileExporter.java | 11 --- .../controller/rdb/data/csv/CSVExporter.java | 24 ++--- .../rdb/data/csv/MultiCSVExporter.java | 23 ----- .../rdb/data/csv/SingleCSVExporter.java | 28 ------ .../rdb/data/json/EasyJsonExportUtil.java | 16 ++-- .../rdb/data/json/JSONExporter.java | 49 ++++++++-- .../rdb/data/json/JSONImporter.java | 15 ++- .../rdb/data/json/MultiJSONExporter.java | 42 --------- .../rdb/data/json/SingleJSONExporter.java | 42 --------- .../rdb/data/sql/EasySqlBuilder.java | 30 ++++-- .../rdb/data/sql/MultiSQLExporter.java | 41 --------- .../controller/rdb/data/sql/SQLExporter.java | 55 ++++++++--- .../controller/rdb/data/sql/SQLImporter.java | 6 +- .../rdb/data/sql/SingleSQLExporter.java | 43 --------- .../rdb/data/xlsx/BaseExcelImporter.java | 8 +- .../BaseXLSXExporter.java} | 29 ++++-- .../rdb/data/xlsx/EasyExcelExportUtil.java | 5 +- .../rdb/data/xlsx/MultiXLSXExporter.java | 22 ----- .../rdb/data/xlsx/NoModelDataListener.java | 4 +- .../rdb/data/xlsx/SingleXLSXExporter.java | 23 ----- .../rdb/data/xlsx/XLSXExporter.java | 18 ++-- .../factory/ExportDBDataStrategyFactory.java | 31 ------- .../request/DatabaseExportDataRequest.java | 4 +- .../request/DatabaseImportDataRequest.java | 4 +- 49 files changed, 313 insertions(+), 602 deletions(-) rename chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/{ => rdb}/data/option/AbstractDataOption.java (80%) rename chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/{ => rdb}/data/option/AbstractDataOptionInterface.java (55%) rename chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/{ => rdb}/data/option/AbstractExportDataOptions.java (71%) rename chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/{ => rdb}/data/option/AbstractImportDataOptions.java (82%) rename chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/{ => rdb}/data/option/BaseExportDataOptions.java (87%) rename chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/{ => rdb}/data/option/BaseImportDataOptions.java (74%) rename chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/{ => rdb}/data/option/BaseImportExcelDataOptions.java (84%) rename chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/{ => rdb}/data/option/json/ExportData2JsonOptions.java (75%) rename chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/{ => rdb}/data/option/json/ImportJsonDataOptions.java (75%) rename chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/{ => rdb}/data/option/sql/BaseExportData2SqlOptions.java (72%) rename chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/{ => rdb}/data/option/table/BaseTableOptions.java (86%) rename chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/{ => rdb}/data/option/table/ImportNewTableOptions.java (80%) rename chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/{ => rdb}/data/option/table/ImportTableOptions.java (90%) rename chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/{ => rdb}/data/option/table/TableOptionInterface.java (76%) create mode 100644 chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/AbstractDataFileExporter.java rename chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/{BaseFileImporter.java => AbstractDataFileImporter.java} (84%) delete mode 100644 chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/AbstractMultiFileExporter.java delete mode 100644 chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/AbstractSingleFileExporter.java delete mode 100644 chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/BaseMultiExcelExporter.java delete mode 100644 chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/MultiFileExporter.java delete mode 100644 chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/SingleFileExporter.java delete mode 100644 chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/csv/MultiCSVExporter.java delete mode 100644 chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/csv/SingleCSVExporter.java delete mode 100644 chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/json/MultiJSONExporter.java delete mode 100644 chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/json/SingleJSONExporter.java delete mode 100644 chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/sql/MultiSQLExporter.java delete mode 100644 chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/sql/SingleSQLExporter.java rename chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/{BaseSingleExcelExporter.java => xlsx/BaseXLSXExporter.java} (52%) delete mode 100644 chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/MultiXLSXExporter.java delete mode 100644 chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/SingleXLSXExporter.java delete mode 100644 chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/factory/ExportDBDataStrategyFactory.java diff --git a/chat2db-server/chat2db-server-domain/chat2db-server-domain-api/src/main/java/ai/chat2db/server/domain/api/param/datasource/DatabaseExportDataParam.java b/chat2db-server/chat2db-server-domain/chat2db-server-domain-api/src/main/java/ai/chat2db/server/domain/api/param/datasource/DatabaseExportDataParam.java index 0ec3ed792..fa9a42b96 100644 --- a/chat2db-server/chat2db-server-domain/chat2db-server-domain-api/src/main/java/ai/chat2db/server/domain/api/param/datasource/DatabaseExportDataParam.java +++ b/chat2db-server/chat2db-server-domain/chat2db-server-domain-api/src/main/java/ai/chat2db/server/domain/api/param/datasource/DatabaseExportDataParam.java @@ -1,7 +1,7 @@ package ai.chat2db.server.domain.api.param.datasource; -import ai.chat2db.server.tools.common.model.data.option.AbstractExportDataOptions; -import ai.chat2db.server.tools.common.model.data.option.table.BaseTableOptions; +import ai.chat2db.server.tools.common.model.rdb.data.option.AbstractExportDataOptions; +import ai.chat2db.server.tools.common.model.rdb.data.option.table.BaseTableOptions; import jakarta.validation.constraints.NotNull; import lombok.AllArgsConstructor; import lombok.Data; diff --git a/chat2db-server/chat2db-server-domain/chat2db-server-domain-api/src/main/java/ai/chat2db/server/domain/api/param/datasource/DatabaseImportDataParam.java b/chat2db-server/chat2db-server-domain/chat2db-server-domain-api/src/main/java/ai/chat2db/server/domain/api/param/datasource/DatabaseImportDataParam.java index 69e6bb1b5..c1a577776 100644 --- a/chat2db-server/chat2db-server-domain/chat2db-server-domain-api/src/main/java/ai/chat2db/server/domain/api/param/datasource/DatabaseImportDataParam.java +++ b/chat2db-server/chat2db-server-domain/chat2db-server-domain-api/src/main/java/ai/chat2db/server/domain/api/param/datasource/DatabaseImportDataParam.java @@ -1,7 +1,7 @@ package ai.chat2db.server.domain.api.param.datasource; -import ai.chat2db.server.tools.common.model.data.option.AbstractImportDataOptions; -import ai.chat2db.server.tools.common.model.data.option.table.ImportTableOptions; +import ai.chat2db.server.tools.common.model.rdb.data.option.AbstractImportDataOptions; +import ai.chat2db.server.tools.common.model.rdb.data.option.table.ImportTableOptions; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; diff --git a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/AbstractDataOption.java b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/rdb/data/option/AbstractDataOption.java similarity index 80% rename from chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/AbstractDataOption.java rename to chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/rdb/data/option/AbstractDataOption.java index f3a94baf3..311f00247 100644 --- a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/AbstractDataOption.java +++ b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/rdb/data/option/AbstractDataOption.java @@ -1,4 +1,4 @@ -package ai.chat2db.server.tools.common.model.data.option; +package ai.chat2db.server.tools.common.model.rdb.data.option; import jakarta.validation.constraints.NotNull; import lombok.Data; diff --git a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/AbstractDataOptionInterface.java b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/rdb/data/option/AbstractDataOptionInterface.java similarity index 55% rename from chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/AbstractDataOptionInterface.java rename to chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/rdb/data/option/AbstractDataOptionInterface.java index 1b7467dbb..a8fb79c7c 100644 --- a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/AbstractDataOptionInterface.java +++ b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/rdb/data/option/AbstractDataOptionInterface.java @@ -1,4 +1,4 @@ -package ai.chat2db.server.tools.common.model.data.option; +package ai.chat2db.server.tools.common.model.rdb.data.option; public interface AbstractDataOptionInterface { String getFileType(); diff --git a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/AbstractExportDataOptions.java b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/rdb/data/option/AbstractExportDataOptions.java similarity index 71% rename from chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/AbstractExportDataOptions.java rename to chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/rdb/data/option/AbstractExportDataOptions.java index c6400c4c8..7e3368368 100644 --- a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/AbstractExportDataOptions.java +++ b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/rdb/data/option/AbstractExportDataOptions.java @@ -1,10 +1,9 @@ -package ai.chat2db.server.tools.common.model.data.option; +package ai.chat2db.server.tools.common.model.rdb.data.option; -import ai.chat2db.server.tools.common.model.data.option.json.ExportData2JsonOptions; -import ai.chat2db.server.tools.common.model.data.option.sql.BaseExportData2SqlOptions; +import ai.chat2db.server.tools.common.model.rdb.data.option.json.ExportData2JsonOptions; +import ai.chat2db.server.tools.common.model.rdb.data.option.sql.BaseExportData2SqlOptions; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; -import jakarta.validation.constraints.NotNull; /** * @author: zgq diff --git a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/AbstractImportDataOptions.java b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/rdb/data/option/AbstractImportDataOptions.java similarity index 82% rename from chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/AbstractImportDataOptions.java rename to chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/rdb/data/option/AbstractImportDataOptions.java index 7fc0aef90..a1bc14cd7 100644 --- a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/AbstractImportDataOptions.java +++ b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/rdb/data/option/AbstractImportDataOptions.java @@ -1,6 +1,6 @@ -package ai.chat2db.server.tools.common.model.data.option; +package ai.chat2db.server.tools.common.model.rdb.data.option; -import ai.chat2db.server.tools.common.model.data.option.json.ImportJsonDataOptions; +import ai.chat2db.server.tools.common.model.rdb.data.option.json.ImportJsonDataOptions; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; diff --git a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/BaseExportDataOptions.java b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/rdb/data/option/BaseExportDataOptions.java similarity index 87% rename from chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/BaseExportDataOptions.java rename to chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/rdb/data/option/BaseExportDataOptions.java index 435e272c8..effffbc7f 100644 --- a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/BaseExportDataOptions.java +++ b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/rdb/data/option/BaseExportDataOptions.java @@ -1,4 +1,4 @@ -package ai.chat2db.server.tools.common.model.data.option; +package ai.chat2db.server.tools.common.model.rdb.data.option; import jakarta.validation.constraints.NotNull; import lombok.Data; diff --git a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/BaseImportDataOptions.java b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/rdb/data/option/BaseImportDataOptions.java similarity index 74% rename from chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/BaseImportDataOptions.java rename to chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/rdb/data/option/BaseImportDataOptions.java index 81ca23958..a61f23057 100644 --- a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/BaseImportDataOptions.java +++ b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/rdb/data/option/BaseImportDataOptions.java @@ -1,6 +1,5 @@ -package ai.chat2db.server.tools.common.model.data.option; +package ai.chat2db.server.tools.common.model.rdb.data.option; -import ai.chat2db.server.tools.common.model.data.option.AbstractImportDataOptions; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/BaseImportExcelDataOptions.java b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/rdb/data/option/BaseImportExcelDataOptions.java similarity index 84% rename from chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/BaseImportExcelDataOptions.java rename to chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/rdb/data/option/BaseImportExcelDataOptions.java index 82925130d..ae667d586 100644 --- a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/BaseImportExcelDataOptions.java +++ b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/rdb/data/option/BaseImportExcelDataOptions.java @@ -1,4 +1,4 @@ -package ai.chat2db.server.tools.common.model.data.option; +package ai.chat2db.server.tools.common.model.rdb.data.option; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/json/ExportData2JsonOptions.java b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/rdb/data/option/json/ExportData2JsonOptions.java similarity index 75% rename from chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/json/ExportData2JsonOptions.java rename to chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/rdb/data/option/json/ExportData2JsonOptions.java index 88a2620bb..c35efd2ef 100644 --- a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/json/ExportData2JsonOptions.java +++ b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/rdb/data/option/json/ExportData2JsonOptions.java @@ -1,6 +1,6 @@ -package ai.chat2db.server.tools.common.model.data.option.json; +package ai.chat2db.server.tools.common.model.rdb.data.option.json; -import ai.chat2db.server.tools.common.model.data.option.AbstractExportDataOptions; +import ai.chat2db.server.tools.common.model.rdb.data.option.AbstractExportDataOptions; import cn.hutool.core.date.DatePattern; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/json/ImportJsonDataOptions.java b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/rdb/data/option/json/ImportJsonDataOptions.java similarity index 75% rename from chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/json/ImportJsonDataOptions.java rename to chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/rdb/data/option/json/ImportJsonDataOptions.java index eeda4aa66..5c71c0165 100644 --- a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/json/ImportJsonDataOptions.java +++ b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/rdb/data/option/json/ImportJsonDataOptions.java @@ -1,6 +1,6 @@ -package ai.chat2db.server.tools.common.model.data.option.json; +package ai.chat2db.server.tools.common.model.rdb.data.option.json; -import ai.chat2db.server.tools.common.model.data.option.BaseImportDataOptions; +import ai.chat2db.server.tools.common.model.rdb.data.option.BaseImportDataOptions; import cn.hutool.core.date.DatePattern; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/sql/BaseExportData2SqlOptions.java b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/rdb/data/option/sql/BaseExportData2SqlOptions.java similarity index 72% rename from chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/sql/BaseExportData2SqlOptions.java rename to chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/rdb/data/option/sql/BaseExportData2SqlOptions.java index 0ff86c371..b54e0f5a5 100644 --- a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/sql/BaseExportData2SqlOptions.java +++ b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/rdb/data/option/sql/BaseExportData2SqlOptions.java @@ -1,6 +1,6 @@ -package ai.chat2db.server.tools.common.model.data.option.sql; +package ai.chat2db.server.tools.common.model.rdb.data.option.sql; -import ai.chat2db.server.tools.common.model.data.option.BaseExportDataOptions; +import ai.chat2db.server.tools.common.model.rdb.data.option.BaseExportDataOptions; import jakarta.validation.constraints.NotBlank; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/table/BaseTableOptions.java b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/rdb/data/option/table/BaseTableOptions.java similarity index 86% rename from chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/table/BaseTableOptions.java rename to chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/rdb/data/option/table/BaseTableOptions.java index d78c052a0..9320fac39 100644 --- a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/table/BaseTableOptions.java +++ b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/rdb/data/option/table/BaseTableOptions.java @@ -1,4 +1,4 @@ -package ai.chat2db.server.tools.common.model.data.option.table; +package ai.chat2db.server.tools.common.model.rdb.data.option.table; import jakarta.validation.constraints.NotBlank; import jakarta.validation.constraints.NotEmpty; diff --git a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/table/ImportNewTableOptions.java b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/rdb/data/option/table/ImportNewTableOptions.java similarity index 80% rename from chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/table/ImportNewTableOptions.java rename to chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/rdb/data/option/table/ImportNewTableOptions.java index 24915e6f5..112964441 100644 --- a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/table/ImportNewTableOptions.java +++ b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/rdb/data/option/table/ImportNewTableOptions.java @@ -1,4 +1,4 @@ -package ai.chat2db.server.tools.common.model.data.option.table; +package ai.chat2db.server.tools.common.model.rdb.data.option.table; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/table/ImportTableOptions.java b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/rdb/data/option/table/ImportTableOptions.java similarity index 90% rename from chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/table/ImportTableOptions.java rename to chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/rdb/data/option/table/ImportTableOptions.java index 3eb0e896a..988a25195 100644 --- a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/table/ImportTableOptions.java +++ b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/rdb/data/option/table/ImportTableOptions.java @@ -1,4 +1,4 @@ -package ai.chat2db.server.tools.common.model.data.option.table; +package ai.chat2db.server.tools.common.model.rdb.data.option.table; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; diff --git a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/table/TableOptionInterface.java b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/rdb/data/option/table/TableOptionInterface.java similarity index 76% rename from chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/table/TableOptionInterface.java rename to chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/rdb/data/option/table/TableOptionInterface.java index 031f97fd3..908b4612b 100644 --- a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/data/option/table/TableOptionInterface.java +++ b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/rdb/data/option/table/TableOptionInterface.java @@ -1,4 +1,4 @@ -package ai.chat2db.server.tools.common.model.data.option.table; +package ai.chat2db.server.tools.common.model.rdb.data.option.table; import java.util.List; diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/DatabaseController.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/DatabaseController.java index 1ada93a4a..d8ef479cb 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/DatabaseController.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/DatabaseController.java @@ -11,7 +11,6 @@ import ai.chat2db.server.web.api.controller.data.source.vo.DatabaseVO; import ai.chat2db.server.web.api.controller.rdb.converter.DatabaseConverter; import ai.chat2db.server.web.api.controller.rdb.converter.RdbWebConverter; -import ai.chat2db.server.web.api.controller.rdb.data.DataFileExporter; import ai.chat2db.server.web.api.controller.rdb.data.DataFileFactoryProducer; import ai.chat2db.server.web.api.controller.rdb.request.*; import ai.chat2db.server.web.api.controller.rdb.vo.MetaSchemaVO; @@ -130,14 +129,8 @@ public void exportData(@Valid @RequestBody DatabaseExportDataRequest request, Ht try { response.setCharacterEncoding("utf-8"); DatabaseExportDataParam param = databaseConverter.request2param(request); - DataFileExporter exporter = DataFileFactoryProducer - .getFactory(request.getExportDataOption().getFileType()) - .createExporter(); - if (request.getExportTableOptions().size() > 1) { - exporter.createMultiFileExporter().doMultiFileExport(param, response); - } else { - exporter.createSingleFileExporter().doSingleFileExport(param, response); - } + DataFileFactoryProducer.getFactory(request.getExportDataOption().getFileType()) + .createExporter().exportDataFile(param, response); } catch (Exception e) { response.reset(); throw new RuntimeException(e); diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/TableController.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/TableController.java index 8550ac0a7..095432ab0 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/TableController.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/TableController.java @@ -12,8 +12,6 @@ import ai.chat2db.server.web.api.aspect.ConnectionInfoAspect; import ai.chat2db.server.web.api.controller.ai.EmbeddingController; import ai.chat2db.server.web.api.controller.rdb.converter.RdbWebConverter; -import ai.chat2db.server.web.api.controller.rdb.data.export.strategy.ExportDBDataStrategy; -import ai.chat2db.server.web.api.controller.rdb.factory.ExportDBDataStrategyFactory; import ai.chat2db.server.web.api.controller.rdb.request.*; import ai.chat2db.server.web.api.controller.rdb.vo.ColumnVO; import ai.chat2db.server.web.api.controller.rdb.vo.IndexVO; @@ -21,13 +19,11 @@ import ai.chat2db.server.web.api.controller.rdb.vo.TableVO; import ai.chat2db.spi.model.*; import com.google.common.collect.Lists; -import jakarta.servlet.http.HttpServletResponse; import jakarta.validation.Valid; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.*; -import java.lang.reflect.Constructor; import java.util.List; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -80,7 +76,7 @@ public WebPageResult list(@Valid TableBriefQueryRequest request) { // log.info("sync table vector finish"); // }); return WebPageResult.of(tableVOS, tableDTOPageResult.getTotal(), request.getPageNo(), - request.getPageSize()); + request.getPageSize()); } /** @@ -97,9 +93,6 @@ public ListResult tableList(@Valid TableBriefQueryRequest request) } - - - /** * Query the table columns under the current DB * @@ -205,7 +198,7 @@ public DataResult query(@Valid TableDetailQueryRequest request) { */ @PostMapping("/modify/sql") public ListResult modifySql(@Valid @RequestBody TableModifySqlRequest request) { - Table table = rdbWebConverter.tableRequest2param(request.getNewTable()); + Table table = rdbWebConverter.tableRequest2param(request.getNewTable()); table.setSchemaName(request.getSchemaName()); table.setDatabaseName(request.getDatabaseName()); for (TableColumn tableColumn : table.getColumnList()) { @@ -219,12 +212,11 @@ public ListResult modifySql(@Valid @RequestBody TableModifySqlRequest req tableIndex.setDatabaseName(request.getDatabaseName()); } - return tableService.buildSql(rdbWebConverter.tableRequest2param(request.getOldTable()),table) + return tableService.buildSql(rdbWebConverter.tableRequest2param(request.getOldTable()), table) .map(rdbWebConverter::dto2vo); } - /** * Data types supported by the database * diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/AbstractDataFileExporter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/AbstractDataFileExporter.java new file mode 100644 index 000000000..1393bbff2 --- /dev/null +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/AbstractDataFileExporter.java @@ -0,0 +1,92 @@ +package ai.chat2db.server.web.api.controller.rdb.data; + +import ai.chat2db.server.domain.api.enums.ExportFileSuffix; +import ai.chat2db.server.domain.api.param.datasource.DatabaseExportDataParam; +import ai.chat2db.server.tools.common.model.rdb.data.option.AbstractExportDataOptions; +import ai.chat2db.server.tools.common.model.rdb.data.option.table.BaseTableOptions; +import ai.chat2db.spi.sql.Chat2DBContext; +import jakarta.servlet.ServletOutputStream; +import jakarta.servlet.http.HttpServletResponse; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.sql.Connection; +import java.sql.SQLException; +import java.util.List; +import java.util.Objects; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; + +/** + * @author: zgq + * @date: 2024年04月27日 15:17 + */ +public abstract class AbstractDataFileExporter implements DataFileExporter { + + + public String suffix; + public String contentType; + + + @Override + public void exportDataFile(DatabaseExportDataParam param, HttpServletResponse response) throws IOException, SQLException { + if (param.getExportTableOptions().size() > 1) { + exportMultiDataFile(param, response); + } else { + exportSingleDataFile(param, response); + } + } + + + public void exportSingleDataFile(DatabaseExportDataParam param, HttpServletResponse response) throws SQLException { + BaseTableOptions tableOptions = param.getExportTableOptions().get(0); + String tableName = tableOptions.getTableName(); + List tableColumns = tableOptions.getTableColumns(); + String schemaName = param.getSchemaName(); + setResponseHeaders(tableName, response); + doTableDataExport(response, Chat2DBContext.getConnection(), param.getDatabaseName(), + schemaName, tableName, tableColumns, param.getExportDataOption()); + } + + public void exportMultiDataFile(DatabaseExportDataParam param, HttpServletResponse response) throws IOException, SQLException { + String databaseName = param.getDatabaseName(); + String schemaName = param.getSchemaName(); + setResponseHeaders(databaseName, schemaName, response); + ServletOutputStream outputStream = response.getOutputStream(); + ZipOutputStream zipOutputStream = new ZipOutputStream(outputStream); + for (BaseTableOptions exportTableOption : param.getExportTableOptions()) { + String tableName = exportTableOption.getTableName(); + String fileName = tableName + suffix; + List tableColumns = exportTableOption.getTableColumns(); + zipOutputStream.putNextEntry(new ZipEntry(fileName)); + ByteArrayOutputStream byteOut = doTableDataExport(Chat2DBContext.getConnection(), databaseName, schemaName, + tableName, tableColumns, param.getExportDataOption()); + byteOut.writeTo(zipOutputStream); + zipOutputStream.closeEntry(); + byteOut.close(); + } + } + protected abstract void doTableDataExport(HttpServletResponse response, Connection connection, + String databaseName, String schemaName, String tableName, + List tableColumns, AbstractExportDataOptions exportDataOption) throws SQLException; + + private void setResponseHeaders(String tableName, HttpServletResponse response) { + response.setContentType(contentType); + response.setHeader("Content-Disposition", "attachment;filename=" + tableName + suffix); + } + + + protected abstract ByteArrayOutputStream doTableDataExport(Connection connection, String databaseName, String schemaName, + String tableName, List tableColumns, + AbstractExportDataOptions exportDataOption) throws SQLException; + + private void setResponseHeaders(String databaseName, String schemaName, HttpServletResponse response) { + response.setContentType("application/zip"); + response.setHeader("Content-disposition", "attachment;filename*=utf-8''" + getFileName(databaseName, schemaName) + ExportFileSuffix.ZIP.getSuffix()); + } + + private String getFileName(String databaseName, String schemaName) { + return Objects.isNull(schemaName) ? databaseName : schemaName; + } + +} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/BaseFileImporter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/AbstractDataFileImporter.java similarity index 84% rename from chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/BaseFileImporter.java rename to chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/AbstractDataFileImporter.java index 7f14e3172..efbf89c5d 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/BaseFileImporter.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/AbstractDataFileImporter.java @@ -1,8 +1,8 @@ package ai.chat2db.server.web.api.controller.rdb.data; import ai.chat2db.server.domain.api.param.datasource.DatabaseImportDataParam; -import ai.chat2db.server.tools.common.model.data.option.AbstractImportDataOptions; -import ai.chat2db.server.tools.common.model.data.option.table.ImportTableOptions; +import ai.chat2db.server.tools.common.model.rdb.data.option.AbstractImportDataOptions; +import ai.chat2db.server.tools.common.model.rdb.data.option.table.ImportTableOptions; import ai.chat2db.spi.sql.Chat2DBContext; import org.springframework.web.multipart.MultipartFile; @@ -14,7 +14,7 @@ * @author: zgq * @date: 2024年04月26日 22:30 */ -public abstract class BaseFileImporter implements DataFileImporter { +public abstract class AbstractDataFileImporter implements DataFileImporter { @Override public void importDataFile(DatabaseImportDataParam param, MultipartFile file) throws IOException { diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/AbstractMultiFileExporter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/AbstractMultiFileExporter.java deleted file mode 100644 index 37e363ccb..000000000 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/AbstractMultiFileExporter.java +++ /dev/null @@ -1,63 +0,0 @@ -package ai.chat2db.server.web.api.controller.rdb.data; - -import ai.chat2db.server.domain.api.enums.ExportFileSuffix; -import ai.chat2db.server.domain.api.param.datasource.DatabaseExportDataParam; -import ai.chat2db.server.tools.common.model.data.option.AbstractExportDataOptions; -import ai.chat2db.server.tools.common.model.data.option.table.BaseTableOptions; -import ai.chat2db.spi.sql.Chat2DBContext; -import jakarta.servlet.ServletOutputStream; -import jakarta.servlet.http.HttpServletResponse; -import lombok.Data; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.sql.Connection; -import java.sql.SQLException; -import java.util.List; -import java.util.Objects; -import java.util.zip.ZipEntry; -import java.util.zip.ZipOutputStream; - -/** - * @author: zgq - * @date: 2024年04月26日 11:41 - */ -@Data -public abstract class AbstractMultiFileExporter implements MultiFileExporter { - - public String suffix; - - @Override - public void doMultiFileExport(DatabaseExportDataParam param, HttpServletResponse response) throws IOException, SQLException { - String databaseName = param.getDatabaseName(); - String schemaName = param.getSchemaName(); - setResponseHeaders(databaseName, schemaName, response); - ServletOutputStream outputStream = response.getOutputStream(); - ZipOutputStream zipOutputStream = new ZipOutputStream(outputStream); - for (BaseTableOptions exportTableOption : param.getExportTableOptions()) { - String tableName = exportTableOption.getTableName(); - String fileName = tableName + getSuffix(); - List tableColumns = exportTableOption.getTableColumns(); - zipOutputStream.putNextEntry(new ZipEntry(fileName)); - ByteArrayOutputStream byteOut = doTableDataExport(Chat2DBContext.getConnection(), databaseName, schemaName, - tableName, tableColumns, param.getExportDataOption()); - byteOut.writeTo(zipOutputStream); - zipOutputStream.closeEntry(); - byteOut.close(); - } - } - - protected abstract ByteArrayOutputStream doTableDataExport(Connection connection, String databaseName, String schemaName, - String tableName, List tableColumns, - AbstractExportDataOptions exportDataOption) throws SQLException; - - private void setResponseHeaders(String databaseName, String schemaName, HttpServletResponse response) { - response.setContentType("application/zip"); - response.setHeader("Content-disposition", "attachment;filename*=utf-8''" + getFileName(databaseName, schemaName) + ExportFileSuffix.ZIP.getSuffix()); - } - - private String getFileName(String databaseName, String schemaName) { - return Objects.isNull(schemaName) ? databaseName : schemaName; - } - -} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/AbstractSingleFileExporter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/AbstractSingleFileExporter.java deleted file mode 100644 index 70b36517c..000000000 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/AbstractSingleFileExporter.java +++ /dev/null @@ -1,46 +0,0 @@ -package ai.chat2db.server.web.api.controller.rdb.data; - -import ai.chat2db.server.domain.api.param.datasource.DatabaseExportDataParam; -import ai.chat2db.server.tools.common.model.data.option.AbstractExportDataOptions; -import ai.chat2db.server.tools.common.model.data.option.table.BaseTableOptions; -import ai.chat2db.spi.sql.Chat2DBContext; -import jakarta.servlet.http.HttpServletResponse; - -import java.sql.Connection; -import java.sql.SQLException; -import java.util.List; - -/** - * 功能描述 - * - * @author: zgq - * @date: 2024年04月26日 11:39 - */ -public abstract class AbstractSingleFileExporter implements SingleFileExporter { - - public String suffix; - public String contentType; - - - @Override - public void doSingleFileExport(DatabaseExportDataParam param, HttpServletResponse response) throws SQLException { - BaseTableOptions tableOptions = param.getExportTableOptions().get(0); - String tableName = tableOptions.getTableName(); - List tableColumns = tableOptions.getTableColumns(); - String schemaName = param.getSchemaName(); - setResponseHeaders(tableName, response); - doTableDataExport(response, Chat2DBContext.getConnection(), param.getDatabaseName(), - schemaName, tableName, tableColumns, param.getExportDataOption()); - } - - protected abstract void doTableDataExport(HttpServletResponse response, Connection connection, - String databaseName, String schemaName, String tableName, - List tableColumns, AbstractExportDataOptions exportDataOption) throws SQLException; - - private void setResponseHeaders(String tableName, HttpServletResponse response) { - response.setContentType(contentType); - response.setHeader("Content-Disposition", "attachment;filename=" + tableName + suffix); - } - - -} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/BaseMultiExcelExporter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/BaseMultiExcelExporter.java deleted file mode 100644 index 3d01b053e..000000000 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/BaseMultiExcelExporter.java +++ /dev/null @@ -1,37 +0,0 @@ -package ai.chat2db.server.web.api.controller.rdb.data; - -import ai.chat2db.server.tools.common.model.data.option.AbstractExportDataOptions; -import ai.chat2db.server.web.api.controller.rdb.data.sql.EasySqlBuilder; -import ai.chat2db.server.web.api.controller.rdb.data.xlsx.EasyExcelExportUtil; -import ai.chat2db.spi.util.ResultSetUtils; -import com.alibaba.excel.support.ExcelTypeEnum; - -import java.io.ByteArrayOutputStream; -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.List; - -/** - * @author: zgq - * @date: 2024年04月26日 13:17 - */ -public abstract class BaseMultiExcelExporter extends AbstractMultiFileExporter { - - - @Override - protected ByteArrayOutputStream doTableDataExport(Connection connection, String databaseName, String schemaName, - String tableName, List tableColumns, - AbstractExportDataOptions exportDataOption) throws SQLException { - ByteArrayOutputStream byteOut = new ByteArrayOutputStream(); - String sql = EasySqlBuilder.buildQuerySql(databaseName, schemaName, tableName); - try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { - EasyExcelExportUtil.write(byteOut, EasyExcelExportUtil.getDataList(resultSet, tableColumns), - tableName, ResultSetUtils.getRsHeader(resultSet), - tableColumns, getExcelType(), exportDataOption); - return byteOut; - } - } - - protected abstract ExcelTypeEnum getExcelType(); -} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/DataFileExporter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/DataFileExporter.java index 831594d35..0751f95d4 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/DataFileExporter.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/DataFileExporter.java @@ -1,14 +1,17 @@ package ai.chat2db.server.web.api.controller.rdb.data; +import ai.chat2db.server.domain.api.param.datasource.DatabaseExportDataParam; +import jakarta.servlet.http.HttpServletResponse; + +import java.io.IOException; +import java.sql.SQLException; + /** * @author: zgq * @date: 2024年04月26日 10:44 */ public interface DataFileExporter { - - SingleFileExporter createSingleFileExporter(); - - MultiFileExporter createMultiFileExporter(); + void exportDataFile(DatabaseExportDataParam param, HttpServletResponse response) throws IOException, SQLException; } diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/MultiFileExporter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/MultiFileExporter.java deleted file mode 100644 index cdd1b4026..000000000 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/MultiFileExporter.java +++ /dev/null @@ -1,12 +0,0 @@ -package ai.chat2db.server.web.api.controller.rdb.data; - -import ai.chat2db.server.domain.api.param.datasource.DatabaseExportDataParam; -import jakarta.servlet.http.HttpServletResponse; - -import java.io.IOException; -import java.sql.SQLException; - -public interface MultiFileExporter { - - void doMultiFileExport(DatabaseExportDataParam param, HttpServletResponse response) throws IOException, SQLException; -} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/SingleFileExporter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/SingleFileExporter.java deleted file mode 100644 index ad0e2914e..000000000 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/SingleFileExporter.java +++ /dev/null @@ -1,11 +0,0 @@ -package ai.chat2db.server.web.api.controller.rdb.data; - -import ai.chat2db.server.domain.api.param.datasource.DatabaseExportDataParam; -import jakarta.servlet.http.HttpServletResponse; - -import java.sql.SQLException; - -public interface SingleFileExporter { - - void doSingleFileExport(DatabaseExportDataParam param, HttpServletResponse response) throws SQLException; -} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/csv/CSVExporter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/csv/CSVExporter.java index b8b238055..faea1b36c 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/csv/CSVExporter.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/csv/CSVExporter.java @@ -1,29 +1,23 @@ package ai.chat2db.server.web.api.controller.rdb.data.csv; +import ai.chat2db.server.domain.api.enums.ExportFileSuffix; import ai.chat2db.server.web.api.controller.rdb.data.DataFileExporter; -import ai.chat2db.server.web.api.controller.rdb.data.MultiFileExporter; -import ai.chat2db.server.web.api.controller.rdb.data.SingleFileExporter; +import ai.chat2db.server.web.api.controller.rdb.data.xlsx.BaseXLSXExporter; +import com.alibaba.excel.support.ExcelTypeEnum; /** * @author: zgq * @date: 2024年04月26日 11:27 */ -public class CSVExporter implements DataFileExporter { +public class CSVExporter extends BaseXLSXExporter implements DataFileExporter { - /** - * @return - */ - @Override - public SingleFileExporter createSingleFileExporter() { - return new SingleCSVExporter(); + public CSVExporter() { + suffix = ExportFileSuffix.CSV.getSuffix(); + contentType = "text/csv"; } - - /** - * @return - */ @Override - public MultiFileExporter createMultiFileExporter() { - return new MultiCSVExporter(); + protected ExcelTypeEnum getExcelType() { + return ExcelTypeEnum.CSV; } } diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/csv/MultiCSVExporter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/csv/MultiCSVExporter.java deleted file mode 100644 index d0f879f64..000000000 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/csv/MultiCSVExporter.java +++ /dev/null @@ -1,23 +0,0 @@ -package ai.chat2db.server.web.api.controller.rdb.data.csv; - -import ai.chat2db.server.domain.api.enums.ExportFileSuffix; -import ai.chat2db.server.web.api.controller.rdb.data.BaseMultiExcelExporter; -import com.alibaba.excel.support.ExcelTypeEnum; - -/** - * @author: zgq - * @date: 2024年04月26日 11:29 - */ -public class MultiCSVExporter extends BaseMultiExcelExporter { - - public MultiCSVExporter() { - suffix = ExportFileSuffix.CSV.getSuffix(); - } - - - @Override - protected ExcelTypeEnum getExcelType() { - return ExcelTypeEnum.CSV; - - } -} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/csv/SingleCSVExporter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/csv/SingleCSVExporter.java deleted file mode 100644 index 374b8af0f..000000000 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/csv/SingleCSVExporter.java +++ /dev/null @@ -1,28 +0,0 @@ -package ai.chat2db.server.web.api.controller.rdb.data.csv; - -import ai.chat2db.server.domain.api.enums.ExportFileSuffix; -import ai.chat2db.server.web.api.controller.rdb.data.BaseSingleExcelExporter; -import ai.chat2db.server.web.api.controller.rdb.data.SingleFileExporter; -import com.alibaba.excel.support.ExcelTypeEnum; - -/** - * 功能描述 - * - * @author: zgq - * @date: 2024年04月26日 11:28 - */ -public class SingleCSVExporter extends BaseSingleExcelExporter implements SingleFileExporter { - - public SingleCSVExporter() { - suffix = ExportFileSuffix.CSV.getSuffix(); - contentType = "text/csv"; - } - - /** - * @return - */ - @Override - protected ExcelTypeEnum getExcelType() { - return ExcelTypeEnum.CSV; - } -} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/json/EasyJsonExportUtil.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/json/EasyJsonExportUtil.java index fb307966c..ba395320c 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/json/EasyJsonExportUtil.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/json/EasyJsonExportUtil.java @@ -1,9 +1,10 @@ package ai.chat2db.server.web.api.controller.rdb.data.json; import ai.chat2db.server.tools.base.excption.BusinessException; -import ai.chat2db.server.tools.common.model.data.option.json.ExportData2JsonOptions; +import ai.chat2db.server.tools.common.model.rdb.data.option.json.ExportData2JsonOptions; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.SerializationFeature; +import lombok.extern.slf4j.Slf4j; import java.io.IOException; import java.io.PrintWriter; @@ -20,6 +21,7 @@ * @author: zgq * @date: 2024年04月26日 14:46 */ +@Slf4j public class EasyJsonExportUtil { @@ -40,10 +42,12 @@ public static List> getDataMap(ResultSet resultSet, List row = new LinkedHashMap<>(); for (int i = 1; i <= columnCount; i++) { - if (filedNames != null && !filedNames.contains(metaData.getColumnName(i))) { + String columnName = metaData.getColumnName(i); + if (filedNames != null && !filedNames.contains(columnName)) { + log.info("{} is not in the export field list", columnName); continue; } - row.put(metaData.getColumnName(i), resultSet.getObject(i)); + row.put(columnName, resultSet.getObject(i)); } data.add(row); } @@ -53,9 +57,9 @@ public static List> getDataMap(ResultSet resultSet, List tableColumns, + AbstractExportDataOptions exportDataOption) throws SQLException { + String sql = EasySqlBuilder.buildQuerySql(databaseName, schemaName, tableName); + try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { + EasyJsonExportUtil.write(tableColumns, (ExportData2JsonOptions) exportDataOption, resultSet, response.getWriter()); + } catch (IOException e) { + throw new RuntimeException(e); + } + } @Override - public MultiFileExporter createMultiFileExporter() { - return new MultiJSONExporter(); + protected ByteArrayOutputStream doTableDataExport(Connection connection, String databaseName, String schemaName, + String tableName, List tableColumns, + AbstractExportDataOptions exportDataOption) throws SQLException { + ByteArrayOutputStream byteOut = new ByteArrayOutputStream(); + String sql = EasySqlBuilder.buildQuerySql(databaseName, schemaName, tableName); + try (ResultSet resultSet = connection.createStatement().executeQuery(sql); + PrintWriter writer = new PrintWriter(new OutputStreamWriter(byteOut, StandardCharsets.UTF_8))) { + EasyJsonExportUtil.write(tableColumns, (ExportData2JsonOptions) exportDataOption, resultSet, writer); + } + return byteOut; } } diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/json/JSONImporter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/json/JSONImporter.java index 336b9d3c8..472c2874e 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/json/JSONImporter.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/json/JSONImporter.java @@ -1,9 +1,9 @@ package ai.chat2db.server.web.api.controller.rdb.data.json; import ai.chat2db.server.tools.base.excption.BusinessException; -import ai.chat2db.server.tools.common.model.data.option.AbstractImportDataOptions; -import ai.chat2db.server.tools.common.model.data.option.json.ImportJsonDataOptions; -import ai.chat2db.server.web.api.controller.rdb.data.BaseFileImporter; +import ai.chat2db.server.tools.common.model.rdb.data.option.AbstractImportDataOptions; +import ai.chat2db.server.tools.common.model.rdb.data.option.json.ImportJsonDataOptions; +import ai.chat2db.server.web.api.controller.rdb.data.AbstractDataFileImporter; import ai.chat2db.server.web.api.controller.rdb.data.DataFileImporter; import ai.chat2db.server.web.api.controller.rdb.data.sql.EasySqlBuilder; import cn.hutool.core.date.DatePattern; @@ -18,6 +18,7 @@ import java.sql.SQLException; import java.sql.Statement; import java.text.SimpleDateFormat; +import java.time.Instant; import java.util.ArrayList; import java.util.Iterator; import java.util.List; @@ -28,7 +29,7 @@ * @date: 2024年04月26日 14:20 */ @Slf4j -public class JSONImporter extends BaseFileImporter implements DataFileImporter { +public class JSONImporter extends AbstractDataFileImporter implements DataFileImporter { @Override protected void doImportData(Connection connection, String databaseName, String schemaName, String tableName, List tableColumns, @@ -53,12 +54,18 @@ protected void doImportData(Connection connection, String databaseName, String s addSql(tableName, fileColumns, sqlBuilder, statement, values); sqlCount++; if (sqlCount == 1000) { + log.info("execute batch sqlCount:{}/1000", sqlCount); + Instant startTime = Instant.now(); executeBatch(statement); + log.info("execute batch cost:{}ms", Instant.now().toEpochMilli() - startTime.toEpochMilli()); sqlCount = 0; } } if (sqlCount > 0) { + log.info("execute batch sqlCount:{}/1000", sqlCount); + Instant startTime = Instant.now(); executeBatch(statement); + log.info("execute batch cost:{}ms", Instant.now().toEpochMilli() - startTime.toEpochMilli()); } } catch (Exception e) { throw new RuntimeException(e); diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/json/MultiJSONExporter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/json/MultiJSONExporter.java deleted file mode 100644 index 135ff1b7c..000000000 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/json/MultiJSONExporter.java +++ /dev/null @@ -1,42 +0,0 @@ -package ai.chat2db.server.web.api.controller.rdb.data.json; - -import ai.chat2db.server.domain.api.enums.ExportFileSuffix; -import ai.chat2db.server.tools.common.model.data.option.AbstractExportDataOptions; -import ai.chat2db.server.tools.common.model.data.option.json.ExportData2JsonOptions; -import ai.chat2db.server.web.api.controller.rdb.data.AbstractMultiFileExporter; -import ai.chat2db.server.web.api.controller.rdb.data.MultiFileExporter; -import ai.chat2db.server.web.api.controller.rdb.data.sql.EasySqlBuilder; - -import java.io.ByteArrayOutputStream; -import java.io.OutputStreamWriter; -import java.io.PrintWriter; -import java.nio.charset.StandardCharsets; -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.List; - -/** - * @author: zgq - * @date: 2024年04月26日 14:22 - */ -public class MultiJSONExporter extends AbstractMultiFileExporter implements MultiFileExporter { - - - public MultiJSONExporter() { - suffix = ExportFileSuffix.JSON.getSuffix(); - } - - @Override - protected ByteArrayOutputStream doTableDataExport(Connection connection, String databaseName, String schemaName, - String tableName, List tableColumns, - AbstractExportDataOptions exportDataOption) throws SQLException { - ByteArrayOutputStream byteOut = new ByteArrayOutputStream(); - String sql = EasySqlBuilder.buildQuerySql(databaseName, schemaName, tableName); - try (ResultSet resultSet = connection.createStatement().executeQuery(sql); - PrintWriter writer = new PrintWriter(new OutputStreamWriter(byteOut, StandardCharsets.UTF_8))) { - EasyJsonExportUtil.write(tableColumns, (ExportData2JsonOptions) exportDataOption, resultSet, writer); - } - return byteOut; - } -} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/json/SingleJSONExporter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/json/SingleJSONExporter.java deleted file mode 100644 index f9df8f83f..000000000 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/json/SingleJSONExporter.java +++ /dev/null @@ -1,42 +0,0 @@ -package ai.chat2db.server.web.api.controller.rdb.data.json; - -import ai.chat2db.server.domain.api.enums.ExportFileSuffix; -import ai.chat2db.server.tools.common.model.data.option.AbstractExportDataOptions; -import ai.chat2db.server.tools.common.model.data.option.json.ExportData2JsonOptions; -import ai.chat2db.server.web.api.controller.rdb.data.AbstractSingleFileExporter; -import ai.chat2db.server.web.api.controller.rdb.data.sql.EasySqlBuilder; -import ai.chat2db.server.web.api.controller.rdb.data.SingleFileExporter; -import jakarta.servlet.http.HttpServletResponse; - -import java.io.IOException; -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.List; - -/** - * 功能描述 - * - * @author: zgq - * @date: 2024年04月26日 14:21 - */ -public class SingleJSONExporter extends AbstractSingleFileExporter implements SingleFileExporter { - - public SingleJSONExporter() { - suffix = ExportFileSuffix.JSON.getSuffix(); - contentType = "application/json"; - } - - @Override - protected void doTableDataExport(HttpServletResponse response, Connection connection, String databaseName, - String schemaName, String tableName, List tableColumns, - AbstractExportDataOptions exportDataOption) throws SQLException { - String sql = EasySqlBuilder.buildQuerySql(databaseName, schemaName, tableName); - try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { - EasyJsonExportUtil.write(tableColumns, (ExportData2JsonOptions) exportDataOption, resultSet, response.getWriter()); - } catch (IOException e) { - throw new RuntimeException(e); - } - - } -} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/sql/EasySqlBuilder.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/sql/EasySqlBuilder.java index 08a3b924d..d802b93fa 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/sql/EasySqlBuilder.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/sql/EasySqlBuilder.java @@ -1,8 +1,9 @@ package ai.chat2db.server.web.api.controller.rdb.data.sql; -import ai.chat2db.server.tools.common.model.data.option.AbstractExportDataOptions; -import ai.chat2db.server.tools.common.model.data.option.sql.BaseExportData2SqlOptions; +import ai.chat2db.server.tools.common.model.rdb.data.option.AbstractExportDataOptions; +import ai.chat2db.server.tools.common.model.rdb.data.option.sql.BaseExportData2SqlOptions; import ai.chat2db.spi.util.ResultSetUtils; +import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import java.io.PrintWriter; @@ -18,6 +19,7 @@ * @author: zgq * @date: 2024年04月26日 12:46 */ +@Slf4j public class EasySqlBuilder { public static String buildQuerySql(String databaseName, String schemaName, String tableName) { @@ -49,9 +51,18 @@ public static void exportData2Sql(String tableName, List tableColumns, A } String sqlType = ((BaseExportData2SqlOptions) exportDataOption).getSqlType(); switch (sqlType) { - case "single" -> buildSingleInsert(writer, resultSet, metaData, tableName, headList, exportDataOption); - case "multi" -> buildMultiInsert(writer, resultSet, metaData, tableName, headList, exportDataOption); - case "update" -> buildUpdateStatement(writer, resultSet, metaData, tableName, headList); + case "single" -> { + log.info("Exporting single insert SQL for table: {}", tableName); + buildSingleInsert(writer, resultSet, metaData, tableName, headList, exportDataOption); + } + case "multi" -> { + log.info("Exporting multi insert SQL for table: {}", tableName); + buildMultiInsert(writer, resultSet, metaData, tableName, headList, exportDataOption); + } + case "update" -> { + log.info("Exporting multi update SQL for table: {}", tableName); + buildUpdateStatement(writer, resultSet, metaData, tableName, headList); + } default -> throw new IllegalStateException("Unexpected value: " + sqlType); } } @@ -65,6 +76,7 @@ public static void buildUpdateStatement(PrintWriter writer, ResultSet resultSet, for (int i = 1; i <= metaData.getColumnCount(); i++) { String columnName = metaData.getColumnName(i); if (!headList.contains(columnName) || Objects.equals("id", columnName)) { + log.info("{} is not in the export field list", columnName); continue; } columnCount++; @@ -94,7 +106,9 @@ public static void buildMultiInsert(PrintWriter writer, ResultSet resultSet, Res while (resultSet.next()) { int columnCount = 0; for (int i = 1; i < metaData.getColumnCount(); i++) { - if (!headList.contains(metaData.getColumnName(i))) { + String columnName = metaData.getColumnName(i); + if (!headList.contains(columnName)) { + log.info("{} is not in the export field list", columnName); continue; } columnCount++; @@ -128,7 +142,9 @@ public static void buildSingleInsert(PrintWriter writer, ResultSet resultSet, Re while (resultSet.next()) { int columnCount = 0; for (int i = 1; i < metaData.getColumnCount(); i++) { - if (!headList.contains(metaData.getColumnName(i))) { + String columnName = metaData.getColumnName(i); + if (!headList.contains(columnName)) { + log.info("{} is not in the export field list", columnName); continue; } columnCount++; diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/sql/MultiSQLExporter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/sql/MultiSQLExporter.java deleted file mode 100644 index 56755966e..000000000 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/sql/MultiSQLExporter.java +++ /dev/null @@ -1,41 +0,0 @@ -package ai.chat2db.server.web.api.controller.rdb.data.sql; - -import ai.chat2db.server.domain.api.enums.ExportFileSuffix; -import ai.chat2db.server.domain.api.enums.ExportTypeEnum; -import ai.chat2db.server.tools.common.model.data.option.AbstractExportDataOptions; -import ai.chat2db.server.web.api.controller.rdb.data.AbstractMultiFileExporter; -import ai.chat2db.server.web.api.controller.rdb.data.MultiFileExporter; - -import java.io.ByteArrayOutputStream; -import java.io.OutputStreamWriter; -import java.io.PrintWriter; -import java.nio.charset.StandardCharsets; -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.List; - -/** - * 功能描述 - * - * @author: zgq - * @date: 2024年04月26日 15:42 - */ -public class MultiSQLExporter extends AbstractMultiFileExporter implements MultiFileExporter { - - public MultiSQLExporter() { - suffix = ExportFileSuffix.SQL.getSuffix(); - } - @Override - protected ByteArrayOutputStream doTableDataExport(Connection connection, String databaseName, String schemaName, - String tableName, List tableColumns, - AbstractExportDataOptions exportDataOption) throws SQLException { - ByteArrayOutputStream byteOut = new ByteArrayOutputStream(); - String sql = EasySqlBuilder.buildQuerySql(databaseName, schemaName, tableName); - try (ResultSet resultSet = connection.createStatement().executeQuery(sql); - PrintWriter writer = new PrintWriter(new OutputStreamWriter(byteOut, StandardCharsets.UTF_8))) { - EasySqlBuilder.exportData2Sql(tableName, tableColumns, exportDataOption, resultSet, writer); - } - return byteOut; - } -} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/sql/SQLExporter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/sql/SQLExporter.java index 59f4330f4..a03a185f6 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/sql/SQLExporter.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/sql/SQLExporter.java @@ -1,27 +1,56 @@ package ai.chat2db.server.web.api.controller.rdb.data.sql; +import ai.chat2db.server.domain.api.enums.ExportFileSuffix; +import ai.chat2db.server.tools.common.model.rdb.data.option.AbstractExportDataOptions; +import ai.chat2db.server.web.api.controller.rdb.data.AbstractDataFileExporter; import ai.chat2db.server.web.api.controller.rdb.data.DataFileExporter; -import ai.chat2db.server.web.api.controller.rdb.data.MultiFileExporter; -import ai.chat2db.server.web.api.controller.rdb.data.SingleFileExporter; +import jakarta.servlet.http.HttpServletResponse; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStreamWriter; +import java.io.PrintWriter; +import java.nio.charset.StandardCharsets; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.List; /** * @author: zgq * @date: 2024年04月26日 15:33 */ -public class SQLExporter implements DataFileExporter { - /** - * @return - */ +public class SQLExporter extends AbstractDataFileExporter implements DataFileExporter { + + public SQLExporter() { + suffix = ExportFileSuffix.SQL.getSuffix(); + contentType = "text/sql"; + } + @Override - public SingleFileExporter createSingleFileExporter() { - return new SingleSQLExporter(); + protected void doTableDataExport(HttpServletResponse response, Connection connection, + String databaseName, String schemaName, + String tableName, List tableColumns, + AbstractExportDataOptions exportDataOption) throws SQLException { + String sql = EasySqlBuilder.buildQuerySql(databaseName, schemaName, tableName); + try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { + PrintWriter writer = response.getWriter(); + EasySqlBuilder.exportData2Sql(tableName, tableColumns, exportDataOption, resultSet, writer); + } catch (IOException e) { + throw new RuntimeException(e); + } } - /** - * @return - */ @Override - public MultiFileExporter createMultiFileExporter() { - return new MultiSQLExporter(); + protected ByteArrayOutputStream doTableDataExport(Connection connection, String databaseName, String schemaName, + String tableName, List tableColumns, + AbstractExportDataOptions exportDataOption) throws SQLException { + ByteArrayOutputStream byteOut = new ByteArrayOutputStream(); + String sql = EasySqlBuilder.buildQuerySql(databaseName, schemaName, tableName); + try (ResultSet resultSet = connection.createStatement().executeQuery(sql); + PrintWriter writer = new PrintWriter(new OutputStreamWriter(byteOut, StandardCharsets.UTF_8))) { + EasySqlBuilder.exportData2Sql(tableName, tableColumns, exportDataOption, resultSet, writer); + } + return byteOut; } } diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/sql/SQLImporter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/sql/SQLImporter.java index 6a1f8d613..fd5558b70 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/sql/SQLImporter.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/sql/SQLImporter.java @@ -1,7 +1,7 @@ package ai.chat2db.server.web.api.controller.rdb.data.sql; -import ai.chat2db.server.tools.common.model.data.option.AbstractImportDataOptions; -import ai.chat2db.server.web.api.controller.rdb.data.BaseFileImporter; +import ai.chat2db.server.tools.common.model.rdb.data.option.AbstractImportDataOptions; +import ai.chat2db.server.web.api.controller.rdb.data.AbstractDataFileImporter; import ai.chat2db.server.web.api.controller.rdb.data.DataFileImporter; import org.springframework.web.multipart.MultipartFile; @@ -14,7 +14,7 @@ * @author: zgq * @date: 2024年04月26日 15:33 */ -public class SQLImporter extends BaseFileImporter implements DataFileImporter { +public class SQLImporter extends AbstractDataFileImporter implements DataFileImporter { @Override protected void doImportData(Connection connection, String databaseName, String schemaName, String tableName, diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/sql/SingleSQLExporter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/sql/SingleSQLExporter.java deleted file mode 100644 index db998ec70..000000000 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/sql/SingleSQLExporter.java +++ /dev/null @@ -1,43 +0,0 @@ -package ai.chat2db.server.web.api.controller.rdb.data.sql; - -import ai.chat2db.server.domain.api.enums.ExportFileSuffix; -import ai.chat2db.server.tools.common.model.data.option.AbstractExportDataOptions; -import ai.chat2db.server.web.api.controller.rdb.data.AbstractSingleFileExporter; -import ai.chat2db.server.web.api.controller.rdb.data.SingleFileExporter; -import jakarta.servlet.http.HttpServletResponse; - -import java.io.IOException; -import java.io.PrintWriter; -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.List; - -/** - * 功能描述 - * - * @author: zgq - * @date: 2024年04月26日 15:41 - */ -public class SingleSQLExporter extends AbstractSingleFileExporter implements SingleFileExporter { - - - public SingleSQLExporter() { - suffix = ExportFileSuffix.CSV.getSuffix(); - contentType = "text/sql"; - } - - @Override - protected void doTableDataExport(HttpServletResponse response, Connection connection, - String databaseName, String schemaName, - String tableName, List tableColumns, - AbstractExportDataOptions exportDataOption) throws SQLException { - String sql = EasySqlBuilder.buildQuerySql(databaseName, schemaName, tableName); - try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { - PrintWriter writer = response.getWriter(); - EasySqlBuilder.exportData2Sql(tableName, tableColumns, exportDataOption, resultSet, writer); - } catch (IOException e) { - throw new RuntimeException(e); - } - } -} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/BaseExcelImporter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/BaseExcelImporter.java index aeed8b0cf..4eb18dd6e 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/BaseExcelImporter.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/BaseExcelImporter.java @@ -1,8 +1,8 @@ package ai.chat2db.server.web.api.controller.rdb.data.xlsx; -import ai.chat2db.server.tools.common.model.data.option.BaseImportExcelDataOptions; -import ai.chat2db.server.tools.common.model.data.option.AbstractImportDataOptions; -import ai.chat2db.server.web.api.controller.rdb.data.BaseFileImporter; +import ai.chat2db.server.tools.common.model.rdb.data.option.BaseImportExcelDataOptions; +import ai.chat2db.server.tools.common.model.rdb.data.option.AbstractImportDataOptions; +import ai.chat2db.server.web.api.controller.rdb.data.AbstractDataFileImporter; import com.alibaba.excel.EasyExcel; import org.springframework.web.multipart.MultipartFile; @@ -14,7 +14,7 @@ * @author: zgq * @date: 2024年04月27日 11:16 */ -public class BaseExcelImporter extends BaseFileImporter { +public class BaseExcelImporter extends AbstractDataFileImporter { @Override protected void doImportData(Connection connection, String databaseName, String schemaName, String tableName, List tableColumns, List fileColumns, diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/BaseSingleExcelExporter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/BaseXLSXExporter.java similarity index 52% rename from chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/BaseSingleExcelExporter.java rename to chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/BaseXLSXExporter.java index 1e1759033..b5479838f 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/BaseSingleExcelExporter.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/BaseXLSXExporter.java @@ -1,13 +1,14 @@ -package ai.chat2db.server.web.api.controller.rdb.data; +package ai.chat2db.server.web.api.controller.rdb.data.xlsx; -import ai.chat2db.server.tools.common.model.data.option.AbstractExportDataOptions; +import ai.chat2db.server.tools.common.model.rdb.data.option.AbstractExportDataOptions; +import ai.chat2db.server.web.api.controller.rdb.data.AbstractDataFileExporter; import ai.chat2db.server.web.api.controller.rdb.data.sql.EasySqlBuilder; -import ai.chat2db.server.web.api.controller.rdb.data.xlsx.EasyExcelExportUtil; import ai.chat2db.spi.util.ResultSetUtils; import com.alibaba.excel.support.ExcelTypeEnum; import jakarta.servlet.ServletOutputStream; import jakarta.servlet.http.HttpServletResponse; +import java.io.ByteArrayOutputStream; import java.io.IOException; import java.sql.Connection; import java.sql.ResultSet; @@ -15,12 +16,11 @@ import java.util.List; /** - * 功能描述 - * * @author: zgq - * @date: 2024年04月26日 13:04 + * @date: 2024年04月27日 15:23 */ -public abstract class BaseSingleExcelExporter extends AbstractSingleFileExporter { +public abstract class BaseXLSXExporter extends AbstractDataFileExporter { + @Override protected void doTableDataExport(HttpServletResponse response, Connection connection, String databaseName, @@ -39,6 +39,21 @@ protected void doTableDataExport(HttpServletResponse response, Connection connec } + @Override + protected ByteArrayOutputStream doTableDataExport(Connection connection, String databaseName, String schemaName, + String tableName, List tableColumns, + AbstractExportDataOptions exportDataOption) throws SQLException { + ByteArrayOutputStream byteOut = new ByteArrayOutputStream(); + String sql = EasySqlBuilder.buildQuerySql(databaseName, schemaName, tableName); + try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { + EasyExcelExportUtil.write(byteOut, EasyExcelExportUtil.getDataList(resultSet, tableColumns), + tableName, ResultSetUtils.getRsHeader(resultSet), + tableColumns, getExcelType(), exportDataOption); + return byteOut; + } + } + + protected abstract ExcelTypeEnum getExcelType(); } diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/EasyExcelExportUtil.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/EasyExcelExportUtil.java index 34884912e..ab571fecb 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/EasyExcelExportUtil.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/EasyExcelExportUtil.java @@ -1,8 +1,7 @@ package ai.chat2db.server.web.api.controller.rdb.data.xlsx; -import ai.chat2db.server.tools.common.model.data.option.AbstractExportDataOptions; -import ai.chat2db.server.tools.common.model.data.option.BaseExportDataOptions; -import ai.chat2db.spi.util.ResultSetUtils; +import ai.chat2db.server.tools.common.model.rdb.data.option.AbstractExportDataOptions; +import ai.chat2db.server.tools.common.model.rdb.data.option.BaseExportDataOptions; import com.alibaba.excel.EasyExcel; import com.alibaba.excel.support.ExcelTypeEnum; import com.alibaba.excel.write.builder.ExcelWriterSheetBuilder; diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/MultiXLSXExporter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/MultiXLSXExporter.java deleted file mode 100644 index 4a2626268..000000000 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/MultiXLSXExporter.java +++ /dev/null @@ -1,22 +0,0 @@ -package ai.chat2db.server.web.api.controller.rdb.data.xlsx; - -import ai.chat2db.server.domain.api.enums.ExportFileSuffix; -import ai.chat2db.server.web.api.controller.rdb.data.BaseMultiExcelExporter; -import com.alibaba.excel.support.ExcelTypeEnum; - -/** - * @author: zgq - * @date: 2024年04月26日 14:06 - */ -public class MultiXLSXExporter extends BaseMultiExcelExporter { - - - public MultiXLSXExporter() { - suffix = ExportFileSuffix.EXCEL.getSuffix(); - } - - @Override - protected ExcelTypeEnum getExcelType() { - return ExcelTypeEnum.XLSX; - } -} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/NoModelDataListener.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/NoModelDataListener.java index acc02e0a5..04b267472 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/NoModelDataListener.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/NoModelDataListener.java @@ -1,7 +1,7 @@ package ai.chat2db.server.web.api.controller.rdb.data.xlsx; -import ai.chat2db.server.tools.common.model.data.option.BaseImportExcelDataOptions; -import ai.chat2db.server.tools.common.model.data.option.AbstractImportDataOptions; +import ai.chat2db.server.tools.common.model.rdb.data.option.BaseImportExcelDataOptions; +import ai.chat2db.server.tools.common.model.rdb.data.option.AbstractImportDataOptions; import ai.chat2db.server.web.api.controller.rdb.data.sql.EasySqlBuilder; import com.alibaba.excel.context.AnalysisContext; import com.alibaba.excel.event.AnalysisEventListener; diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/SingleXLSXExporter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/SingleXLSXExporter.java deleted file mode 100644 index a5826e774..000000000 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/SingleXLSXExporter.java +++ /dev/null @@ -1,23 +0,0 @@ -package ai.chat2db.server.web.api.controller.rdb.data.xlsx; - -import ai.chat2db.server.domain.api.enums.ExportFileSuffix; -import ai.chat2db.server.web.api.controller.rdb.data.BaseSingleExcelExporter; -import com.alibaba.excel.support.ExcelTypeEnum; - -/** - * 功能描述 - * - * @author: zgq - * @date: 2024年04月26日 14:05 - */ -public class SingleXLSXExporter extends BaseSingleExcelExporter { - - public SingleXLSXExporter() { - suffix = ExportFileSuffix.EXCEL.getSuffix(); - contentType = "application/vnd.ms-excel"; - } - @Override - protected ExcelTypeEnum getExcelType() { - return ExcelTypeEnum.XLSX; - } -} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/XLSXExporter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/XLSXExporter.java index c4fcccea6..c77bf3896 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/XLSXExporter.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/XLSXExporter.java @@ -1,24 +1,22 @@ package ai.chat2db.server.web.api.controller.rdb.data.xlsx; +import ai.chat2db.server.domain.api.enums.ExportFileSuffix; import ai.chat2db.server.web.api.controller.rdb.data.DataFileExporter; -import ai.chat2db.server.web.api.controller.rdb.data.MultiFileExporter; -import ai.chat2db.server.web.api.controller.rdb.data.SingleFileExporter; +import com.alibaba.excel.support.ExcelTypeEnum; /** * @author: zgq * @date: 2024年04月26日 14:05 */ -public class XLSXExporter implements DataFileExporter { +public class XLSXExporter extends BaseXLSXExporter implements DataFileExporter { - - @Override - public SingleFileExporter createSingleFileExporter() { - return new SingleXLSXExporter(); + public XLSXExporter() { + suffix = ExportFileSuffix.EXCEL.getSuffix(); + contentType = "application/vnd.ms-excel"; } - @Override - public MultiFileExporter createMultiFileExporter() { - return new MultiXLSXExporter(); + protected ExcelTypeEnum getExcelType() { + return ExcelTypeEnum.XLSX; } } diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/factory/ExportDBDataStrategyFactory.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/factory/ExportDBDataStrategyFactory.java deleted file mode 100644 index 638557b17..000000000 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/factory/ExportDBDataStrategyFactory.java +++ /dev/null @@ -1,31 +0,0 @@ -package ai.chat2db.server.web.api.controller.rdb.factory; - -import ai.chat2db.server.domain.api.enums.ExportTypeEnum; -import ai.chat2db.server.web.api.controller.rdb.data.export.strategy.*; -import lombok.SneakyThrows; - -import java.util.Map; - -/** - * @author: zgq - * @date: 2024年03月24日 12:53 - */ -public class ExportDBDataStrategyFactory { - - public static final Map> SERVICE_MAP = Map.of( - ExportTypeEnum.SQL.getCode(), ExportDBData2SqlStrategy.class, - ExportTypeEnum.CSV.getCode(), ExportDBData2CsvStrategy.class, - ExportTypeEnum.EXCEL.getCode(), ExportDBData2ExcelStrategy.class, - ExportTypeEnum.JSON.getCode(), ExportDBData2JsonStrategy.class - ); - - @SneakyThrows - public static Class get(String type) { - Class dataResult = SERVICE_MAP.get(type); - if (dataResult == null) { - throw new ClassNotFoundException("no ExportUI was found"); - } else { - return dataResult; - } - } -} \ No newline at end of file diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/request/DatabaseExportDataRequest.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/request/DatabaseExportDataRequest.java index 5d10cea49..9a6a0c35f 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/request/DatabaseExportDataRequest.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/request/DatabaseExportDataRequest.java @@ -1,7 +1,7 @@ package ai.chat2db.server.web.api.controller.rdb.request; -import ai.chat2db.server.tools.common.model.data.option.AbstractExportDataOptions; -import ai.chat2db.server.tools.common.model.data.option.table.BaseTableOptions; +import ai.chat2db.server.tools.common.model.rdb.data.option.AbstractExportDataOptions; +import ai.chat2db.server.tools.common.model.rdb.data.option.table.BaseTableOptions; import ai.chat2db.server.web.api.controller.data.source.request.DataSourceBaseRequest; import jakarta.validation.constraints.NotNull; import lombok.AllArgsConstructor; diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/request/DatabaseImportDataRequest.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/request/DatabaseImportDataRequest.java index a9b66fab9..b2f29e3b7 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/request/DatabaseImportDataRequest.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/request/DatabaseImportDataRequest.java @@ -1,7 +1,7 @@ package ai.chat2db.server.web.api.controller.rdb.request; -import ai.chat2db.server.tools.common.model.data.option.AbstractImportDataOptions; -import ai.chat2db.server.tools.common.model.data.option.table.ImportTableOptions; +import ai.chat2db.server.tools.common.model.rdb.data.option.AbstractImportDataOptions; +import ai.chat2db.server.tools.common.model.rdb.data.option.table.ImportTableOptions; import ai.chat2db.server.web.api.controller.data.source.request.DataSourceBaseRequest; import lombok.AllArgsConstructor; import lombok.Data; From 157eb333ea3381d27e48bb3022c419d4ff168790 Mon Sep 17 00:00:00 2001 From: zgq <203083679@qq.com> Date: Sat, 27 Apr 2024 17:37:14 +0800 Subject: [PATCH 07/11] add LoggingDataExportObserver --- .../api/controller/rdb/DatabaseController.java | 6 +++++- .../rdb/data/AbstractDataFileExporter.java | 6 ++++++ .../rdb/data/DataFileFactoryProducer.java | 17 +++++++++++++++++ .../rdb/data/observer/DataExportObserver.java | 5 +++++ .../observer/LoggingDataExportObserver.java | 17 +++++++++++++++++ .../controller/rdb/data/sql/EasySqlBuilder.java | 11 ++++++----- .../controller/rdb/data/sql/SQLExporter.java | 2 ++ 7 files changed, 58 insertions(+), 6 deletions(-) create mode 100644 chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/observer/DataExportObserver.java create mode 100644 chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/observer/LoggingDataExportObserver.java diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/DatabaseController.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/DatabaseController.java index d8ef479cb..49ae22d73 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/DatabaseController.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/DatabaseController.java @@ -12,6 +12,7 @@ import ai.chat2db.server.web.api.controller.rdb.converter.DatabaseConverter; import ai.chat2db.server.web.api.controller.rdb.converter.RdbWebConverter; import ai.chat2db.server.web.api.controller.rdb.data.DataFileFactoryProducer; +import ai.chat2db.server.web.api.controller.rdb.data.observer.LoggingDataExportObserver; import ai.chat2db.server.web.api.controller.rdb.request.*; import ai.chat2db.server.web.api.controller.rdb.vo.MetaSchemaVO; import ai.chat2db.spi.model.Database; @@ -129,13 +130,16 @@ public void exportData(@Valid @RequestBody DatabaseExportDataRequest request, Ht try { response.setCharacterEncoding("utf-8"); DatabaseExportDataParam param = databaseConverter.request2param(request); + LoggingDataExportObserver observer = new LoggingDataExportObserver(); + DataFileFactoryProducer.addObserver(observer); DataFileFactoryProducer.getFactory(request.getExportDataOption().getFileType()) .createExporter().exportDataFile(param, response); } catch (Exception e) { response.reset(); throw new RuntimeException(e); + } finally { + DataFileFactoryProducer.removeObserver(); } - } @PostMapping("/import_data") diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/AbstractDataFileExporter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/AbstractDataFileExporter.java index 1393bbff2..3d042dc34 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/AbstractDataFileExporter.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/AbstractDataFileExporter.java @@ -7,6 +7,7 @@ import ai.chat2db.spi.sql.Chat2DBContext; import jakarta.servlet.ServletOutputStream; import jakarta.servlet.http.HttpServletResponse; +import lombok.extern.slf4j.Slf4j; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -21,6 +22,7 @@ * @author: zgq * @date: 2024年04月27日 15:17 */ +@Slf4j public abstract class AbstractDataFileExporter implements DataFileExporter { @@ -31,14 +33,18 @@ public abstract class AbstractDataFileExporter implements DataFileExporter { @Override public void exportDataFile(DatabaseExportDataParam param, HttpServletResponse response) throws IOException, SQLException { if (param.getExportTableOptions().size() > 1) { + DataFileFactoryProducer.notifyObservers("export multi table data file"); exportMultiDataFile(param, response); } else { + DataFileFactoryProducer.notifyObservers("export single table data file"); exportSingleDataFile(param, response); } + DataFileFactoryProducer.notifyObservers("Finished successfully"); } public void exportSingleDataFile(DatabaseExportDataParam param, HttpServletResponse response) throws SQLException { + DataFileFactoryProducer.notifyObservers("export start"); BaseTableOptions tableOptions = param.getExportTableOptions().get(0); String tableName = tableOptions.getTableName(); List tableColumns = tableOptions.getTableColumns(); diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/DataFileFactoryProducer.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/DataFileFactoryProducer.java index e9a08d96b..2b0e36fed 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/DataFileFactoryProducer.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/DataFileFactoryProducer.java @@ -2,6 +2,8 @@ import ai.chat2db.server.web.api.controller.rdb.data.csv.CSVImportExportFactory; import ai.chat2db.server.web.api.controller.rdb.data.json.JSONImportExportFactory; +import ai.chat2db.server.web.api.controller.rdb.data.observer.DataExportObserver; +import ai.chat2db.server.web.api.controller.rdb.data.observer.LoggingDataExportObserver; import ai.chat2db.server.web.api.controller.rdb.data.sql.SQLImportExportFactory; import ai.chat2db.server.web.api.controller.rdb.data.xlsx.XLSXImportExportFactory; @@ -14,6 +16,21 @@ */ public class DataFileFactoryProducer { + private static final ThreadLocal observerThreadLocal = ThreadLocal.withInitial(LoggingDataExportObserver::new); + + public static void addObserver(DataExportObserver observer) { + observerThreadLocal.set(observer); + } + + public static void removeObserver() { + observerThreadLocal.remove(); + } + + public static void notifyObservers(String log) { + DataExportObserver observer = observerThreadLocal.get(); + observer.onDataExported(log); + } + public static final Map factories = new HashMap<>(); static { diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/observer/DataExportObserver.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/observer/DataExportObserver.java new file mode 100644 index 000000000..2d5cce39e --- /dev/null +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/observer/DataExportObserver.java @@ -0,0 +1,5 @@ +package ai.chat2db.server.web.api.controller.rdb.data.observer; + +public interface DataExportObserver { + void onDataExported(String log); +} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/observer/LoggingDataExportObserver.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/observer/LoggingDataExportObserver.java new file mode 100644 index 000000000..0be7da0bb --- /dev/null +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/observer/LoggingDataExportObserver.java @@ -0,0 +1,17 @@ +package ai.chat2db.server.web.api.controller.rdb.data.observer; + +import java.util.ArrayList; +import java.util.List; + +public class LoggingDataExportObserver implements DataExportObserver { + private final List logs = new ArrayList<>(); + + @Override + public void onDataExported(String log) { + logs.add(log); + } + + public List getLogs() { + return new ArrayList<>(logs); + } +} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/sql/EasySqlBuilder.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/sql/EasySqlBuilder.java index d802b93fa..608fda70c 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/sql/EasySqlBuilder.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/sql/EasySqlBuilder.java @@ -2,6 +2,7 @@ import ai.chat2db.server.tools.common.model.rdb.data.option.AbstractExportDataOptions; import ai.chat2db.server.tools.common.model.rdb.data.option.sql.BaseExportData2SqlOptions; +import ai.chat2db.server.web.api.controller.rdb.data.DataFileFactoryProducer; import ai.chat2db.spi.util.ResultSetUtils; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; @@ -52,15 +53,15 @@ public static void exportData2Sql(String tableName, List tableColumns, A String sqlType = ((BaseExportData2SqlOptions) exportDataOption).getSqlType(); switch (sqlType) { case "single" -> { - log.info("Exporting single insert SQL for table: {}", tableName); + DataFileFactoryProducer.notifyObservers("Exporting single insert SQL for table:" + tableName); buildSingleInsert(writer, resultSet, metaData, tableName, headList, exportDataOption); } case "multi" -> { - log.info("Exporting multi insert SQL for table: {}", tableName); + DataFileFactoryProducer.notifyObservers("Exporting multi insert SQL for table: " + tableName); buildMultiInsert(writer, resultSet, metaData, tableName, headList, exportDataOption); } case "update" -> { - log.info("Exporting multi update SQL for table: {}", tableName); + DataFileFactoryProducer.notifyObservers("Exporting update SQL for table: " + tableName); buildUpdateStatement(writer, resultSet, metaData, tableName, headList); } default -> throw new IllegalStateException("Unexpected value: " + sqlType); @@ -87,7 +88,7 @@ public static void buildUpdateStatement(PrintWriter writer, ResultSet resultSet, sqlBuilder.append(columnName).append(" = '").append(value).append("'"); } - if (columnCount < headList.size()-1) { + if (columnCount < headList.size() - 1) { sqlBuilder.append(", "); } @@ -187,7 +188,7 @@ public static void buildUpdateConditions(String value, StringBuilder sqlBuilder) } public static String buildValues(List list) { - return list.stream() + return list.stream() .map(s -> s == null ? null : "'" + s + "'") .collect(Collectors.joining(",", "(", ")")); } diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/sql/SQLExporter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/sql/SQLExporter.java index a03a185f6..c407d9d48 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/sql/SQLExporter.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/sql/SQLExporter.java @@ -4,6 +4,7 @@ import ai.chat2db.server.tools.common.model.rdb.data.option.AbstractExportDataOptions; import ai.chat2db.server.web.api.controller.rdb.data.AbstractDataFileExporter; import ai.chat2db.server.web.api.controller.rdb.data.DataFileExporter; +import ai.chat2db.server.web.api.controller.rdb.data.DataFileFactoryProducer; import jakarta.servlet.http.HttpServletResponse; import java.io.ByteArrayOutputStream; @@ -33,6 +34,7 @@ protected void doTableDataExport(HttpServletResponse response, Connection connec String tableName, List tableColumns, AbstractExportDataOptions exportDataOption) throws SQLException { String sql = EasySqlBuilder.buildQuerySql(databaseName, schemaName, tableName); + DataFileFactoryProducer.notifyObservers("Export File Format SQL file"); try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { PrintWriter writer = response.getWriter(); EasySqlBuilder.exportData2Sql(tableName, tableColumns, exportDataOption, resultSet, writer); From 6add95e707ac57c65ee41ca3da08b04df110e875 Mon Sep 17 00:00:00 2001 From: zgq <203083679@qq.com> Date: Mon, 29 Apr 2024 11:42:15 +0800 Subject: [PATCH 08/11] Support importing data to a new table --- .../option/table/ImportNewTableOptions.java | 4 ++- .../controller/rdb/DatabaseController.java | 5 ---- .../rdb/data/AbstractDataFileExporter.java | 7 ++--- .../rdb/data/AbstractDataFileImporter.java | 30 ++++++++++++++++++- .../rdb/data/DataFileFactoryProducer.java | 29 +++++++++--------- .../controller/rdb/data/csv/CSVExporter.java | 4 +-- .../controller/rdb/data/csv/CSVImporter.java | 5 ++++ .../rdb/data/json/JSONExporter.java | 4 +++ .../rdb/data/json/JSONImporter.java | 18 ++++++----- .../rdb/data/sql/EasySqlBuilder.java | 10 ++----- .../controller/rdb/data/sql/SQLExporter.java | 6 ++-- ...SXExporter.java => BaseExcelExporter.java} | 12 ++++++-- .../rdb/data/xlsx/BaseExcelImporter.java | 8 +++-- .../rdb/data/xlsx/NoModelDataListener.java | 1 - .../rdb/data/xlsx/XLSXExporter.java | 2 +- .../rdb/data/xlsx/XLSXImporter.java | 6 +++- 16 files changed, 98 insertions(+), 53 deletions(-) rename chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/{BaseXLSXExporter.java => BaseExcelExporter.java} (82%) diff --git a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/rdb/data/option/table/ImportNewTableOptions.java b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/rdb/data/option/table/ImportNewTableOptions.java index 112964441..e891001a6 100644 --- a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/rdb/data/option/table/ImportNewTableOptions.java +++ b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/rdb/data/option/table/ImportNewTableOptions.java @@ -3,6 +3,8 @@ import lombok.Data; import lombok.EqualsAndHashCode; +import java.util.List; + /** * @author: zgq * @date: 2024年04月26日 9:32 @@ -14,5 +16,5 @@ public class ImportNewTableOptions extends ImportTableOptions{ /* * create table sql * */ - private String sql; + private List sql; } diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/DatabaseController.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/DatabaseController.java index 49ae22d73..1912f31a4 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/DatabaseController.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/DatabaseController.java @@ -12,7 +12,6 @@ import ai.chat2db.server.web.api.controller.rdb.converter.DatabaseConverter; import ai.chat2db.server.web.api.controller.rdb.converter.RdbWebConverter; import ai.chat2db.server.web.api.controller.rdb.data.DataFileFactoryProducer; -import ai.chat2db.server.web.api.controller.rdb.data.observer.LoggingDataExportObserver; import ai.chat2db.server.web.api.controller.rdb.request.*; import ai.chat2db.server.web.api.controller.rdb.vo.MetaSchemaVO; import ai.chat2db.spi.model.Database; @@ -130,15 +129,11 @@ public void exportData(@Valid @RequestBody DatabaseExportDataRequest request, Ht try { response.setCharacterEncoding("utf-8"); DatabaseExportDataParam param = databaseConverter.request2param(request); - LoggingDataExportObserver observer = new LoggingDataExportObserver(); - DataFileFactoryProducer.addObserver(observer); DataFileFactoryProducer.getFactory(request.getExportDataOption().getFileType()) .createExporter().exportDataFile(param, response); } catch (Exception e) { response.reset(); throw new RuntimeException(e); - } finally { - DataFileFactoryProducer.removeObserver(); } } diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/AbstractDataFileExporter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/AbstractDataFileExporter.java index 3d042dc34..03384bca4 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/AbstractDataFileExporter.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/AbstractDataFileExporter.java @@ -33,18 +33,17 @@ public abstract class AbstractDataFileExporter implements DataFileExporter { @Override public void exportDataFile(DatabaseExportDataParam param, HttpServletResponse response) throws IOException, SQLException { if (param.getExportTableOptions().size() > 1) { - DataFileFactoryProducer.notifyObservers("export multi table data file"); + log.info("export multi table data file"); exportMultiDataFile(param, response); } else { - DataFileFactoryProducer.notifyObservers("export single table data file"); + log.info("export single table data file"); exportSingleDataFile(param, response); } - DataFileFactoryProducer.notifyObservers("Finished successfully"); + log.info("Finished successfully"); } public void exportSingleDataFile(DatabaseExportDataParam param, HttpServletResponse response) throws SQLException { - DataFileFactoryProducer.notifyObservers("export start"); BaseTableOptions tableOptions = param.getExportTableOptions().get(0); String tableName = tableOptions.getTableName(); List tableColumns = tableOptions.getTableColumns(); diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/AbstractDataFileImporter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/AbstractDataFileImporter.java index efbf89c5d..9636b3c13 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/AbstractDataFileImporter.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/AbstractDataFileImporter.java @@ -2,29 +2,57 @@ import ai.chat2db.server.domain.api.param.datasource.DatabaseImportDataParam; import ai.chat2db.server.tools.common.model.rdb.data.option.AbstractImportDataOptions; +import ai.chat2db.server.tools.common.model.rdb.data.option.table.ImportNewTableOptions; import ai.chat2db.server.tools.common.model.rdb.data.option.table.ImportTableOptions; import ai.chat2db.spi.sql.Chat2DBContext; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections.CollectionUtils; import org.springframework.web.multipart.MultipartFile; import java.io.IOException; import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; import java.util.List; /** * @author: zgq * @date: 2024年04月26日 22:30 */ +@Slf4j public abstract class AbstractDataFileImporter implements DataFileImporter { @Override public void importDataFile(DatabaseImportDataParam param, MultipartFile file) throws IOException { + Connection connection = Chat2DBContext.getConnection(); String databaseName = param.getDatabaseName(); String schemaName = param.getSchemaName(); ImportTableOptions importTableOption = param.getImportTableOption(); String tableName = importTableOption.getTableName(); + createNewTable(connection, importTableOption,tableName); List tableColumns = importTableOption.getTableColumns(); List fileColumns = importTableOption.getFileColumns(); - doImportData(Chat2DBContext.getConnection(),databaseName, schemaName, tableName, tableColumns, fileColumns, param.getImportDataOption(), file); + doImportData(connection,databaseName, schemaName, tableName, tableColumns, fileColumns, param.getImportDataOption(), file); + log.info("Finished successfully"); + } + + private void createNewTable(Connection connection, ImportTableOptions importTableOption, String tableName) { + log.info("create new table:{}",tableName); + if (importTableOption instanceof ImportNewTableOptions importNewTableOptions) { + List sqlList = importNewTableOptions.getSql(); + if (CollectionUtils.isEmpty(sqlList)) { + return; + } + try (Statement statement = connection.createStatement()){ + for (String sql : sqlList) { + statement.addBatch(sql); + } + statement.executeBatch(); + log.info("Successfully created new table"); + } catch (SQLException e) { + throw new RuntimeException(e); + } + } } protected abstract void doImportData(Connection connection, String databaseName, String schemaName, String tableName, diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/DataFileFactoryProducer.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/DataFileFactoryProducer.java index 2b0e36fed..be2c45152 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/DataFileFactoryProducer.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/DataFileFactoryProducer.java @@ -2,8 +2,6 @@ import ai.chat2db.server.web.api.controller.rdb.data.csv.CSVImportExportFactory; import ai.chat2db.server.web.api.controller.rdb.data.json.JSONImportExportFactory; -import ai.chat2db.server.web.api.controller.rdb.data.observer.DataExportObserver; -import ai.chat2db.server.web.api.controller.rdb.data.observer.LoggingDataExportObserver; import ai.chat2db.server.web.api.controller.rdb.data.sql.SQLImportExportFactory; import ai.chat2db.server.web.api.controller.rdb.data.xlsx.XLSXImportExportFactory; @@ -16,20 +14,21 @@ */ public class DataFileFactoryProducer { - private static final ThreadLocal observerThreadLocal = ThreadLocal.withInitial(LoggingDataExportObserver::new); +// private static final ThreadLocal observerThreadLocal = ThreadLocal.withInitial(LoggingDataExportObserver::new); +// +// public static void removeObserver() { +// observerThreadLocal.remove(); +// } +// +// public static void notifyObserver(String log) { +// DataExportObserver observer =getObserver(); +// observer.onDataExported(log); +// } +// +// public static DataExportObserver getObserver() { +// return observerThreadLocal.get(); +// } - public static void addObserver(DataExportObserver observer) { - observerThreadLocal.set(observer); - } - - public static void removeObserver() { - observerThreadLocal.remove(); - } - - public static void notifyObservers(String log) { - DataExportObserver observer = observerThreadLocal.get(); - observer.onDataExported(log); - } public static final Map factories = new HashMap<>(); diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/csv/CSVExporter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/csv/CSVExporter.java index faea1b36c..9293cdca4 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/csv/CSVExporter.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/csv/CSVExporter.java @@ -2,14 +2,14 @@ import ai.chat2db.server.domain.api.enums.ExportFileSuffix; import ai.chat2db.server.web.api.controller.rdb.data.DataFileExporter; -import ai.chat2db.server.web.api.controller.rdb.data.xlsx.BaseXLSXExporter; +import ai.chat2db.server.web.api.controller.rdb.data.xlsx.BaseExcelExporter; import com.alibaba.excel.support.ExcelTypeEnum; /** * @author: zgq * @date: 2024年04月26日 11:27 */ -public class CSVExporter extends BaseXLSXExporter implements DataFileExporter { +public class CSVExporter extends BaseExcelExporter implements DataFileExporter { public CSVExporter() { diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/csv/CSVImporter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/csv/CSVImporter.java index 6139e2fac..64efdf731 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/csv/CSVImporter.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/csv/CSVImporter.java @@ -2,6 +2,7 @@ import ai.chat2db.server.web.api.controller.rdb.data.DataFileImporter; import ai.chat2db.server.web.api.controller.rdb.data.xlsx.BaseExcelImporter; +import com.alibaba.excel.support.ExcelTypeEnum; /** * 功能描述 @@ -12,4 +13,8 @@ public class CSVImporter extends BaseExcelImporter implements DataFileImporter { + @Override + protected ExcelTypeEnum getExcelType() { + return ExcelTypeEnum.CSV; + } } diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/json/JSONExporter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/json/JSONExporter.java index c01129da5..1e44ec42a 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/json/JSONExporter.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/json/JSONExporter.java @@ -7,6 +7,7 @@ import ai.chat2db.server.web.api.controller.rdb.data.DataFileExporter; import ai.chat2db.server.web.api.controller.rdb.data.sql.EasySqlBuilder; import jakarta.servlet.http.HttpServletResponse; +import lombok.extern.slf4j.Slf4j; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -24,6 +25,7 @@ * @author: zgq * @date: 2024年04月26日 14:20 */ +@Slf4j public class JSONExporter extends AbstractDataFileExporter implements DataFileExporter { public JSONExporter() { @@ -35,6 +37,7 @@ public JSONExporter() { protected void doTableDataExport(HttpServletResponse response, Connection connection, String databaseName, String schemaName, String tableName, List tableColumns, AbstractExportDataOptions exportDataOption) throws SQLException { + log.info("Export File Format JSON file"); String sql = EasySqlBuilder.buildQuerySql(databaseName, schemaName, tableName); try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { EasyJsonExportUtil.write(tableColumns, (ExportData2JsonOptions) exportDataOption, resultSet, response.getWriter()); @@ -49,6 +52,7 @@ protected void doTableDataExport(HttpServletResponse response, Connection connec protected ByteArrayOutputStream doTableDataExport(Connection connection, String databaseName, String schemaName, String tableName, List tableColumns, AbstractExportDataOptions exportDataOption) throws SQLException { + log.info("Export File Format JSON file"); ByteArrayOutputStream byteOut = new ByteArrayOutputStream(); String sql = EasySqlBuilder.buildQuerySql(databaseName, schemaName, tableName); try (ResultSet resultSet = connection.createStatement().executeQuery(sql); diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/json/JSONImporter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/json/JSONImporter.java index 472c2874e..86f14bc97 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/json/JSONImporter.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/json/JSONImporter.java @@ -34,6 +34,7 @@ public class JSONImporter extends AbstractDataFileImporter implements DataFileIm @Override protected void doImportData(Connection connection, String databaseName, String schemaName, String tableName, List tableColumns, List fileColumns, AbstractImportDataOptions importDataOption, MultipartFile file) { + log.info("import JSON data file"); String rootNodeName = ((ImportJsonDataOptions) importDataOption).getRootNodeName(); String dataTimeFormat = ((ImportJsonDataOptions) importDataOption).getDataTimeFormat(); Integer dataStartRowNum = ((ImportJsonDataOptions) importDataOption).getDataStartRowNum(); @@ -54,24 +55,25 @@ protected void doImportData(Connection connection, String databaseName, String s addSql(tableName, fileColumns, sqlBuilder, statement, values); sqlCount++; if (sqlCount == 1000) { - log.info("execute batch sqlCount:{}/1000", sqlCount); - Instant startTime = Instant.now(); - executeBatch(statement); - log.info("execute batch cost:{}ms", Instant.now().toEpochMilli() - startTime.toEpochMilli()); + executeBatch(sqlCount, statement); sqlCount = 0; } } if (sqlCount > 0) { - log.info("execute batch sqlCount:{}/1000", sqlCount); - Instant startTime = Instant.now(); - executeBatch(statement); - log.info("execute batch cost:{}ms", Instant.now().toEpochMilli() - startTime.toEpochMilli()); + executeBatch(sqlCount, statement); } } catch (Exception e) { throw new RuntimeException(e); } } + private void executeBatch(int sqlCount, Statement statement) throws SQLException { + log.info("execute batch sqlCount:{}/1000", sqlCount); + Instant startTime = Instant.now(); + executeBatch(statement); + log.info("execute batch cost:{}ms", Instant.now().toEpochMilli() - startTime.toEpochMilli()); + } + private void iteratorValues(List fileColumns, String dataTimeFormat, List values, JsonNode recordNode) throws Exception { for (String columnName : fileColumns) { diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/sql/EasySqlBuilder.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/sql/EasySqlBuilder.java index 608fda70c..9da8f3581 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/sql/EasySqlBuilder.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/sql/EasySqlBuilder.java @@ -2,7 +2,6 @@ import ai.chat2db.server.tools.common.model.rdb.data.option.AbstractExportDataOptions; import ai.chat2db.server.tools.common.model.rdb.data.option.sql.BaseExportData2SqlOptions; -import ai.chat2db.server.web.api.controller.rdb.data.DataFileFactoryProducer; import ai.chat2db.spi.util.ResultSetUtils; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; @@ -53,15 +52,15 @@ public static void exportData2Sql(String tableName, List tableColumns, A String sqlType = ((BaseExportData2SqlOptions) exportDataOption).getSqlType(); switch (sqlType) { case "single" -> { - DataFileFactoryProducer.notifyObservers("Exporting single insert SQL for table:" + tableName); + log.info("Exporting single insert SQL for table:" + tableName); buildSingleInsert(writer, resultSet, metaData, tableName, headList, exportDataOption); } case "multi" -> { - DataFileFactoryProducer.notifyObservers("Exporting multi insert SQL for table: " + tableName); + log.info("Exporting multi insert SQL for table: " + tableName); buildMultiInsert(writer, resultSet, metaData, tableName, headList, exportDataOption); } case "update" -> { - DataFileFactoryProducer.notifyObservers("Exporting update SQL for table: " + tableName); + log.info("Exporting update SQL for table: " + tableName); buildUpdateStatement(writer, resultSet, metaData, tableName, headList); } default -> throw new IllegalStateException("Unexpected value: " + sqlType); @@ -77,7 +76,6 @@ public static void buildUpdateStatement(PrintWriter writer, ResultSet resultSet, for (int i = 1; i <= metaData.getColumnCount(); i++) { String columnName = metaData.getColumnName(i); if (!headList.contains(columnName) || Objects.equals("id", columnName)) { - log.info("{} is not in the export field list", columnName); continue; } columnCount++; @@ -109,7 +107,6 @@ public static void buildMultiInsert(PrintWriter writer, ResultSet resultSet, Res for (int i = 1; i < metaData.getColumnCount(); i++) { String columnName = metaData.getColumnName(i); if (!headList.contains(columnName)) { - log.info("{} is not in the export field list", columnName); continue; } columnCount++; @@ -145,7 +142,6 @@ public static void buildSingleInsert(PrintWriter writer, ResultSet resultSet, Re for (int i = 1; i < metaData.getColumnCount(); i++) { String columnName = metaData.getColumnName(i); if (!headList.contains(columnName)) { - log.info("{} is not in the export field list", columnName); continue; } columnCount++; diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/sql/SQLExporter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/sql/SQLExporter.java index c407d9d48..151b2e93c 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/sql/SQLExporter.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/sql/SQLExporter.java @@ -4,8 +4,8 @@ import ai.chat2db.server.tools.common.model.rdb.data.option.AbstractExportDataOptions; import ai.chat2db.server.web.api.controller.rdb.data.AbstractDataFileExporter; import ai.chat2db.server.web.api.controller.rdb.data.DataFileExporter; -import ai.chat2db.server.web.api.controller.rdb.data.DataFileFactoryProducer; import jakarta.servlet.http.HttpServletResponse; +import lombok.extern.slf4j.Slf4j; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -21,6 +21,7 @@ * @author: zgq * @date: 2024年04月26日 15:33 */ +@Slf4j public class SQLExporter extends AbstractDataFileExporter implements DataFileExporter { public SQLExporter() { @@ -34,7 +35,7 @@ protected void doTableDataExport(HttpServletResponse response, Connection connec String tableName, List tableColumns, AbstractExportDataOptions exportDataOption) throws SQLException { String sql = EasySqlBuilder.buildQuerySql(databaseName, schemaName, tableName); - DataFileFactoryProducer.notifyObservers("Export File Format SQL file"); + log.info("Export File Format SQL file"); try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { PrintWriter writer = response.getWriter(); EasySqlBuilder.exportData2Sql(tableName, tableColumns, exportDataOption, resultSet, writer); @@ -47,6 +48,7 @@ protected void doTableDataExport(HttpServletResponse response, Connection connec protected ByteArrayOutputStream doTableDataExport(Connection connection, String databaseName, String schemaName, String tableName, List tableColumns, AbstractExportDataOptions exportDataOption) throws SQLException { + log.info("Export File Format SQL file"); ByteArrayOutputStream byteOut = new ByteArrayOutputStream(); String sql = EasySqlBuilder.buildQuerySql(databaseName, schemaName, tableName); try (ResultSet resultSet = connection.createStatement().executeQuery(sql); diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/BaseXLSXExporter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/BaseExcelExporter.java similarity index 82% rename from chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/BaseXLSXExporter.java rename to chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/BaseExcelExporter.java index b5479838f..e8472552e 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/BaseXLSXExporter.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/BaseExcelExporter.java @@ -7,6 +7,7 @@ import com.alibaba.excel.support.ExcelTypeEnum; import jakarta.servlet.ServletOutputStream; import jakarta.servlet.http.HttpServletResponse; +import lombok.extern.slf4j.Slf4j; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -19,19 +20,22 @@ * @author: zgq * @date: 2024年04月27日 15:23 */ -public abstract class BaseXLSXExporter extends AbstractDataFileExporter { +@Slf4j +public abstract class BaseExcelExporter extends AbstractDataFileExporter { @Override protected void doTableDataExport(HttpServletResponse response, Connection connection, String databaseName, String schemaName, String tableName, List tableColumns, AbstractExportDataOptions exportDataOption) throws SQLException { + ExcelTypeEnum excelType = getExcelType(); + log.info("Export File Format {} file", excelType.name()); String sql = EasySqlBuilder.buildQuerySql(databaseName, schemaName, tableName); try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { ServletOutputStream outputStream = response.getOutputStream(); EasyExcelExportUtil.write(outputStream, EasyExcelExportUtil.getDataList(resultSet, tableColumns), tableName, ResultSetUtils.getRsHeader(resultSet), - tableColumns, getExcelType(), exportDataOption); + tableColumns, excelType, exportDataOption); } catch (IOException e) { throw new RuntimeException(e); } @@ -43,12 +47,14 @@ protected void doTableDataExport(HttpServletResponse response, Connection connec protected ByteArrayOutputStream doTableDataExport(Connection connection, String databaseName, String schemaName, String tableName, List tableColumns, AbstractExportDataOptions exportDataOption) throws SQLException { + ExcelTypeEnum excelType = getExcelType(); + log.info("Export File Format {} file", excelType.name()); ByteArrayOutputStream byteOut = new ByteArrayOutputStream(); String sql = EasySqlBuilder.buildQuerySql(databaseName, schemaName, tableName); try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { EasyExcelExportUtil.write(byteOut, EasyExcelExportUtil.getDataList(resultSet, tableColumns), tableName, ResultSetUtils.getRsHeader(resultSet), - tableColumns, getExcelType(), exportDataOption); + tableColumns, excelType, exportDataOption); return byteOut; } } diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/BaseExcelImporter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/BaseExcelImporter.java index 4eb18dd6e..fb4be0e90 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/BaseExcelImporter.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/BaseExcelImporter.java @@ -1,9 +1,10 @@ package ai.chat2db.server.web.api.controller.rdb.data.xlsx; -import ai.chat2db.server.tools.common.model.rdb.data.option.BaseImportExcelDataOptions; import ai.chat2db.server.tools.common.model.rdb.data.option.AbstractImportDataOptions; +import ai.chat2db.server.tools.common.model.rdb.data.option.BaseImportExcelDataOptions; import ai.chat2db.server.web.api.controller.rdb.data.AbstractDataFileImporter; import com.alibaba.excel.EasyExcel; +import com.alibaba.excel.support.ExcelTypeEnum; import org.springframework.web.multipart.MultipartFile; import java.io.IOException; @@ -14,7 +15,7 @@ * @author: zgq * @date: 2024年04月27日 11:16 */ -public class BaseExcelImporter extends AbstractDataFileImporter { +public abstract class BaseExcelImporter extends AbstractDataFileImporter { @Override protected void doImportData(Connection connection, String databaseName, String schemaName, String tableName, List tableColumns, List fileColumns, @@ -23,9 +24,12 @@ protected void doImportData(Connection connection, String databaseName, String s fileColumns, importDataOption, connection); Integer headerRowNum = ((BaseImportExcelDataOptions) importDataOption).getHeaderRowNum(); EasyExcel.read(file.getInputStream(), noModelDataListener) + .excelType(getExcelType()) .sheet() .headRowNumber(headerRowNum) .doRead(); } + + protected abstract ExcelTypeEnum getExcelType(); } diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/NoModelDataListener.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/NoModelDataListener.java index 04b267472..7530ff4fb 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/NoModelDataListener.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/NoModelDataListener.java @@ -70,7 +70,6 @@ public void invokeHead(Map> headMap, AnalysisContext co String value = entry.getValue(); if (fileColumns.contains(value)) { Integer key = entry.getKey(); - log.info("head index: {} value: {}", key, value); columnIndexList.add(key); } } diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/XLSXExporter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/XLSXExporter.java index c77bf3896..bcf1eaf7d 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/XLSXExporter.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/XLSXExporter.java @@ -8,7 +8,7 @@ * @author: zgq * @date: 2024年04月26日 14:05 */ -public class XLSXExporter extends BaseXLSXExporter implements DataFileExporter { +public class XLSXExporter extends BaseExcelExporter implements DataFileExporter { public XLSXExporter() { suffix = ExportFileSuffix.EXCEL.getSuffix(); diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/XLSXImporter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/XLSXImporter.java index 07a39dd8c..d2c15e8af 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/XLSXImporter.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/XLSXImporter.java @@ -1,6 +1,7 @@ package ai.chat2db.server.web.api.controller.rdb.data.xlsx; import ai.chat2db.server.web.api.controller.rdb.data.DataFileImporter; +import com.alibaba.excel.support.ExcelTypeEnum; /** * @author: zgq @@ -8,5 +9,8 @@ */ public class XLSXImporter extends BaseExcelImporter implements DataFileImporter { - + @Override + protected ExcelTypeEnum getExcelType() { + return ExcelTypeEnum.XLSX; + } } From 5aecf6d9125cc71059fece5593f0b4545f99e5d7 Mon Sep 17 00:00:00 2001 From: zgq <203083679@qq.com> Date: Mon, 29 Apr 2024 22:17:45 +0800 Subject: [PATCH 09/11] Supports xls format --- .../domain/api/enums/ExportFileSuffix.java | 4 ++-- .../option/AbstractExportDataOptions.java | 1 + .../rdb/data/DataFileFactoryProducer.java | 2 ++ .../controller/rdb/data/xls/XLSExporter.java | 22 +++++++++++++++++ .../rdb/data/xls/XLSImportExportFactory.java | 24 +++++++++++++++++++ .../controller/rdb/data/xls/XLSImporter.java | 17 +++++++++++++ .../rdb/data/xlsx/XLSXExporter.java | 2 +- .../rdb/doc/export/ExportExcelService.java | 2 +- .../controller/task/biz/TaskBizService.java | 2 +- 9 files changed, 71 insertions(+), 5 deletions(-) create mode 100644 chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xls/XLSExporter.java create mode 100644 chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xls/XLSImportExportFactory.java create mode 100644 chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xls/XLSImporter.java diff --git a/chat2db-server/chat2db-server-domain/chat2db-server-domain-api/src/main/java/ai/chat2db/server/domain/api/enums/ExportFileSuffix.java b/chat2db-server/chat2db-server-domain/chat2db-server-domain-api/src/main/java/ai/chat2db/server/domain/api/enums/ExportFileSuffix.java index 9b5d854ee..2acea7dec 100644 --- a/chat2db-server/chat2db-server-domain/chat2db-server-domain-api/src/main/java/ai/chat2db/server/domain/api/enums/ExportFileSuffix.java +++ b/chat2db-server/chat2db-server-domain/chat2db-server-domain-api/src/main/java/ai/chat2db/server/domain/api/enums/ExportFileSuffix.java @@ -13,7 +13,7 @@ public enum ExportFileSuffix { //word WORD(".docx"), //excel - EXCEL(".xlsx"), + XLSX(".xlsx"), //markdown MARKDOWN(".md"), //html @@ -26,7 +26,7 @@ public enum ExportFileSuffix { JSON(".json"), CSV(".csv"), - + XLS(".xls"), ZIP(".zip"); private String suffix; diff --git a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/rdb/data/option/AbstractExportDataOptions.java b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/rdb/data/option/AbstractExportDataOptions.java index 7e3368368..b36bf00d8 100644 --- a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/rdb/data/option/AbstractExportDataOptions.java +++ b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/rdb/data/option/AbstractExportDataOptions.java @@ -17,6 +17,7 @@ @JsonSubTypes.Type(value = BaseExportDataOptions.class, name = "CSV"), @JsonSubTypes.Type(value = BaseExportData2SqlOptions.class, name = "SQL"), @JsonSubTypes.Type(value = BaseExportDataOptions.class, name = "XLSX"), + @JsonSubTypes.Type(value = BaseExportDataOptions.class, name = "XLS"), @JsonSubTypes.Type(value = ExportData2JsonOptions.class, name = "JSON") }) public abstract class AbstractExportDataOptions extends AbstractDataOption { diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/DataFileFactoryProducer.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/DataFileFactoryProducer.java index be2c45152..1f3a33739 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/DataFileFactoryProducer.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/DataFileFactoryProducer.java @@ -3,6 +3,7 @@ import ai.chat2db.server.web.api.controller.rdb.data.csv.CSVImportExportFactory; import ai.chat2db.server.web.api.controller.rdb.data.json.JSONImportExportFactory; import ai.chat2db.server.web.api.controller.rdb.data.sql.SQLImportExportFactory; +import ai.chat2db.server.web.api.controller.rdb.data.xls.XLSImportExportFactory; import ai.chat2db.server.web.api.controller.rdb.data.xlsx.XLSXImportExportFactory; import java.util.HashMap; @@ -35,6 +36,7 @@ public class DataFileFactoryProducer { static { factories.put("CSV", new CSVImportExportFactory()); factories.put("XLSX", new XLSXImportExportFactory()); + factories.put("XLS", new XLSImportExportFactory()); factories.put("JSON", new JSONImportExportFactory()); factories.put("SQL", new SQLImportExportFactory()); } diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xls/XLSExporter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xls/XLSExporter.java new file mode 100644 index 000000000..14e541f6c --- /dev/null +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xls/XLSExporter.java @@ -0,0 +1,22 @@ +package ai.chat2db.server.web.api.controller.rdb.data.xls; + +import ai.chat2db.server.domain.api.enums.ExportFileSuffix; +import ai.chat2db.server.web.api.controller.rdb.data.DataFileExporter; +import ai.chat2db.server.web.api.controller.rdb.data.xlsx.BaseExcelExporter; +import com.alibaba.excel.support.ExcelTypeEnum; + +/** + * @author: zgq + * @date: 2024年04月29日 21:48 + */ +public class XLSExporter extends BaseExcelExporter implements DataFileExporter { + + public XLSExporter() { + suffix = ExportFileSuffix.XLS.getSuffix(); + contentType = "application/vnd.ms-excel"; + } + @Override + protected ExcelTypeEnum getExcelType() { + return ExcelTypeEnum.XLS; + } +} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xls/XLSImportExportFactory.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xls/XLSImportExportFactory.java new file mode 100644 index 000000000..e95c10855 --- /dev/null +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xls/XLSImportExportFactory.java @@ -0,0 +1,24 @@ +package ai.chat2db.server.web.api.controller.rdb.data.xls; + +import ai.chat2db.server.web.api.controller.rdb.data.DataFileExporter; +import ai.chat2db.server.web.api.controller.rdb.data.DataFileImportExportFactory; +import ai.chat2db.server.web.api.controller.rdb.data.DataFileImporter; + +/** + * @author: zgq + * @date: 2024年04月29日 21:48 + */ +public class XLSImportExportFactory implements DataFileImportExportFactory { + + + @Override + public DataFileImporter createImporter() { + return new XLSImporter(); + } + + + @Override + public DataFileExporter createExporter() { + return new XLSExporter(); + } +} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xls/XLSImporter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xls/XLSImporter.java new file mode 100644 index 000000000..bca4ba4a1 --- /dev/null +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xls/XLSImporter.java @@ -0,0 +1,17 @@ +package ai.chat2db.server.web.api.controller.rdb.data.xls; + +import ai.chat2db.server.web.api.controller.rdb.data.DataFileImporter; +import ai.chat2db.server.web.api.controller.rdb.data.xlsx.BaseExcelImporter; +import com.alibaba.excel.support.ExcelTypeEnum; + +/** + * @author: zgq + * @date: 2024年04月29日 21:48 + */ +public class XLSImporter extends BaseExcelImporter implements DataFileImporter { + + @Override + protected ExcelTypeEnum getExcelType() { + return ExcelTypeEnum.XLS; + } +} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/XLSXExporter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/XLSXExporter.java index bcf1eaf7d..5621edc52 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/XLSXExporter.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/XLSXExporter.java @@ -11,7 +11,7 @@ public class XLSXExporter extends BaseExcelExporter implements DataFileExporter { public XLSXExporter() { - suffix = ExportFileSuffix.EXCEL.getSuffix(); + suffix = ExportFileSuffix.XLSX.getSuffix(); contentType = "application/vnd.ms-excel"; } diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/doc/export/ExportExcelService.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/doc/export/ExportExcelService.java index 79b97a271..21f744dbd 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/doc/export/ExportExcelService.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/doc/export/ExportExcelService.java @@ -29,7 +29,7 @@ public class ExportExcelService extends DatabaseExportService { public ExportExcelService() { exportTypeEnum = ExportTypeEnum.EXCEL; - suffix = ExportFileSuffix.EXCEL.getSuffix(); + suffix = ExportFileSuffix.XLSX.getSuffix(); contentType = "text/csv"; } diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/task/biz/TaskBizService.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/task/biz/TaskBizService.java index 2623c580e..6230af2cc 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/task/biz/TaskBizService.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/task/biz/TaskBizService.java @@ -217,7 +217,7 @@ private File createTempFile(String tableName, String exportType) { } else if (ExportTypeEnum.INSERT.getCode().equals(exportType)) { return FileUtil.createTempFile(fileName, ".sql", true); } else if (ExportTypeEnum.EXCEL.getCode().equals(exportType)) { - return FileUtil.createTempFile(fileName, ExportFileSuffix.EXCEL.getSuffix(), true); + return FileUtil.createTempFile(fileName, ExportFileSuffix.XLSX.getSuffix(), true); } else if (ExportTypeEnum.MARKDOWN.getCode().equals(exportType)) { return FileUtil.createTempFile(fileName, ExportFileSuffix.MARKDOWN.getSuffix(), true); } else if (ExportTypeEnum.WORD.getCode().equals(exportType)) { From a445509171e4d8ff4356d0cfbf06ec58a0134b6f Mon Sep 17 00:00:00 2001 From: zgq <203083679@qq.com> Date: Tue, 30 Apr 2024 23:08:58 +0800 Subject: [PATCH 10/11] optimize code --- .../option/sql/BaseExportData2SqlOptions.java | 2 + .../rdb/data/AbstractDataFileImporter.java | 2 +- .../rdb/data/json/EasyJsonExportUtil.java | 4 +- .../rdb/data/json/JSONImporter.java | 6 +-- .../rdb/data/sql/EasySqlBuilder.java | 49 +++++++++++-------- .../controller/rdb/data/sql/SQLExporter.java | 4 +- .../rdb/data/xlsx/BaseExcelImporter.java | 8 ++- .../rdb/data/xlsx/NoModelDataListener.java | 12 +++-- 8 files changed, 52 insertions(+), 35 deletions(-) diff --git a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/rdb/data/option/sql/BaseExportData2SqlOptions.java b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/rdb/data/option/sql/BaseExportData2SqlOptions.java index b54e0f5a5..05aa52a65 100644 --- a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/rdb/data/option/sql/BaseExportData2SqlOptions.java +++ b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/rdb/data/option/sql/BaseExportData2SqlOptions.java @@ -14,7 +14,9 @@ public class BaseExportData2SqlOptions extends BaseExportDataOptions { @NotBlank public String sqlType; + public String updateCondition; public BaseExportData2SqlOptions() { sqlType= "single"; + updateCondition= "id"; } } diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/AbstractDataFileImporter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/AbstractDataFileImporter.java index 9636b3c13..a24212325 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/AbstractDataFileImporter.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/AbstractDataFileImporter.java @@ -37,8 +37,8 @@ public void importDataFile(DatabaseImportDataParam param, MultipartFile file) th } private void createNewTable(Connection connection, ImportTableOptions importTableOption, String tableName) { - log.info("create new table:{}",tableName); if (importTableOption instanceof ImportNewTableOptions importNewTableOptions) { + log.info("create new table:{}",tableName); List sqlList = importNewTableOptions.getSql(); if (CollectionUtils.isEmpty(sqlList)) { return; diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/json/EasyJsonExportUtil.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/json/EasyJsonExportUtil.java index ba395320c..e4ed27e16 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/json/EasyJsonExportUtil.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/json/EasyJsonExportUtil.java @@ -5,6 +5,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.SerializationFeature; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections.CollectionUtils; import java.io.IOException; import java.io.PrintWriter; @@ -43,8 +44,7 @@ public static List> getDataMap(ResultSet resultSet, List row = new LinkedHashMap<>(); for (int i = 1; i <= columnCount; i++) { String columnName = metaData.getColumnName(i); - if (filedNames != null && !filedNames.contains(columnName)) { - log.info("{} is not in the export field list", columnName); + if (CollectionUtils.isNotEmpty(filedNames) && !filedNames.contains(columnName)) { continue; } row.put(columnName, resultSet.getObject(i)); diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/json/JSONImporter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/json/JSONImporter.java index 86f14bc97..7aa403118 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/json/JSONImporter.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/json/JSONImporter.java @@ -52,7 +52,7 @@ protected void doImportData(Connection connection, String databaseName, String s while (records.hasNext() && recordCount++ < limitRowSize) { JsonNode recordNode = records.next(); iteratorValues(fileColumns, dataTimeFormat, values, recordNode); - addSql(tableName, fileColumns, sqlBuilder, statement, values); + addSql(databaseName,schemaName,tableName, fileColumns, sqlBuilder, statement, values); sqlCount++; if (sqlCount == 1000) { executeBatch(sqlCount, statement); @@ -122,9 +122,9 @@ private void executeBatch(Statement statement) throws SQLException { statement.clearBatch(); } - private void addSql(String tableName, List fileColumns, StringBuilder sqlBuilder, + private void addSql(String databaseName, String schemaName, String tableName, List fileColumns, StringBuilder sqlBuilder, Statement statement, List values) throws SQLException { - EasySqlBuilder.buildInsert(tableName, true, fileColumns, sqlBuilder); + EasySqlBuilder.buildInsert(databaseName, schemaName, tableName, true, fileColumns, sqlBuilder); EasySqlBuilder.buildInsertValues(values, sqlBuilder); sqlBuilder.append(";"); values.clear(); diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/sql/EasySqlBuilder.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/sql/EasySqlBuilder.java index 9da8f3581..33cb60c9c 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/sql/EasySqlBuilder.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/sql/EasySqlBuilder.java @@ -3,6 +3,7 @@ import ai.chat2db.server.tools.common.model.rdb.data.option.AbstractExportDataOptions; import ai.chat2db.server.tools.common.model.rdb.data.option.sql.BaseExportData2SqlOptions; import ai.chat2db.spi.util.ResultSetUtils; +import jakarta.validation.constraints.NotBlank; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; @@ -42,7 +43,7 @@ private static void buildTableName(String databaseName, String schemaName, Strin } } - public static void exportData2Sql(String tableName, List tableColumns, AbstractExportDataOptions exportDataOption, + public static void exportData2Sql(String databaseName, String schemaName, String tableName, List tableColumns, AbstractExportDataOptions exportDataOption, ResultSet resultSet, PrintWriter writer) throws SQLException { ResultSetMetaData metaData = resultSet.getMetaData(); List headList = ResultSetUtils.getRsHeader(resultSet); @@ -53,29 +54,30 @@ public static void exportData2Sql(String tableName, List tableColumns, A switch (sqlType) { case "single" -> { log.info("Exporting single insert SQL for table:" + tableName); - buildSingleInsert(writer, resultSet, metaData, tableName, headList, exportDataOption); + buildSingleInsert(writer, resultSet, metaData, databaseName, schemaName, tableName, headList, exportDataOption); } case "multi" -> { log.info("Exporting multi insert SQL for table: " + tableName); - buildMultiInsert(writer, resultSet, metaData, tableName, headList, exportDataOption); + buildMultiInsert(writer, resultSet, metaData, databaseName, schemaName, tableName, headList, exportDataOption); } case "update" -> { log.info("Exporting update SQL for table: " + tableName); - buildUpdateStatement(writer, resultSet, metaData, tableName, headList); + buildUpdateStatement(writer, resultSet, metaData, databaseName, schemaName, tableName, headList, exportDataOption); } default -> throw new IllegalStateException("Unexpected value: " + sqlType); } } public static void buildUpdateStatement(PrintWriter writer, ResultSet resultSet, ResultSetMetaData metaData, - String tableName, List headList) throws SQLException { + String databaseName, String schemaName, String tableName, List headList, AbstractExportDataOptions exportDataOption) throws SQLException { + String updateCondition = ((BaseExportData2SqlOptions) exportDataOption).getUpdateCondition(); StringBuilder sqlBuilder = new StringBuilder(); while (resultSet.next()) { - EasySqlBuilder.buildUpdate(tableName, sqlBuilder); + EasySqlBuilder.buildUpdate(databaseName, schemaName, tableName, sqlBuilder); int columnCount = 0; for (int i = 1; i <= metaData.getColumnCount(); i++) { String columnName = metaData.getColumnName(i); - if (!headList.contains(columnName) || Objects.equals("id", columnName)) { + if (!headList.contains(columnName) || Objects.equals(updateCondition, columnName)) { continue; } columnCount++; @@ -91,20 +93,20 @@ public static void buildUpdateStatement(PrintWriter writer, ResultSet resultSet, } } - EasySqlBuilder.buildUpdateConditions(resultSet.getString("id"), sqlBuilder); + EasySqlBuilder.buildUpdateConditions(updateCondition, resultSet.getString(updateCondition), sqlBuilder); } writer.println(sqlBuilder); } public static void buildMultiInsert(PrintWriter writer, ResultSet resultSet, ResultSetMetaData metaData, - String tableName, List headList, AbstractExportDataOptions exportDataOption) throws SQLException { + String databaseName, String schemaName, String tableName, List headList, AbstractExportDataOptions exportDataOption) throws SQLException { Boolean containsHeader = ((BaseExportData2SqlOptions) exportDataOption).getContainsHeader(); StringBuilder sqlBuilder = new StringBuilder(); List values = new ArrayList<>(); - EasySqlBuilder.buildInsert(tableName, containsHeader, headList, sqlBuilder); + EasySqlBuilder.buildInsert(databaseName, schemaName, tableName, containsHeader, headList, sqlBuilder); while (resultSet.next()) { int columnCount = 0; - for (int i = 1; i < metaData.getColumnCount(); i++) { + for (int i = 1; i <= metaData.getColumnCount(); i++) { String columnName = metaData.getColumnName(i); if (!headList.contains(columnName)) { continue; @@ -133,13 +135,13 @@ public static void buildMultiInsert(PrintWriter writer, ResultSet resultSet, Res } public static void buildSingleInsert(PrintWriter writer, ResultSet resultSet, ResultSetMetaData metaData, - String tableName, List headList, AbstractExportDataOptions exportDataOption) throws SQLException { + String databaseName, String schemaName, String tableName, List headList, AbstractExportDataOptions exportDataOption) throws SQLException { Boolean containsHeader = ((BaseExportData2SqlOptions) exportDataOption).getContainsHeader(); StringBuilder sqlBuilder = new StringBuilder(); List values = new ArrayList<>(); while (resultSet.next()) { int columnCount = 0; - for (int i = 1; i < metaData.getColumnCount(); i++) { + for (int i = 1; i <= metaData.getColumnCount(); i++) { String columnName = metaData.getColumnName(i); if (!headList.contains(columnName)) { continue; @@ -155,7 +157,7 @@ public static void buildSingleInsert(PrintWriter writer, ResultSet resultSet, Re break; } } - EasySqlBuilder.buildInsert(tableName, containsHeader, headList, sqlBuilder); + EasySqlBuilder.buildInsert(databaseName, schemaName, tableName, containsHeader, headList, sqlBuilder); EasySqlBuilder.buildInsertValues(values, sqlBuilder); sqlBuilder.append(";\n"); values.clear(); @@ -163,24 +165,29 @@ public static void buildSingleInsert(PrintWriter writer, ResultSet resultSet, Re writer.println(sqlBuilder); } - public static void buildInsert(String tableName, boolean containsHeader, List filedNames, StringBuilder sqlBuilder) { - sqlBuilder.append("INSERT INTO ").append(tableName); + public static void buildInsert(String databaseName, String schemaName, @NotBlank String tableName, boolean containsHeader, List fieldNames, StringBuilder sqlBuilder) { + sqlBuilder.append("INSERT INTO "); + buildTableName(databaseName, schemaName, tableName, sqlBuilder); + if (containsHeader) { - sqlBuilder.append(" ").append(buildColumns(filedNames)); + sqlBuilder.append(" ").append(buildColumns(fieldNames)); } sqlBuilder.append(" VALUES "); } + public static void buildInsertValues(List values, StringBuilder sqlBuilder) { sqlBuilder.append(buildValues(values)); } - public static void buildUpdate(String tableName, StringBuilder sqlBuilder) { - sqlBuilder.append("UPDATE ").append(tableName).append(" SET "); + public static void buildUpdate(String databaseName, String schemaName, @NotBlank String tableName, StringBuilder sqlBuilder) { + sqlBuilder.append("UPDATE "); + buildTableName(databaseName, schemaName, tableName, sqlBuilder); + sqlBuilder.append(" SET "); } - public static void buildUpdateConditions(String value, StringBuilder sqlBuilder) { - sqlBuilder.append(" WHERE id = ").append("'").append(value).append("'").append(";\n"); + public static void buildUpdateConditions(String updateCondition, String value, StringBuilder sqlBuilder) { + sqlBuilder.append(" WHERE ").append(updateCondition).append(" = ").append("'").append(value).append("'").append(";\n"); } public static String buildValues(List list) { diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/sql/SQLExporter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/sql/SQLExporter.java index 151b2e93c..6da8c5b6d 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/sql/SQLExporter.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/sql/SQLExporter.java @@ -38,7 +38,7 @@ protected void doTableDataExport(HttpServletResponse response, Connection connec log.info("Export File Format SQL file"); try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { PrintWriter writer = response.getWriter(); - EasySqlBuilder.exportData2Sql(tableName, tableColumns, exportDataOption, resultSet, writer); + EasySqlBuilder.exportData2Sql(databaseName,schemaName,tableName, tableColumns, exportDataOption, resultSet, writer); } catch (IOException e) { throw new RuntimeException(e); } @@ -53,7 +53,7 @@ protected ByteArrayOutputStream doTableDataExport(Connection connection, String String sql = EasySqlBuilder.buildQuerySql(databaseName, schemaName, tableName); try (ResultSet resultSet = connection.createStatement().executeQuery(sql); PrintWriter writer = new PrintWriter(new OutputStreamWriter(byteOut, StandardCharsets.UTF_8))) { - EasySqlBuilder.exportData2Sql(tableName, tableColumns, exportDataOption, resultSet, writer); + EasySqlBuilder.exportData2Sql(databaseName, schemaName, tableName, tableColumns, exportDataOption, resultSet, writer); } return byteOut; } diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/BaseExcelImporter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/BaseExcelImporter.java index fb4be0e90..23816e499 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/BaseExcelImporter.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/BaseExcelImporter.java @@ -5,6 +5,7 @@ import ai.chat2db.server.web.api.controller.rdb.data.AbstractDataFileImporter; import com.alibaba.excel.EasyExcel; import com.alibaba.excel.support.ExcelTypeEnum; +import lombok.extern.slf4j.Slf4j; import org.springframework.web.multipart.MultipartFile; import java.io.IOException; @@ -15,16 +16,19 @@ * @author: zgq * @date: 2024年04月27日 11:16 */ +@Slf4j public abstract class BaseExcelImporter extends AbstractDataFileImporter { @Override protected void doImportData(Connection connection, String databaseName, String schemaName, String tableName, List tableColumns, List fileColumns, AbstractImportDataOptions importDataOption, MultipartFile file) throws IOException { - NoModelDataListener noModelDataListener = new NoModelDataListener(schemaName, tableName, tableColumns, + ExcelTypeEnum excelType = getExcelType(); + log.info("import {} data file", excelType.name()); + NoModelDataListener noModelDataListener = new NoModelDataListener(databaseName,schemaName, tableName, tableColumns, fileColumns, importDataOption, connection); Integer headerRowNum = ((BaseImportExcelDataOptions) importDataOption).getHeaderRowNum(); EasyExcel.read(file.getInputStream(), noModelDataListener) - .excelType(getExcelType()) + .excelType(excelType) .sheet() .headRowNumber(headerRowNum) .doRead(); diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/NoModelDataListener.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/NoModelDataListener.java index 7530ff4fb..7ea127724 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/NoModelDataListener.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/NoModelDataListener.java @@ -1,7 +1,7 @@ package ai.chat2db.server.web.api.controller.rdb.data.xlsx; -import ai.chat2db.server.tools.common.model.rdb.data.option.BaseImportExcelDataOptions; import ai.chat2db.server.tools.common.model.rdb.data.option.AbstractImportDataOptions; +import ai.chat2db.server.tools.common.model.rdb.data.option.BaseImportExcelDataOptions; import ai.chat2db.server.web.api.controller.rdb.data.sql.EasySqlBuilder; import com.alibaba.excel.context.AnalysisContext; import com.alibaba.excel.event.AnalysisEventListener; @@ -9,6 +9,7 @@ import com.alibaba.excel.util.ConverterUtils; import com.alibaba.excel.util.ListUtils; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; import java.sql.Connection; import java.sql.SQLException; @@ -22,6 +23,7 @@ @Slf4j public class NoModelDataListener extends AnalysisEventListener> { + private final String databaseName; private final String schemaName; private final String tableName; private final List tableColumns; @@ -34,8 +36,9 @@ public class NoModelDataListener extends AnalysisEventListener tableColumns, + public NoModelDataListener(String databaseName, String schemaName, String tableName, List tableColumns, List fileColumns, AbstractImportDataOptions importDataOption, Connection connection) { + this.databaseName = databaseName; this.schemaName = schemaName; this.tableName = tableName; this.tableColumns = tableColumns; @@ -91,7 +94,8 @@ public void invoke(Map data, AnalysisContext context) { values.add(value); } } - String sql = String.format("INSERT INTO %s %s VALUES %s", tableName, + String sql = String.format("INSERT INTO %s.%s %s VALUES %s", + StringUtils.isBlank(databaseName) ? schemaName : databaseName, tableName, EasySqlBuilder.buildColumns(tableColumns), EasySqlBuilder.buildValues(values)); values.clear(); @@ -109,7 +113,7 @@ private void executeBatchInsert() throws SQLException { for (String sql : sqlCacheList) { stmt.addBatch(sql); } - log.info("execute batch {}/1000 sql start",sqlCacheList.size()); + log.info("execute batch {}/1000 sql start", sqlCacheList.size()); Instant startTime = Instant.now(); stmt.executeBatch(); log.info("execute batch sql success,cost time: {}ms", Instant.now().toEpochMilli() - startTime.toEpochMilli()); From 16f951757fc97c2e46bbb8853e3d46adc411465f Mon Sep 17 00:00:00 2001 From: zgq <203083679@qq.com> Date: Wed, 8 May 2024 12:45:20 +0800 Subject: [PATCH 11/11] support continueBatchOnError --- .../base/enums/JdbcUrlParameterEnum.java | 27 +++++++ .../option/table/ImportNewTableOptions.java | 2 +- .../controller/rdb/DatabaseController.java | 9 ++- .../rdb/data/AbstractDataFileImporter.java | 56 ++++++++++--- .../rdb/data/DataFileFactoryProducer.java | 34 ++++---- .../rdb/data/json/JSONImporter.java | 46 +++-------- .../rdb/data/observer/DataExportObserver.java | 5 -- .../observer/LoggingDataExportObserver.java | 17 ---- .../rdb/data/observer/LoggingObserver.java | 37 +++++++++ .../rdb/data/observer/Observer.java | 6 ++ .../rdb/data/util/EasyBatchSqlExecutor.java | 46 +++++++++++ .../rdb/data/xlsx/BaseExcelImporter.java | 3 +- .../rdb/data/xlsx/NoModelDataListener.java | 79 ++++++------------- 13 files changed, 231 insertions(+), 136 deletions(-) create mode 100644 chat2db-server/chat2db-server-tools/chat2db-server-tools-base/src/main/java/ai/chat2db/server/tools/base/enums/JdbcUrlParameterEnum.java delete mode 100644 chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/observer/DataExportObserver.java delete mode 100644 chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/observer/LoggingDataExportObserver.java create mode 100644 chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/observer/LoggingObserver.java create mode 100644 chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/observer/Observer.java create mode 100644 chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/util/EasyBatchSqlExecutor.java diff --git a/chat2db-server/chat2db-server-tools/chat2db-server-tools-base/src/main/java/ai/chat2db/server/tools/base/enums/JdbcUrlParameterEnum.java b/chat2db-server/chat2db-server-tools/chat2db-server-tools-base/src/main/java/ai/chat2db/server/tools/base/enums/JdbcUrlParameterEnum.java new file mode 100644 index 000000000..147ec3808 --- /dev/null +++ b/chat2db-server/chat2db-server-tools/chat2db-server-tools-base/src/main/java/ai/chat2db/server/tools/base/enums/JdbcUrlParameterEnum.java @@ -0,0 +1,27 @@ +package ai.chat2db.server.tools.base.enums; + +import lombok.Getter; + +/** + * @author: zgq + * @date: 2024年05月01日 19:57 + */ +@Getter +public enum JdbcUrlParameterEnum implements BaseEnum { + + CONTINUE_BATCH_ON_ERROR("continueBatchOnError"); + + JdbcUrlParameterEnum(String description) { + this.description = description; + } + + final String description; + + /** + * @return + */ + @Override + public String getCode() { + return this.name(); + } +} diff --git a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/rdb/data/option/table/ImportNewTableOptions.java b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/rdb/data/option/table/ImportNewTableOptions.java index e891001a6..d4cec36ea 100644 --- a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/rdb/data/option/table/ImportNewTableOptions.java +++ b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/rdb/data/option/table/ImportNewTableOptions.java @@ -16,5 +16,5 @@ public class ImportNewTableOptions extends ImportTableOptions{ /* * create table sql * */ - private List sql; + private String sql; } diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/DatabaseController.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/DatabaseController.java index 1912f31a4..c714b81ba 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/DatabaseController.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/DatabaseController.java @@ -12,6 +12,7 @@ import ai.chat2db.server.web.api.controller.rdb.converter.DatabaseConverter; import ai.chat2db.server.web.api.controller.rdb.converter.RdbWebConverter; import ai.chat2db.server.web.api.controller.rdb.data.DataFileFactoryProducer; +import ai.chat2db.server.web.api.controller.rdb.data.observer.LoggingObserver; import ai.chat2db.server.web.api.controller.rdb.request.*; import ai.chat2db.server.web.api.controller.rdb.vo.MetaSchemaVO; import ai.chat2db.spi.model.Database; @@ -138,16 +139,20 @@ public void exportData(@Valid @RequestBody DatabaseExportDataRequest request, Ht } @PostMapping("/import_data") - public ActionResult importData(@Valid @ModelAttribute(value = "file") MultipartFile file, @ModelAttribute("json") String jsonData) { + public void importData(@Valid @ModelAttribute(value = "file") MultipartFile file, + @ModelAttribute(value = "json") String jsonData, HttpServletResponse response) { try { ObjectMapper objectMapper = new ObjectMapper(); DatabaseImportDataRequest request = objectMapper.readValue(jsonData, DatabaseImportDataRequest.class); DatabaseImportDataParam param = databaseConverter.request2param(request); DataFileFactoryProducer.getFactory(request.getImportDataOption().getFileType()) .createImporter().importDataFile(param, file); - return ActionResult.isSuccess(); + response.getWriter().println(((LoggingObserver) DataFileFactoryProducer.getObserver()).getLogs()); } catch (Exception e) { + response.reset(); throw new RuntimeException(e); + } finally { + DataFileFactoryProducer.removeObserver(); } } } diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/AbstractDataFileImporter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/AbstractDataFileImporter.java index a24212325..0225e2a34 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/AbstractDataFileImporter.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/AbstractDataFileImporter.java @@ -1,15 +1,23 @@ package ai.chat2db.server.web.api.controller.rdb.data; import ai.chat2db.server.domain.api.param.datasource.DatabaseImportDataParam; +import ai.chat2db.server.tools.base.enums.JdbcUrlParameterEnum; import ai.chat2db.server.tools.common.model.rdb.data.option.AbstractImportDataOptions; import ai.chat2db.server.tools.common.model.rdb.data.option.table.ImportNewTableOptions; import ai.chat2db.server.tools.common.model.rdb.data.option.table.ImportTableOptions; +import ai.chat2db.server.web.api.controller.rdb.data.observer.LoggingObserver; +import ai.chat2db.spi.model.KeyValue; import ai.chat2db.spi.sql.Chat2DBContext; +import ai.chat2db.spi.sql.ConnectInfo; +import ai.chat2db.spi.util.JdbcUtils; +import ai.chat2db.spi.util.SqlUtils; +import com.alibaba.druid.DbType; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections.CollectionUtils; import org.springframework.web.multipart.MultipartFile; import java.io.IOException; +import java.sql.BatchUpdateException; import java.sql.Connection; import java.sql.SQLException; import java.sql.Statement; @@ -22,34 +30,62 @@ @Slf4j public abstract class AbstractDataFileImporter implements DataFileImporter { + public static final int BATCH_SIZE = 100; @Override public void importDataFile(DatabaseImportDataParam param, MultipartFile file) throws IOException { + configureUrlParameter(); + AbstractImportDataOptions importDataOption = param.getImportDataOption(); Connection connection = Chat2DBContext.getConnection(); + ImportTableOptions importTableOption = param.getImportTableOption(); String databaseName = param.getDatabaseName(); String schemaName = param.getSchemaName(); - ImportTableOptions importTableOption = param.getImportTableOption(); String tableName = importTableOption.getTableName(); - createNewTable(connection, importTableOption,tableName); + createNewTable(connection, importTableOption, tableName); List tableColumns = importTableOption.getTableColumns(); List fileColumns = importTableOption.getFileColumns(); - doImportData(connection,databaseName, schemaName, tableName, tableColumns, fileColumns, param.getImportDataOption(), file); - log.info("Finished successfully"); + doImportData(connection, databaseName, schemaName, tableName, tableColumns, fileColumns, importDataOption, file); + int errorCounter = ((LoggingObserver) DataFileFactoryProducer.getObserver()).getErrorCounter(); + if (errorCounter > 0) { + DataFileFactoryProducer.notifyInfo(String.format("Finished data file importing with errors: %s", errorCounter)); + } else { + DataFileFactoryProducer.notifyInfo("Finished data file importing"); + + } + } + + + private void configureUrlParameter() { + KeyValue keyValue = new KeyValue(); + keyValue.setKey(JdbcUrlParameterEnum.CONTINUE_BATCH_ON_ERROR.getDescription()); + keyValue.setValue(String.valueOf(true)); + ConnectInfo connectInfo = Chat2DBContext.getConnectInfo(); + connectInfo.setExtendInfo(List.of(keyValue)); + } private void createNewTable(Connection connection, ImportTableOptions importTableOption, String tableName) { if (importTableOption instanceof ImportNewTableOptions importNewTableOptions) { - log.info("create new table:{}",tableName); - List sqlList = importNewTableOptions.getSql(); + DataFileFactoryProducer.notifyInfo(String.format("create new table:%s", tableName)); + String createTableSql = importNewTableOptions.getSql(); + String type = Chat2DBContext.getConnectInfo().getDbType(); + DbType dbType = JdbcUtils.parse2DruidDbType(type); + List sqlList = SqlUtils.parse(createTableSql, dbType); if (CollectionUtils.isEmpty(sqlList)) { - return; + return; } - try (Statement statement = connection.createStatement()){ + try (Statement statement = connection.createStatement()) { for (String sql : sqlList) { statement.addBatch(sql); } - statement.executeBatch(); - log.info("Successfully created new table"); + try { + statement.executeBatch(); + } catch (BatchUpdateException e) { + DataFileFactoryProducer.notifyError("created new table failed"); + throw new RuntimeException(e); + } + DataFileFactoryProducer.notifyInfo("Successfully created new table"); } catch (SQLException e) { + DataFileFactoryProducer.notifyError("created new table failed"); throw new RuntimeException(e); } } diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/DataFileFactoryProducer.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/DataFileFactoryProducer.java index 1f3a33739..1691eb744 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/DataFileFactoryProducer.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/DataFileFactoryProducer.java @@ -2,11 +2,14 @@ import ai.chat2db.server.web.api.controller.rdb.data.csv.CSVImportExportFactory; import ai.chat2db.server.web.api.controller.rdb.data.json.JSONImportExportFactory; +import ai.chat2db.server.web.api.controller.rdb.data.observer.LoggingObserver; +import ai.chat2db.server.web.api.controller.rdb.data.observer.Observer; import ai.chat2db.server.web.api.controller.rdb.data.sql.SQLImportExportFactory; import ai.chat2db.server.web.api.controller.rdb.data.xls.XLSImportExportFactory; import ai.chat2db.server.web.api.controller.rdb.data.xlsx.XLSXImportExportFactory; import java.util.HashMap; +import java.util.List; import java.util.Map; /** @@ -15,20 +18,23 @@ */ public class DataFileFactoryProducer { -// private static final ThreadLocal observerThreadLocal = ThreadLocal.withInitial(LoggingDataExportObserver::new); -// -// public static void removeObserver() { -// observerThreadLocal.remove(); -// } -// -// public static void notifyObserver(String log) { -// DataExportObserver observer =getObserver(); -// observer.onDataExported(log); -// } -// -// public static DataExportObserver getObserver() { -// return observerThreadLocal.get(); -// } + private static final ThreadLocal observerThreadLocal = ThreadLocal.withInitial(LoggingObserver::new); + + public static void removeObserver() { + observerThreadLocal.remove(); + } + + public static void notifyInfo(String log) { + getObserver().notifyInfo(log); + } + + public static void notifyError(String log) { + getObserver().notifyError(log); + } + + public static Observer getObserver() { + return observerThreadLocal.get(); + } public static final Map factories = new HashMap<>(); diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/json/JSONImporter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/json/JSONImporter.java index 7aa403118..89c9c5493 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/json/JSONImporter.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/json/JSONImporter.java @@ -6,6 +6,7 @@ import ai.chat2db.server.web.api.controller.rdb.data.AbstractDataFileImporter; import ai.chat2db.server.web.api.controller.rdb.data.DataFileImporter; import ai.chat2db.server.web.api.controller.rdb.data.sql.EasySqlBuilder; +import ai.chat2db.server.web.api.controller.rdb.data.util.EasyBatchSqlExecutor; import cn.hutool.core.date.DatePattern; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; @@ -15,10 +16,7 @@ import org.springframework.web.multipart.MultipartFile; import java.sql.Connection; -import java.sql.SQLException; -import java.sql.Statement; import java.text.SimpleDateFormat; -import java.time.Instant; import java.util.ArrayList; import java.util.Iterator; import java.util.List; @@ -40,11 +38,11 @@ protected void doImportData(Connection connection, String databaseName, String s Integer dataStartRowNum = ((ImportJsonDataOptions) importDataOption).getDataStartRowNum(); Integer dataEndRowNum = ((ImportJsonDataOptions) importDataOption).getDataEndRowNum(); int limitRowSize = dataEndRowNum - dataStartRowNum + 1; - int sqlCount = 0; + List sqlCacheList = new ArrayList<>(BATCH_SIZE); int recordCount = 0; ObjectMapper objectMapper = new ObjectMapper(); StringBuilder sqlBuilder = new StringBuilder(); - try (Statement statement = connection.createStatement()) { + try { JsonNode jsonNode = objectMapper.readTree(file.getInputStream()); jsonNode = getJsonNode(rootNodeName, jsonNode); Iterator records = jsonNode.elements(); @@ -52,26 +50,23 @@ protected void doImportData(Connection connection, String databaseName, String s while (records.hasNext() && recordCount++ < limitRowSize) { JsonNode recordNode = records.next(); iteratorValues(fileColumns, dataTimeFormat, values, recordNode); - addSql(databaseName,schemaName,tableName, fileColumns, sqlBuilder, statement, values); - sqlCount++; - if (sqlCount == 1000) { - executeBatch(sqlCount, statement); - sqlCount = 0; + EasySqlBuilder.buildInsert(databaseName, schemaName, tableName, true, fileColumns, sqlBuilder); + EasySqlBuilder.buildInsertValues(values, sqlBuilder); + sqlBuilder.append(";"); + values.clear(); + sqlCacheList.add(sqlBuilder.toString()); + sqlBuilder.setLength(0); + if (sqlCacheList.size() >= BATCH_SIZE) { + EasyBatchSqlExecutor.executeBatchInsert(connection, sqlCacheList); } } - if (sqlCount > 0) { - executeBatch(sqlCount, statement); + if (sqlCacheList.size() > 0) { + EasyBatchSqlExecutor.executeBatchInsert(connection, sqlCacheList); } } catch (Exception e) { throw new RuntimeException(e); } - } - private void executeBatch(int sqlCount, Statement statement) throws SQLException { - log.info("execute batch sqlCount:{}/1000", sqlCount); - Instant startTime = Instant.now(); - executeBatch(statement); - log.info("execute batch cost:{}ms", Instant.now().toEpochMilli() - startTime.toEpochMilli()); } private void iteratorValues(List fileColumns, String dataTimeFormat, @@ -116,19 +111,4 @@ private static String formatDate(String dateString, String dataTimeFormat) throw SimpleDateFormat outputFormat = new SimpleDateFormat(DatePattern.NORM_DATETIME_PATTERN); return outputFormat.format(inputFormat.parse(dateString)); } - - private void executeBatch(Statement statement) throws SQLException { - statement.executeBatch(); - statement.clearBatch(); - } - - private void addSql(String databaseName, String schemaName, String tableName, List fileColumns, StringBuilder sqlBuilder, - Statement statement, List values) throws SQLException { - EasySqlBuilder.buildInsert(databaseName, schemaName, tableName, true, fileColumns, sqlBuilder); - EasySqlBuilder.buildInsertValues(values, sqlBuilder); - sqlBuilder.append(";"); - values.clear(); - statement.addBatch(sqlBuilder.toString()); - sqlBuilder.setLength(0); - } } diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/observer/DataExportObserver.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/observer/DataExportObserver.java deleted file mode 100644 index 2d5cce39e..000000000 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/observer/DataExportObserver.java +++ /dev/null @@ -1,5 +0,0 @@ -package ai.chat2db.server.web.api.controller.rdb.data.observer; - -public interface DataExportObserver { - void onDataExported(String log); -} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/observer/LoggingDataExportObserver.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/observer/LoggingDataExportObserver.java deleted file mode 100644 index 0be7da0bb..000000000 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/observer/LoggingDataExportObserver.java +++ /dev/null @@ -1,17 +0,0 @@ -package ai.chat2db.server.web.api.controller.rdb.data.observer; - -import java.util.ArrayList; -import java.util.List; - -public class LoggingDataExportObserver implements DataExportObserver { - private final List logs = new ArrayList<>(); - - @Override - public void onDataExported(String log) { - logs.add(log); - } - - public List getLogs() { - return new ArrayList<>(logs); - } -} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/observer/LoggingObserver.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/observer/LoggingObserver.java new file mode 100644 index 000000000..7f9e325d9 --- /dev/null +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/observer/LoggingObserver.java @@ -0,0 +1,37 @@ +package ai.chat2db.server.web.api.controller.rdb.data.observer; + +import cn.hutool.core.date.DatePattern; +import lombok.Getter; + +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.List; + +@Getter +public class LoggingObserver implements Observer { + private final List logs = new ArrayList<>(); + private int errorCounter = 0; + + @Override + public void notifyInfo(String log) { + logs.add(String.format("[%s] [Info]:%s", getTime(), log)); + } + + + @Override + public void notifyError(String log) { + errorCounter++; + logs.add("[" + getTime() + "] [Error]:execute sql failed, error sql:"+log+""); + } + + public String getLogs() { + return String.join("\n", logs); + } + + private String getTime() { + LocalDateTime now = LocalDateTime.now(); + DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SS"); + return now.format(dateTimeFormatter); + } +} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/observer/Observer.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/observer/Observer.java new file mode 100644 index 000000000..4ad55ce9f --- /dev/null +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/observer/Observer.java @@ -0,0 +1,6 @@ +package ai.chat2db.server.web.api.controller.rdb.data.observer; + +public interface Observer { + void notifyInfo(String log); + void notifyError(String log); +} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/util/EasyBatchSqlExecutor.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/util/EasyBatchSqlExecutor.java new file mode 100644 index 000000000..bd14ac01f --- /dev/null +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/util/EasyBatchSqlExecutor.java @@ -0,0 +1,46 @@ +package ai.chat2db.server.web.api.controller.rdb.data.util; + +import ai.chat2db.server.web.api.controller.rdb.data.DataFileFactoryProducer; + +import java.sql.BatchUpdateException; +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.List; + +/** + * @author: zgq + * @date: 2024年05月08日 12:30 + */ +public class EasyBatchSqlExecutor { + public static void executeBatchInsert(Connection connection, List sqlCacheList) { + try (Statement stmt = connection.createStatement()) { + for (String sql : sqlCacheList) { + stmt.addBatch(sql); + } + connection.setAutoCommit(false); + try { + stmt.executeBatch(); + connection.commit(); + } catch (BatchUpdateException e) { + connection.rollback(); + handleErrorSql(stmt,sqlCacheList); + } + stmt.clearBatch(); + sqlCacheList.clear(); + } catch (SQLException e) { + throw new RuntimeException(e); + } + } + + public static void handleErrorSql(Statement stmt, List sqlCacheList) { + for (String sql : sqlCacheList) { + try { + stmt.execute(sql); + } catch (SQLException ex) { + DataFileFactoryProducer.notifyError(sql + "error msg:" + ex.getMessage() + ""); + } + } + } + +} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/BaseExcelImporter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/BaseExcelImporter.java index 23816e499..af578d356 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/BaseExcelImporter.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/BaseExcelImporter.java @@ -3,6 +3,7 @@ import ai.chat2db.server.tools.common.model.rdb.data.option.AbstractImportDataOptions; import ai.chat2db.server.tools.common.model.rdb.data.option.BaseImportExcelDataOptions; import ai.chat2db.server.web.api.controller.rdb.data.AbstractDataFileImporter; +import ai.chat2db.server.web.api.controller.rdb.data.DataFileFactoryProducer; import com.alibaba.excel.EasyExcel; import com.alibaba.excel.support.ExcelTypeEnum; import lombok.extern.slf4j.Slf4j; @@ -23,10 +24,10 @@ protected void doImportData(Connection connection, String databaseName, String s List tableColumns, List fileColumns, AbstractImportDataOptions importDataOption, MultipartFile file) throws IOException { ExcelTypeEnum excelType = getExcelType(); - log.info("import {} data file", excelType.name()); NoModelDataListener noModelDataListener = new NoModelDataListener(databaseName,schemaName, tableName, tableColumns, fileColumns, importDataOption, connection); Integer headerRowNum = ((BaseImportExcelDataOptions) importDataOption).getHeaderRowNum(); + DataFileFactoryProducer.notifyInfo(String.format("import %s data file start", excelType.name())); EasyExcel.read(file.getInputStream(), noModelDataListener) .excelType(excelType) .sheet() diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/NoModelDataListener.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/NoModelDataListener.java index 7ea127724..896f367c3 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/NoModelDataListener.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/NoModelDataListener.java @@ -2,7 +2,9 @@ import ai.chat2db.server.tools.common.model.rdb.data.option.AbstractImportDataOptions; import ai.chat2db.server.tools.common.model.rdb.data.option.BaseImportExcelDataOptions; +import ai.chat2db.server.web.api.controller.rdb.data.DataFileFactoryProducer; import ai.chat2db.server.web.api.controller.rdb.data.sql.EasySqlBuilder; +import ai.chat2db.server.web.api.controller.rdb.data.util.EasyBatchSqlExecutor; import com.alibaba.excel.context.AnalysisContext; import com.alibaba.excel.event.AnalysisEventListener; import com.alibaba.excel.metadata.data.ReadCellData; @@ -11,10 +13,10 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; +import java.sql.BatchUpdateException; import java.sql.Connection; import java.sql.SQLException; import java.sql.Statement; -import java.time.Instant; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -29,7 +31,7 @@ public class NoModelDataListener extends AnalysisEventListener tableColumns; private final List fileColumns; private final Connection connection; - private final int BATCH_SIZE = 1000; + private final int BATCH_SIZE = 100; private final List sqlCacheList = ListUtils.newArrayListWithExpectedSize(BATCH_SIZE); private final List columnIndexList; private int recordIndex = 0; @@ -53,13 +55,9 @@ public NoModelDataListener(String databaseName, String schemaName, String tableN @Override public boolean hasNext(AnalysisContext context) { if (recordIndex++ >= limitRowSize) { - log.info("read {} records，stopping reading", limitRowSize); - try { - if (sqlCacheList.size() > 0) { - executeBatchInsert(); - } - } catch (SQLException e) { - throw new RuntimeException(e); + DataFileFactoryProducer.notifyInfo(String.format("read %s records，stopping reading", limitRowSize)); + if (sqlCacheList.size() > 0) { + EasyBatchSqlExecutor.executeBatchInsert(connection, sqlCacheList); } return false; } @@ -78,60 +76,35 @@ public void invokeHead(Map> headMap, AnalysisContext co } } - @Override public void invoke(Map data, AnalysisContext context) { - try { - List values = new ArrayList<>(); - for (int i = 0; i < data.size(); i++) { - if (!columnIndexList.contains(i)) { - continue; - } - String value = data.get(i); - if (Objects.isNull(value)) { - values.add(null); - } else { - values.add(value); - } + List values = new ArrayList<>(); + for (int i = 0; i < data.size(); i++) { + if (!columnIndexList.contains(i)) { + continue; } - String sql = String.format("INSERT INTO %s.%s %s VALUES %s", - StringUtils.isBlank(databaseName) ? schemaName : databaseName, tableName, - EasySqlBuilder.buildColumns(tableColumns), - EasySqlBuilder.buildValues(values)); - values.clear(); - sqlCacheList.add(sql); - if (sqlCacheList.size() >= 1000) { - executeBatchInsert(); + String value = data.get(i); + if (Objects.isNull(value)) { + values.add(null); + } else { + values.add(value); } - } catch (SQLException e) { - throw new RuntimeException(e); } - } - - private void executeBatchInsert() throws SQLException { - try (Statement stmt = connection.createStatement()) { - for (String sql : sqlCacheList) { - stmt.addBatch(sql); - } - log.info("execute batch {}/1000 sql start", sqlCacheList.size()); - Instant startTime = Instant.now(); - stmt.executeBatch(); - log.info("execute batch sql success,cost time: {}ms", Instant.now().toEpochMilli() - startTime.toEpochMilli()); - stmt.clearBatch(); - sqlCacheList.clear(); + String sql = String.format("INSERT INTO %s.%s %s VALUES %s", + StringUtils.isBlank(databaseName) ? schemaName : databaseName, tableName, + EasySqlBuilder.buildColumns(tableColumns), + EasySqlBuilder.buildValues(values)); + values.clear(); + sqlCacheList.add(sql); + if (sqlCacheList.size() >= BATCH_SIZE) { + EasyBatchSqlExecutor.executeBatchInsert(connection, sqlCacheList); } } - @Override public void doAfterAllAnalysed(AnalysisContext context) { - try { - if (sqlCacheList.size() > 0) { - executeBatchInsert(); - } - } catch (SQLException e) { - throw new RuntimeException(e); + if (sqlCacheList.size() > 0) { + EasyBatchSqlExecutor.executeBatchInsert(connection, sqlCacheList); } - } }