diff --git a/chat2db-server/chat2db-server-domain/chat2db-server-domain-api/src/main/java/ai/chat2db/server/domain/api/enums/ExportFileSuffix.java b/chat2db-server/chat2db-server-domain/chat2db-server-domain-api/src/main/java/ai/chat2db/server/domain/api/enums/ExportFileSuffix.java index 9b5d854ee..2acea7dec 100644 --- a/chat2db-server/chat2db-server-domain/chat2db-server-domain-api/src/main/java/ai/chat2db/server/domain/api/enums/ExportFileSuffix.java +++ b/chat2db-server/chat2db-server-domain/chat2db-server-domain-api/src/main/java/ai/chat2db/server/domain/api/enums/ExportFileSuffix.java @@ -13,7 +13,7 @@ public enum ExportFileSuffix { //word WORD(".docx"), //excel - EXCEL(".xlsx"), + XLSX(".xlsx"), //markdown MARKDOWN(".md"), //html @@ -26,7 +26,7 @@ public enum ExportFileSuffix { JSON(".json"), CSV(".csv"), - + XLS(".xls"), ZIP(".zip"); private String suffix; diff --git a/chat2db-server/chat2db-server-domain/chat2db-server-domain-api/src/main/java/ai/chat2db/server/domain/api/param/datasource/DatabaseExportDataParam.java b/chat2db-server/chat2db-server-domain/chat2db-server-domain-api/src/main/java/ai/chat2db/server/domain/api/param/datasource/DatabaseExportDataParam.java index 4e562cedf..fa9a42b96 100644 --- a/chat2db-server/chat2db-server-domain/chat2db-server-domain-api/src/main/java/ai/chat2db/server/domain/api/param/datasource/DatabaseExportDataParam.java +++ b/chat2db-server/chat2db-server-domain/chat2db-server-domain-api/src/main/java/ai/chat2db/server/domain/api/param/datasource/DatabaseExportDataParam.java @@ -1,9 +1,14 @@ package ai.chat2db.server.domain.api.param.datasource; +import ai.chat2db.server.tools.common.model.rdb.data.option.AbstractExportDataOptions; +import ai.chat2db.server.tools.common.model.rdb.data.option.table.BaseTableOptions; +import jakarta.validation.constraints.NotNull; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; +import java.util.List; + /** * @author: zgq * @date: 2024年03月24日 13:17 @@ -14,5 +19,7 @@ public class DatabaseExportDataParam { private String databaseName; private String schemaName; - private String exportType; + @NotNull + private AbstractExportDataOptions exportDataOption; + private List exportTableOptions; } \ No newline at end of file diff --git a/chat2db-server/chat2db-server-domain/chat2db-server-domain-api/src/main/java/ai/chat2db/server/domain/api/param/datasource/DatabaseImportDataParam.java b/chat2db-server/chat2db-server-domain/chat2db-server-domain-api/src/main/java/ai/chat2db/server/domain/api/param/datasource/DatabaseImportDataParam.java new file mode 100644 index 000000000..c1a577776 --- /dev/null +++ b/chat2db-server/chat2db-server-domain/chat2db-server-domain-api/src/main/java/ai/chat2db/server/domain/api/param/datasource/DatabaseImportDataParam.java @@ -0,0 +1,23 @@ +package ai.chat2db.server.domain.api.param.datasource; + +import ai.chat2db.server.tools.common.model.rdb.data.option.AbstractImportDataOptions; +import ai.chat2db.server.tools.common.model.rdb.data.option.table.ImportTableOptions; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * 功能描述 + * + * @author: zgq + * @date: 2024年04月23日 13:52 + */ +@Data +@AllArgsConstructor +@NoArgsConstructor +public class DatabaseImportDataParam { + private String databaseName; + private String schemaName; + private ImportTableOptions importTableOption; + private AbstractImportDataOptions importDataOption; +} diff --git a/chat2db-server/chat2db-server-tools/chat2db-server-tools-base/src/main/java/ai/chat2db/server/tools/base/enums/JdbcUrlParameterEnum.java b/chat2db-server/chat2db-server-tools/chat2db-server-tools-base/src/main/java/ai/chat2db/server/tools/base/enums/JdbcUrlParameterEnum.java new file mode 100644 index 000000000..147ec3808 --- /dev/null +++ b/chat2db-server/chat2db-server-tools/chat2db-server-tools-base/src/main/java/ai/chat2db/server/tools/base/enums/JdbcUrlParameterEnum.java @@ -0,0 +1,27 @@ +package ai.chat2db.server.tools.base.enums; + +import lombok.Getter; + +/** + * @author: zgq + * @date: 2024年05月01日 19:57 + */ +@Getter +public enum JdbcUrlParameterEnum implements BaseEnum { + + CONTINUE_BATCH_ON_ERROR("continueBatchOnError"); + + JdbcUrlParameterEnum(String description) { + this.description = description; + } + + final String description; + + /** + * @return + */ + @Override + public String getCode() { + return this.name(); + } +} diff --git a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/rdb/data/option/AbstractDataOption.java b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/rdb/data/option/AbstractDataOption.java new file mode 100644 index 000000000..311f00247 --- /dev/null +++ b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/rdb/data/option/AbstractDataOption.java @@ -0,0 +1,15 @@ +package ai.chat2db.server.tools.common.model.rdb.data.option; + +import jakarta.validation.constraints.NotNull; +import lombok.Data; + +/** + * @author: zgq + * @date: 2024年04月25日 22:59 + */ +@Data +public abstract class AbstractDataOption implements AbstractDataOptionInterface { + @NotNull + protected String fileType; + +} diff --git a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/rdb/data/option/AbstractDataOptionInterface.java b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/rdb/data/option/AbstractDataOptionInterface.java new file mode 100644 index 000000000..a8fb79c7c --- /dev/null +++ b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/rdb/data/option/AbstractDataOptionInterface.java @@ -0,0 +1,5 @@ +package ai.chat2db.server.tools.common.model.rdb.data.option; + +public interface AbstractDataOptionInterface { + String getFileType(); +} diff --git a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/rdb/data/option/AbstractExportDataOptions.java b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/rdb/data/option/AbstractExportDataOptions.java new file mode 100644 index 000000000..b36bf00d8 --- /dev/null +++ b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/rdb/data/option/AbstractExportDataOptions.java @@ -0,0 +1,25 @@ +package ai.chat2db.server.tools.common.model.rdb.data.option; + +import ai.chat2db.server.tools.common.model.rdb.data.option.json.ExportData2JsonOptions; +import ai.chat2db.server.tools.common.model.rdb.data.option.sql.BaseExportData2SqlOptions; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; + +/** + * @author: zgq + * @date: 2024年04月25日 23:31 + */ +@JsonTypeInfo( + use = JsonTypeInfo.Id.NAME, + property = "fileType", + visible = true) +@JsonSubTypes({ + @JsonSubTypes.Type(value = BaseExportDataOptions.class, name = "CSV"), + @JsonSubTypes.Type(value = BaseExportData2SqlOptions.class, name = "SQL"), + @JsonSubTypes.Type(value = BaseExportDataOptions.class, name = "XLSX"), + @JsonSubTypes.Type(value = BaseExportDataOptions.class, name = "XLS"), + @JsonSubTypes.Type(value = ExportData2JsonOptions.class, name = "JSON") +}) +public abstract class AbstractExportDataOptions extends AbstractDataOption { + +} diff --git a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/rdb/data/option/AbstractImportDataOptions.java b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/rdb/data/option/AbstractImportDataOptions.java new file mode 100644 index 000000000..a1bc14cd7 --- /dev/null +++ b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/rdb/data/option/AbstractImportDataOptions.java @@ -0,0 +1,24 @@ +package ai.chat2db.server.tools.common.model.rdb.data.option; + +import ai.chat2db.server.tools.common.model.rdb.data.option.json.ImportJsonDataOptions; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; + +/** + * @author: zgq + * @date: 2024年04月25日 23:04 + */ +@JsonTypeInfo( + use = JsonTypeInfo.Id.NAME, + property = "fileType", + visible = true +) +@JsonSubTypes({ + @JsonSubTypes.Type(value = BaseImportExcelDataOptions.class, name = "CSV"), + @JsonSubTypes.Type(value = BaseImportExcelDataOptions.class, name = "XLSX"), + @JsonSubTypes.Type(value = BaseImportExcelDataOptions.class, name = "XLS"), + @JsonSubTypes.Type(value = ImportJsonDataOptions.class, name = "JSON") +}) +public abstract class AbstractImportDataOptions extends AbstractDataOption { + +} diff --git a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/rdb/data/option/BaseExportDataOptions.java b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/rdb/data/option/BaseExportDataOptions.java new file mode 100644 index 000000000..effffbc7f --- /dev/null +++ b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/rdb/data/option/BaseExportDataOptions.java @@ -0,0 +1,22 @@ +package ai.chat2db.server.tools.common.model.rdb.data.option; + +import jakarta.validation.constraints.NotNull; +import lombok.Data; +import lombok.EqualsAndHashCode; + +/** + * 功能描述 + * + * @author: zgq + * @date: 2024年04月26日 0:02 + */ +@Data +@EqualsAndHashCode(callSuper =true) +public class BaseExportDataOptions extends AbstractExportDataOptions { + @NotNull + public Boolean containsHeader; + + public BaseExportDataOptions() { + this.containsHeader = true; + } +} diff --git a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/rdb/data/option/BaseImportDataOptions.java b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/rdb/data/option/BaseImportDataOptions.java new file mode 100644 index 000000000..a61f23057 --- /dev/null +++ b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/rdb/data/option/BaseImportDataOptions.java @@ -0,0 +1,20 @@ +package ai.chat2db.server.tools.common.model.rdb.data.option; + +import lombok.Data; +import lombok.EqualsAndHashCode; + +/** + * @author: zgq + * @date: 2024年04月25日 23:40 + */ +@Data +@EqualsAndHashCode(callSuper = true) +public class BaseImportDataOptions extends AbstractImportDataOptions { + public Integer dataStartRowNum; + public Integer dataEndRowNum; + + public BaseImportDataOptions() { + dataStartRowNum = 1; + dataEndRowNum = Integer.MAX_VALUE; + } +} diff --git a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/rdb/data/option/BaseImportExcelDataOptions.java b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/rdb/data/option/BaseImportExcelDataOptions.java new file mode 100644 index 000000000..ae667d586 --- /dev/null +++ b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/rdb/data/option/BaseImportExcelDataOptions.java @@ -0,0 +1,18 @@ +package ai.chat2db.server.tools.common.model.rdb.data.option; + +import lombok.Data; +import lombok.EqualsAndHashCode; + +/** + * @author: zgq + * @date: 2024年04月26日 10:06 + */ +@Data +@EqualsAndHashCode(callSuper = true) +public class BaseImportExcelDataOptions extends BaseImportDataOptions { + public Integer headerRowNum; + + public BaseImportExcelDataOptions() { + this.headerRowNum = 1; + } +} diff --git a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/rdb/data/option/json/ExportData2JsonOptions.java b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/rdb/data/option/json/ExportData2JsonOptions.java new file mode 100644 index 000000000..c35efd2ef --- /dev/null +++ b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/rdb/data/option/json/ExportData2JsonOptions.java @@ -0,0 +1,24 @@ +package ai.chat2db.server.tools.common.model.rdb.data.option.json; + +import ai.chat2db.server.tools.common.model.rdb.data.option.AbstractExportDataOptions; +import cn.hutool.core.date.DatePattern; +import lombok.Data; +import lombok.EqualsAndHashCode; + +/** + * @author: zgq + * @date: 2024年04月26日 0:20 + */ +@Data +@EqualsAndHashCode(callSuper =true) +public class ExportData2JsonOptions extends AbstractExportDataOptions { + + private String dataTimeFormat; + + private Boolean isTimestamps; + + public ExportData2JsonOptions() { + dataTimeFormat = DatePattern.NORM_DATETIME_PATTERN; + isTimestamps = false; + } +} diff --git a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/rdb/data/option/json/ImportJsonDataOptions.java b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/rdb/data/option/json/ImportJsonDataOptions.java new file mode 100644 index 000000000..5c71c0165 --- /dev/null +++ b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/rdb/data/option/json/ImportJsonDataOptions.java @@ -0,0 +1,22 @@ +package ai.chat2db.server.tools.common.model.rdb.data.option.json; + +import ai.chat2db.server.tools.common.model.rdb.data.option.BaseImportDataOptions; +import cn.hutool.core.date.DatePattern; +import lombok.Data; +import lombok.EqualsAndHashCode; + +/** + * @author: zgq + * @date: 2024年04月25日 23:08 + */ +@Data +@EqualsAndHashCode(callSuper = true) +public class ImportJsonDataOptions extends BaseImportDataOptions { + private String rootNodeName; + private String dataTimeFormat; + + public ImportJsonDataOptions() { + dataTimeFormat = DatePattern.NORM_DATETIME_PATTERN; + rootNodeName = ""; + } +} diff --git a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/rdb/data/option/sql/BaseExportData2SqlOptions.java b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/rdb/data/option/sql/BaseExportData2SqlOptions.java new file mode 100644 index 000000000..05aa52a65 --- /dev/null +++ b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/rdb/data/option/sql/BaseExportData2SqlOptions.java @@ -0,0 +1,22 @@ +package ai.chat2db.server.tools.common.model.rdb.data.option.sql; + +import ai.chat2db.server.tools.common.model.rdb.data.option.BaseExportDataOptions; +import jakarta.validation.constraints.NotBlank; +import lombok.Data; +import lombok.EqualsAndHashCode; + +/** + * @author: zgq + * @date: 2024年04月26日 0:36 + */ +@Data +@EqualsAndHashCode(callSuper =true) +public class BaseExportData2SqlOptions extends BaseExportDataOptions { + @NotBlank + public String sqlType; + public String updateCondition; + public BaseExportData2SqlOptions() { + sqlType= "single"; + updateCondition= "id"; + } +} diff --git a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/rdb/data/option/table/BaseTableOptions.java b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/rdb/data/option/table/BaseTableOptions.java new file mode 100644 index 000000000..9320fac39 --- /dev/null +++ b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/rdb/data/option/table/BaseTableOptions.java @@ -0,0 +1,21 @@ +package ai.chat2db.server.tools.common.model.rdb.data.option.table; + +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotEmpty; +import lombok.Data; + +import java.util.List; + +/** + * @author: zgq + * @date: 2024年04月25日 23:53 + */ +@Data +public class BaseTableOptions implements TableOptionInterface { + @NotBlank + public String tableName; + @NotEmpty + public List tableColumns; + @NotEmpty + public List fileColumns; +} diff --git a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/rdb/data/option/table/ImportNewTableOptions.java b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/rdb/data/option/table/ImportNewTableOptions.java new file mode 100644 index 000000000..d4cec36ea --- /dev/null +++ b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/rdb/data/option/table/ImportNewTableOptions.java @@ -0,0 +1,20 @@ +package ai.chat2db.server.tools.common.model.rdb.data.option.table; + +import lombok.Data; +import lombok.EqualsAndHashCode; + +import java.util.List; + +/** + * @author: zgq + * @date: 2024年04月26日 9:32 + */ +@Data +@EqualsAndHashCode(callSuper =true) +public class ImportNewTableOptions extends ImportTableOptions{ + + /* + * create table sql + * */ + private String sql; +} diff --git a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/rdb/data/option/table/ImportTableOptions.java b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/rdb/data/option/table/ImportTableOptions.java new file mode 100644 index 000000000..988a25195 --- /dev/null +++ b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/rdb/data/option/table/ImportTableOptions.java @@ -0,0 +1,27 @@ +package ai.chat2db.server.tools.common.model.rdb.data.option.table; + +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import lombok.Data; + +/** + * @author: zgq + * @date: 2024年04月26日 9:39 + */ + +@JsonTypeInfo( + use = JsonTypeInfo.Id.NAME, + property = "importType", + visible = true) +@JsonSubTypes({ + @JsonSubTypes.Type(value = ImportTableOptions.class, name = "OLD"), + @JsonSubTypes.Type(value = ImportNewTableOptions.class, name = "NEW"), +}) +@Data +public class ImportTableOptions extends BaseTableOptions { + private String importType; + + public ImportTableOptions() { + importType = "OLD"; + } +} diff --git a/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/rdb/data/option/table/TableOptionInterface.java b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/rdb/data/option/table/TableOptionInterface.java new file mode 100644 index 000000000..908b4612b --- /dev/null +++ b/chat2db-server/chat2db-server-tools/chat2db-server-tools-common/src/main/java/ai/chat2db/server/tools/common/model/rdb/data/option/table/TableOptionInterface.java @@ -0,0 +1,15 @@ +package ai.chat2db.server.tools.common.model.rdb.data.option.table; + +import java.util.List; + +/** + * @author: zgq + * @date: 2024年04月25日 23:49 + */ +public interface TableOptionInterface { + String getTableName(); + + List getTableColumns(); + + List getFileColumns(); +} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/aspect/ConnectionInfoHandler.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/aspect/ConnectionInfoHandler.java index ff1fe624a..804e14f95 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/aspect/ConnectionInfoHandler.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/aspect/ConnectionInfoHandler.java @@ -12,6 +12,8 @@ import ai.chat2db.spi.config.DriverConfig; import ai.chat2db.spi.sql.Chat2DBContext; import ai.chat2db.spi.sql.ConnectInfo; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; import jakarta.annotation.Resource; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; @@ -42,6 +44,11 @@ public Object connectionInfoHandler(ProceedingJoinPoint proceedingJoinPoint) thr if (params != null && params.length > 0) { for (int i = 0; i < params.length; i++) { Object param = params[i]; + if (param instanceof String && StringUtils.isNotBlank((String) param)) { + ObjectMapper objectMapper = new ObjectMapper(); + objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + param = objectMapper.readValue((String) param, DataSourceBaseRequest.class); + } if (param instanceof DataSourceBaseRequest) { Long dataSourceId = ((DataSourceBaseRequest) param).getDataSourceId(); String schemaName = ((DataSourceBaseRequest) param).getSchemaName(); diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/DatabaseController.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/DatabaseController.java index 6348db871..c714b81ba 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/DatabaseController.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/DatabaseController.java @@ -1,10 +1,7 @@ package ai.chat2db.server.web.api.controller.rdb; import ai.chat2db.server.domain.api.param.MetaDataQueryParam; -import ai.chat2db.server.domain.api.param.datasource.DatabaseCreateParam; -import ai.chat2db.server.domain.api.param.datasource.DatabaseExportDataParam; -import ai.chat2db.server.domain.api.param.datasource.DatabaseExportParam; -import ai.chat2db.server.domain.api.param.datasource.DatabaseQueryAllParam; +import ai.chat2db.server.domain.api.param.datasource.*; import ai.chat2db.server.domain.api.service.DatabaseService; import ai.chat2db.server.tools.base.wrapper.result.ActionResult; import ai.chat2db.server.tools.base.wrapper.result.DataResult; @@ -14,24 +11,22 @@ import ai.chat2db.server.web.api.controller.data.source.vo.DatabaseVO; import ai.chat2db.server.web.api.controller.rdb.converter.DatabaseConverter; import ai.chat2db.server.web.api.controller.rdb.converter.RdbWebConverter; -import ai.chat2db.server.web.api.controller.rdb.data.export.strategy.ExportDBDataStrategy; -import ai.chat2db.server.web.api.controller.rdb.factory.ExportDBDataStrategyFactory; -import ai.chat2db.server.web.api.controller.rdb.request.DatabaseCreateRequest; -import ai.chat2db.server.web.api.controller.rdb.request.DatabaseExportDataRequest; -import ai.chat2db.server.web.api.controller.rdb.request.DatabaseExportRequest; -import ai.chat2db.server.web.api.controller.rdb.request.UpdateDatabaseRequest; +import ai.chat2db.server.web.api.controller.rdb.data.DataFileFactoryProducer; +import ai.chat2db.server.web.api.controller.rdb.data.observer.LoggingObserver; +import ai.chat2db.server.web.api.controller.rdb.request.*; import ai.chat2db.server.web.api.controller.rdb.vo.MetaSchemaVO; import ai.chat2db.spi.model.Database; import ai.chat2db.spi.model.MetaSchema; import ai.chat2db.spi.model.Sql; +import com.fasterxml.jackson.databind.ObjectMapper; import jakarta.servlet.http.HttpServletResponse; import jakarta.validation.Valid; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.*; +import org.springframework.web.multipart.MultipartFile; import java.io.PrintWriter; -import java.lang.reflect.Constructor; import java.util.Objects; /** @@ -59,8 +54,8 @@ public class DatabaseController { @GetMapping("/database_schema_list") public DataResult databaseSchemaList(@Valid DataSourceBaseRequest request) { MetaDataQueryParam queryParam = MetaDataQueryParam.builder().dataSourceId(request.getDataSourceId()) - .refresh( - request.isRefresh()).build(); + .refresh( + request.isRefresh()).build(); DataResult result = databaseService.queryDatabaseSchema(queryParam); MetaSchemaVO schemaDto2vo = rdbWebConverter.metaSchemaDto2vo(result.getData()); return DataResult.of(schemaDto2vo); @@ -69,8 +64,8 @@ public DataResult databaseSchemaList(@Valid DataSourceBaseRequest @GetMapping("list") public ListResult databaseList(@Valid DataSourceBaseRequest request) { DatabaseQueryAllParam queryParam = DatabaseQueryAllParam.builder().dataSourceId(request.getDataSourceId()) - .refresh( - request.isRefresh()).build(); + .refresh( + request.isRefresh()).build(); ListResult result = databaseService.queryAll(queryParam); return ListResult.of(rdbWebConverter.databaseDto2vo(result.getData())); } @@ -95,7 +90,7 @@ public ActionResult deleteDatabase(@Valid @RequestBody DataSourceBaseRequest req */ @PostMapping("/create_database_sql") public DataResult createDatabase(@Valid @RequestBody DatabaseCreateRequest request) { - if(StringUtils.isBlank(request.getName())){ + if (StringUtils.isBlank(request.getName())) { request.setName(request.getDatabaseName()); } Database database = databaseConverter.request2param(request); @@ -111,12 +106,13 @@ public DataResult createDatabase(@Valid @RequestBody DatabaseCreateRequest @PostMapping("/modify_database") public ActionResult modifyDatabase(@Valid @RequestBody UpdateDatabaseRequest request) { DatabaseCreateParam param = DatabaseCreateParam.builder().name(request.getDatabaseName()) - .name(request.getNewDatabaseName()).build(); + .name(request.getNewDatabaseName()).build(); return databaseService.modifyDatabase(param); } + @PostMapping("/export") - public void exportDatabase(@Valid @RequestBody DatabaseExportRequest request, HttpServletResponse response){ - String fileName = Objects.isNull(request.getSchemaName())?request.getDatabaseName() : request.getSchemaName(); + public void exportDatabase(@Valid @RequestBody DatabaseExportRequest request, HttpServletResponse response) { + String fileName = Objects.isNull(request.getSchemaName()) ? request.getDatabaseName() : request.getSchemaName(); response.setContentType("text/sql"); response.setHeader("Content-disposition", "attachment;filename*=utf-8''" + fileName + ".sql"); response.setCharacterEncoding("utf-8"); @@ -130,17 +126,33 @@ public void exportDatabase(@Valid @RequestBody DatabaseExportRequest request, Ht } @PostMapping("/export_data") - public void exportData(@Valid @RequestBody DatabaseExportDataRequest request, HttpServletResponse response) { - Class targetClass = ExportDBDataStrategyFactory.get(request.getExportType()); - response.setCharacterEncoding("utf-8"); - DatabaseExportDataParam param = databaseConverter.request2param(request); + public void exportData(@Valid @RequestBody DatabaseExportDataRequest request, HttpServletResponse response) { try { - Constructor constructor = targetClass.getDeclaredConstructor(); - ExportDBDataStrategy service = (ExportDBDataStrategy) constructor.newInstance(); - service.doExport(param, response); + response.setCharacterEncoding("utf-8"); + DatabaseExportDataParam param = databaseConverter.request2param(request); + DataFileFactoryProducer.getFactory(request.getExportDataOption().getFileType()) + .createExporter().exportDataFile(param, response); } catch (Exception e) { + response.reset(); throw new RuntimeException(e); } + } + @PostMapping("/import_data") + public void importData(@Valid @ModelAttribute(value = "file") MultipartFile file, + @ModelAttribute(value = "json") String jsonData, HttpServletResponse response) { + try { + ObjectMapper objectMapper = new ObjectMapper(); + DatabaseImportDataRequest request = objectMapper.readValue(jsonData, DatabaseImportDataRequest.class); + DatabaseImportDataParam param = databaseConverter.request2param(request); + DataFileFactoryProducer.getFactory(request.getImportDataOption().getFileType()) + .createImporter().importDataFile(param, file); + response.getWriter().println(((LoggingObserver) DataFileFactoryProducer.getObserver()).getLogs()); + } catch (Exception e) { + response.reset(); + throw new RuntimeException(e); + } finally { + DataFileFactoryProducer.removeObserver(); + } } } diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/TableController.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/TableController.java index 23a9f32ec..095432ab0 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/TableController.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/TableController.java @@ -18,8 +18,6 @@ import ai.chat2db.server.web.api.controller.rdb.vo.SqlVO; import ai.chat2db.server.web.api.controller.rdb.vo.TableVO; import ai.chat2db.spi.model.*; -import ai.chat2db.spi.sql.Chat2DBContext; -import ai.chat2db.spi.sql.ConnectInfo; import com.google.common.collect.Lists; import jakarta.validation.Valid; import lombok.extern.slf4j.Slf4j; @@ -78,7 +76,7 @@ public WebPageResult list(@Valid TableBriefQueryRequest request) { // log.info("sync table vector finish"); // }); return WebPageResult.of(tableVOS, tableDTOPageResult.getTotal(), request.getPageNo(), - request.getPageSize()); + request.getPageSize()); } /** @@ -95,9 +93,6 @@ public ListResult tableList(@Valid TableBriefQueryRequest request) } - - - /** * Query the table columns under the current DB * @@ -203,7 +198,7 @@ public DataResult query(@Valid TableDetailQueryRequest request) { */ @PostMapping("/modify/sql") public ListResult modifySql(@Valid @RequestBody TableModifySqlRequest request) { - Table table = rdbWebConverter.tableRequest2param(request.getNewTable()); + Table table = rdbWebConverter.tableRequest2param(request.getNewTable()); table.setSchemaName(request.getSchemaName()); table.setDatabaseName(request.getDatabaseName()); for (TableColumn tableColumn : table.getColumnList()) { @@ -217,12 +212,11 @@ public ListResult modifySql(@Valid @RequestBody TableModifySqlRequest req tableIndex.setDatabaseName(request.getDatabaseName()); } - return tableService.buildSql(rdbWebConverter.tableRequest2param(request.getOldTable()),table) + return tableService.buildSql(rdbWebConverter.tableRequest2param(request.getOldTable()), table) .map(rdbWebConverter::dto2vo); } - /** * Data types supported by the database * @@ -255,4 +249,5 @@ public ActionResult delete(@Valid @RequestBody TableDeleteRequest request) { DropParam dropParam = rdbWebConverter.tableDelete2dropParam(request); return tableService.drop(dropParam); } + } diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/converter/DatabaseConverter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/converter/DatabaseConverter.java index bb9bc4a9a..9f576005b 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/converter/DatabaseConverter.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/converter/DatabaseConverter.java @@ -2,9 +2,11 @@ import ai.chat2db.server.domain.api.param.datasource.DatabaseExportDataParam; import ai.chat2db.server.domain.api.param.datasource.DatabaseExportParam; +import ai.chat2db.server.domain.api.param.datasource.DatabaseImportDataParam; import ai.chat2db.server.web.api.controller.rdb.request.DatabaseCreateRequest; import ai.chat2db.server.web.api.controller.rdb.request.DatabaseExportDataRequest; import ai.chat2db.server.web.api.controller.rdb.request.DatabaseExportRequest; +import ai.chat2db.server.web.api.controller.rdb.request.DatabaseImportDataRequest; import ai.chat2db.spi.model.Database; import org.mapstruct.Mapper; @@ -16,4 +18,8 @@ public abstract class DatabaseConverter { public abstract DatabaseExportParam request2param(DatabaseExportRequest request); public abstract DatabaseExportDataParam request2param(DatabaseExportDataRequest request); + + public abstract DatabaseImportDataParam request2param(DatabaseImportDataRequest request); + + } diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/converter/RdbWebConverter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/converter/RdbWebConverter.java index f9c529a5b..bfa069a4d 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/converter/RdbWebConverter.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/converter/RdbWebConverter.java @@ -256,4 +256,5 @@ public abstract class RdbWebConverter { public abstract TablePageQueryParam schemaReq2page(EsTableSchemaRequest request); public abstract DmlSqlCopyParam dmlRequest2param(DmlSqlCopyRequest request) ; + } diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/AbstractDataFileExporter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/AbstractDataFileExporter.java new file mode 100644 index 000000000..03384bca4 --- /dev/null +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/AbstractDataFileExporter.java @@ -0,0 +1,97 @@ +package ai.chat2db.server.web.api.controller.rdb.data; + +import ai.chat2db.server.domain.api.enums.ExportFileSuffix; +import ai.chat2db.server.domain.api.param.datasource.DatabaseExportDataParam; +import ai.chat2db.server.tools.common.model.rdb.data.option.AbstractExportDataOptions; +import ai.chat2db.server.tools.common.model.rdb.data.option.table.BaseTableOptions; +import ai.chat2db.spi.sql.Chat2DBContext; +import jakarta.servlet.ServletOutputStream; +import jakarta.servlet.http.HttpServletResponse; +import lombok.extern.slf4j.Slf4j; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.sql.Connection; +import java.sql.SQLException; +import java.util.List; +import java.util.Objects; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; + +/** + * @author: zgq + * @date: 2024年04月27日 15:17 + */ +@Slf4j +public abstract class AbstractDataFileExporter implements DataFileExporter { + + + public String suffix; + public String contentType; + + + @Override + public void exportDataFile(DatabaseExportDataParam param, HttpServletResponse response) throws IOException, SQLException { + if (param.getExportTableOptions().size() > 1) { + log.info("export multi table data file"); + exportMultiDataFile(param, response); + } else { + log.info("export single table data file"); + exportSingleDataFile(param, response); + } + log.info("Finished successfully"); + } + + + public void exportSingleDataFile(DatabaseExportDataParam param, HttpServletResponse response) throws SQLException { + BaseTableOptions tableOptions = param.getExportTableOptions().get(0); + String tableName = tableOptions.getTableName(); + List tableColumns = tableOptions.getTableColumns(); + String schemaName = param.getSchemaName(); + setResponseHeaders(tableName, response); + doTableDataExport(response, Chat2DBContext.getConnection(), param.getDatabaseName(), + schemaName, tableName, tableColumns, param.getExportDataOption()); + } + + public void exportMultiDataFile(DatabaseExportDataParam param, HttpServletResponse response) throws IOException, SQLException { + String databaseName = param.getDatabaseName(); + String schemaName = param.getSchemaName(); + setResponseHeaders(databaseName, schemaName, response); + ServletOutputStream outputStream = response.getOutputStream(); + ZipOutputStream zipOutputStream = new ZipOutputStream(outputStream); + for (BaseTableOptions exportTableOption : param.getExportTableOptions()) { + String tableName = exportTableOption.getTableName(); + String fileName = tableName + suffix; + List tableColumns = exportTableOption.getTableColumns(); + zipOutputStream.putNextEntry(new ZipEntry(fileName)); + ByteArrayOutputStream byteOut = doTableDataExport(Chat2DBContext.getConnection(), databaseName, schemaName, + tableName, tableColumns, param.getExportDataOption()); + byteOut.writeTo(zipOutputStream); + zipOutputStream.closeEntry(); + byteOut.close(); + } + } + protected abstract void doTableDataExport(HttpServletResponse response, Connection connection, + String databaseName, String schemaName, String tableName, + List tableColumns, AbstractExportDataOptions exportDataOption) throws SQLException; + + private void setResponseHeaders(String tableName, HttpServletResponse response) { + response.setContentType(contentType); + response.setHeader("Content-Disposition", "attachment;filename=" + tableName + suffix); + } + + + protected abstract ByteArrayOutputStream doTableDataExport(Connection connection, String databaseName, String schemaName, + String tableName, List tableColumns, + AbstractExportDataOptions exportDataOption) throws SQLException; + + private void setResponseHeaders(String databaseName, String schemaName, HttpServletResponse response) { + response.setContentType("application/zip"); + response.setHeader("Content-disposition", "attachment;filename*=utf-8''" + getFileName(databaseName, schemaName) + ExportFileSuffix.ZIP.getSuffix()); + } + + private String getFileName(String databaseName, String schemaName) { + return Objects.isNull(schemaName) ? databaseName : schemaName; + } + +} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/AbstractDataFileImporter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/AbstractDataFileImporter.java new file mode 100644 index 000000000..0225e2a34 --- /dev/null +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/AbstractDataFileImporter.java @@ -0,0 +1,97 @@ +package ai.chat2db.server.web.api.controller.rdb.data; + +import ai.chat2db.server.domain.api.param.datasource.DatabaseImportDataParam; +import ai.chat2db.server.tools.base.enums.JdbcUrlParameterEnum; +import ai.chat2db.server.tools.common.model.rdb.data.option.AbstractImportDataOptions; +import ai.chat2db.server.tools.common.model.rdb.data.option.table.ImportNewTableOptions; +import ai.chat2db.server.tools.common.model.rdb.data.option.table.ImportTableOptions; +import ai.chat2db.server.web.api.controller.rdb.data.observer.LoggingObserver; +import ai.chat2db.spi.model.KeyValue; +import ai.chat2db.spi.sql.Chat2DBContext; +import ai.chat2db.spi.sql.ConnectInfo; +import ai.chat2db.spi.util.JdbcUtils; +import ai.chat2db.spi.util.SqlUtils; +import com.alibaba.druid.DbType; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections.CollectionUtils; +import org.springframework.web.multipart.MultipartFile; + +import java.io.IOException; +import java.sql.BatchUpdateException; +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.List; + +/** + * @author: zgq + * @date: 2024年04月26日 22:30 + */ +@Slf4j +public abstract class AbstractDataFileImporter implements DataFileImporter { + + public static final int BATCH_SIZE = 100; + @Override + public void importDataFile(DatabaseImportDataParam param, MultipartFile file) throws IOException { + configureUrlParameter(); + AbstractImportDataOptions importDataOption = param.getImportDataOption(); + Connection connection = Chat2DBContext.getConnection(); + ImportTableOptions importTableOption = param.getImportTableOption(); + String databaseName = param.getDatabaseName(); + String schemaName = param.getSchemaName(); + String tableName = importTableOption.getTableName(); + createNewTable(connection, importTableOption, tableName); + List tableColumns = importTableOption.getTableColumns(); + List fileColumns = importTableOption.getFileColumns(); + doImportData(connection, databaseName, schemaName, tableName, tableColumns, fileColumns, importDataOption, file); + int errorCounter = ((LoggingObserver) DataFileFactoryProducer.getObserver()).getErrorCounter(); + if (errorCounter > 0) { + DataFileFactoryProducer.notifyInfo(String.format("Finished data file importing with errors: %s", errorCounter)); + } else { + DataFileFactoryProducer.notifyInfo("Finished data file importing"); + + } + } + + + private void configureUrlParameter() { + KeyValue keyValue = new KeyValue(); + keyValue.setKey(JdbcUrlParameterEnum.CONTINUE_BATCH_ON_ERROR.getDescription()); + keyValue.setValue(String.valueOf(true)); + ConnectInfo connectInfo = Chat2DBContext.getConnectInfo(); + connectInfo.setExtendInfo(List.of(keyValue)); + + } + + private void createNewTable(Connection connection, ImportTableOptions importTableOption, String tableName) { + if (importTableOption instanceof ImportNewTableOptions importNewTableOptions) { + DataFileFactoryProducer.notifyInfo(String.format("create new table:%s", tableName)); + String createTableSql = importNewTableOptions.getSql(); + String type = Chat2DBContext.getConnectInfo().getDbType(); + DbType dbType = JdbcUtils.parse2DruidDbType(type); + List sqlList = SqlUtils.parse(createTableSql, dbType); + if (CollectionUtils.isEmpty(sqlList)) { + return; + } + try (Statement statement = connection.createStatement()) { + for (String sql : sqlList) { + statement.addBatch(sql); + } + try { + statement.executeBatch(); + } catch (BatchUpdateException e) { + DataFileFactoryProducer.notifyError("created new table failed"); + throw new RuntimeException(e); + } + DataFileFactoryProducer.notifyInfo("Successfully created new table"); + } catch (SQLException e) { + DataFileFactoryProducer.notifyError("created new table failed"); + throw new RuntimeException(e); + } + } + } + + protected abstract void doImportData(Connection connection, String databaseName, String schemaName, String tableName, + List tableColumns, List fileColumns, + AbstractImportDataOptions importDataOption, MultipartFile file) throws IOException; +} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/DataFileExporter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/DataFileExporter.java new file mode 100644 index 000000000..0751f95d4 --- /dev/null +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/DataFileExporter.java @@ -0,0 +1,17 @@ +package ai.chat2db.server.web.api.controller.rdb.data; + +import ai.chat2db.server.domain.api.param.datasource.DatabaseExportDataParam; +import jakarta.servlet.http.HttpServletResponse; + +import java.io.IOException; +import java.sql.SQLException; + +/** + * @author: zgq + * @date: 2024年04月26日 10:44 + */ +public interface DataFileExporter { + + void exportDataFile(DatabaseExportDataParam param, HttpServletResponse response) throws IOException, SQLException; + +} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/DataFileFactoryProducer.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/DataFileFactoryProducer.java new file mode 100644 index 000000000..1691eb744 --- /dev/null +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/DataFileFactoryProducer.java @@ -0,0 +1,54 @@ +package ai.chat2db.server.web.api.controller.rdb.data; + +import ai.chat2db.server.web.api.controller.rdb.data.csv.CSVImportExportFactory; +import ai.chat2db.server.web.api.controller.rdb.data.json.JSONImportExportFactory; +import ai.chat2db.server.web.api.controller.rdb.data.observer.LoggingObserver; +import ai.chat2db.server.web.api.controller.rdb.data.observer.Observer; +import ai.chat2db.server.web.api.controller.rdb.data.sql.SQLImportExportFactory; +import ai.chat2db.server.web.api.controller.rdb.data.xls.XLSImportExportFactory; +import ai.chat2db.server.web.api.controller.rdb.data.xlsx.XLSXImportExportFactory; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * @author: zgq + * @date: 2024年04月26日 11:51 + */ +public class DataFileFactoryProducer { + + private static final ThreadLocal observerThreadLocal = ThreadLocal.withInitial(LoggingObserver::new); + + public static void removeObserver() { + observerThreadLocal.remove(); + } + + public static void notifyInfo(String log) { + getObserver().notifyInfo(log); + } + + public static void notifyError(String log) { + getObserver().notifyError(log); + } + + public static Observer getObserver() { + return observerThreadLocal.get(); + } + + + public static final Map factories = new HashMap<>(); + + static { + factories.put("CSV", new CSVImportExportFactory()); + factories.put("XLSX", new XLSXImportExportFactory()); + factories.put("XLS", new XLSImportExportFactory()); + factories.put("JSON", new JSONImportExportFactory()); + factories.put("SQL", new SQLImportExportFactory()); + } + + public static DataFileImportExportFactory getFactory(String type) { + return factories.get(type); + } +} + diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/DataFileImportExportFactory.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/DataFileImportExportFactory.java new file mode 100644 index 000000000..2379a1133 --- /dev/null +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/DataFileImportExportFactory.java @@ -0,0 +1,6 @@ +package ai.chat2db.server.web.api.controller.rdb.data; + +public interface DataFileImportExportFactory { + DataFileImporter createImporter(); + DataFileExporter createExporter(); +} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/DataFileImporter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/DataFileImporter.java new file mode 100644 index 000000000..2217e1be0 --- /dev/null +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/DataFileImporter.java @@ -0,0 +1,11 @@ +package ai.chat2db.server.web.api.controller.rdb.data; + +import ai.chat2db.server.domain.api.param.datasource.DatabaseImportDataParam; +import org.springframework.web.multipart.MultipartFile; + +import java.io.IOException; + +public interface DataFileImporter { + + void importDataFile(DatabaseImportDataParam param, MultipartFile file) throws IOException; +} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/csv/CSVExporter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/csv/CSVExporter.java new file mode 100644 index 000000000..9293cdca4 --- /dev/null +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/csv/CSVExporter.java @@ -0,0 +1,23 @@ +package ai.chat2db.server.web.api.controller.rdb.data.csv; + +import ai.chat2db.server.domain.api.enums.ExportFileSuffix; +import ai.chat2db.server.web.api.controller.rdb.data.DataFileExporter; +import ai.chat2db.server.web.api.controller.rdb.data.xlsx.BaseExcelExporter; +import com.alibaba.excel.support.ExcelTypeEnum; + +/** + * @author: zgq + * @date: 2024年04月26日 11:27 + */ +public class CSVExporter extends BaseExcelExporter implements DataFileExporter { + + + public CSVExporter() { + suffix = ExportFileSuffix.CSV.getSuffix(); + contentType = "text/csv"; + } + @Override + protected ExcelTypeEnum getExcelType() { + return ExcelTypeEnum.CSV; + } +} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/csv/CSVImportExportFactory.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/csv/CSVImportExportFactory.java new file mode 100644 index 000000000..c48f6f133 --- /dev/null +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/csv/CSVImportExportFactory.java @@ -0,0 +1,27 @@ +package ai.chat2db.server.web.api.controller.rdb.data.csv; + +import ai.chat2db.server.web.api.controller.rdb.data.DataFileExporter; +import ai.chat2db.server.web.api.controller.rdb.data.DataFileImportExportFactory; +import ai.chat2db.server.web.api.controller.rdb.data.DataFileImporter; + +/** + * @author: zgq + * @date: 2024年04月26日 11:21 + */ +public class CSVImportExportFactory implements DataFileImportExportFactory { + /** + * @return + */ + @Override + public DataFileImporter createImporter() { + return new CSVImporter(); + } + + /** + * @return + */ + @Override + public DataFileExporter createExporter() { + return new CSVExporter(); + } +} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/csv/CSVImporter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/csv/CSVImporter.java new file mode 100644 index 000000000..64efdf731 --- /dev/null +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/csv/CSVImporter.java @@ -0,0 +1,20 @@ +package ai.chat2db.server.web.api.controller.rdb.data.csv; + +import ai.chat2db.server.web.api.controller.rdb.data.DataFileImporter; +import ai.chat2db.server.web.api.controller.rdb.data.xlsx.BaseExcelImporter; +import com.alibaba.excel.support.ExcelTypeEnum; + +/** + * 功能描述 + * + * @author: zgq + * @date: 2024年04月26日 11:26 + */ +public class CSVImporter extends BaseExcelImporter implements DataFileImporter { + + + @Override + protected ExcelTypeEnum getExcelType() { + return ExcelTypeEnum.CSV; + } +} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2CsvStrategy.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2CsvStrategy.java deleted file mode 100644 index d091bf0ca..000000000 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2CsvStrategy.java +++ /dev/null @@ -1,54 +0,0 @@ -package ai.chat2db.server.web.api.controller.rdb.data.export.strategy; - -import ai.chat2db.server.domain.api.enums.ExportFileSuffix; -import ai.chat2db.spi.util.ResultSetUtils; -import com.alibaba.excel.EasyExcel; -import com.alibaba.excel.support.ExcelTypeEnum; - -import java.io.ByteArrayOutputStream; -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.ResultSetMetaData; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Objects; -import java.util.stream.Collectors; - -public class ExportDBData2CsvStrategy extends ExportDBDataStrategy { - - public ExportDBData2CsvStrategy() { - suffix = ExportFileSuffix.CSV.getSuffix(); - contentType = "application/zip"; - } - - @Override - protected ByteArrayOutputStream exportData(Connection connection, String databaseName, String schemaName, String tableName) throws SQLException { - String sql; - if (Objects.isNull(schemaName)) { - sql = String.format("select * from %s", tableName); - } else { - sql = String.format("select * from %s.%s", schemaName, tableName); - } - ByteArrayOutputStream byteOut = new ByteArrayOutputStream(); - try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { - ResultSetMetaData metaData = resultSet.getMetaData(); - List> headList = ResultSetUtils.getRsHeader(resultSet) - .stream() - .map(Collections::singletonList) - .collect(Collectors.toList()); - int columnCount = metaData.getColumnCount(); - List> dataList = new ArrayList<>(); - while (resultSet.next()) { - List row = new ArrayList<>(); - for (int i = 1; i <= columnCount; i++) { - row.add(resultSet.getString(i)); - } - dataList.add(row); - } - EasyExcel.write(byteOut).excelType(ExcelTypeEnum.CSV).sheet(tableName).head(headList).doWrite(dataList); - } - return byteOut; - } -} \ No newline at end of file diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2ExcelStrategy.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2ExcelStrategy.java deleted file mode 100644 index b8fc38ee8..000000000 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2ExcelStrategy.java +++ /dev/null @@ -1,53 +0,0 @@ -package ai.chat2db.server.web.api.controller.rdb.data.export.strategy; - -import ai.chat2db.server.domain.api.enums.ExportFileSuffix; -import ai.chat2db.spi.util.ResultSetUtils; -import com.alibaba.excel.EasyExcel; - -import java.io.ByteArrayOutputStream; -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.ResultSetMetaData; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Objects; -import java.util.stream.Collectors; - -public class ExportDBData2ExcelStrategy extends ExportDBDataStrategy { - - public ExportDBData2ExcelStrategy() { - suffix = ExportFileSuffix.EXCEL.getSuffix(); - contentType = "application/zip"; - } - - @Override - protected ByteArrayOutputStream exportData(Connection connection, String databaseName, String schemaName, String tableName) throws SQLException { - String sql; - if (Objects.isNull(schemaName)) { - sql = String.format("select * from %s", tableName); - } else { - sql = String.format("select * from %s.%s", schemaName, tableName); - } - ByteArrayOutputStream byteOut = new ByteArrayOutputStream(); - try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { - ResultSetMetaData metaData = resultSet.getMetaData(); - int columnCount = metaData.getColumnCount(); - List> headList = ResultSetUtils.getRsHeader(resultSet) - .stream() - .map(Collections::singletonList) - .collect(Collectors.toList()); - List> dataList = new ArrayList<>(); - while (resultSet.next()) { - List row = new ArrayList<>(); - for (int i = 1; i <= columnCount; i++) { - row.add(resultSet.getString(i)); - } - dataList.add(row); - } - EasyExcel.write(byteOut).sheet(tableName).head(headList).doWrite(dataList); - } - return byteOut; - } -} \ No newline at end of file diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2JsonStrategy.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2JsonStrategy.java deleted file mode 100644 index 47c8a59d8..000000000 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2JsonStrategy.java +++ /dev/null @@ -1,61 +0,0 @@ -package ai.chat2db.server.web.api.controller.rdb.data.export.strategy; - -import ai.chat2db.server.domain.api.enums.ExportFileSuffix; -import ai.chat2db.server.tools.base.excption.BusinessException; -import com.fasterxml.jackson.databind.ObjectMapper; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.OutputStreamWriter; -import java.io.PrintWriter; -import java.nio.charset.StandardCharsets; -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.ResultSetMetaData; -import java.sql.SQLException; -import java.util.*; - -public class ExportDBData2JsonStrategy extends ExportDBDataStrategy { - - public ExportDBData2JsonStrategy() { - suffix = ExportFileSuffix.JSON.getSuffix(); - contentType = "application/zip"; - } - - @Override - protected ByteArrayOutputStream exportData(Connection connection, String databaseName, String schemaName, String tableName) throws SQLException { - ByteArrayOutputStream byteOut = new ByteArrayOutputStream(); - try (PrintWriter writer = new PrintWriter(new OutputStreamWriter(byteOut, StandardCharsets.UTF_8))) { - String sql; - if (Objects.isNull(schemaName)) { - sql = String.format("SELECT * FROM %s", tableName); - } else { - sql = String.format("SELECT * FROM %s.%s", schemaName, tableName); - } - try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { - ResultSetMetaData metaData = resultSet.getMetaData(); - int columnCount = metaData.getColumnCount(); - List> data = new ArrayList<>(); - - while (resultSet.next()) { - Map row = new LinkedHashMap<>(); - for (int i = 1; i <= columnCount; i++) { - row.put(metaData.getColumnName(i), resultSet.getObject(i)); - } - data.add(row); - } - - ObjectMapper objectMapper = new ObjectMapper(); - try { - String jsonString = objectMapper.writeValueAsString(data); - writer.println(jsonString); - } catch (IOException e) { - throw new BusinessException("data.export2Json.error",data.toArray(),e); - } - } - } - return byteOut; - } - - -} \ No newline at end of file diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2SqlStrategy.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2SqlStrategy.java deleted file mode 100644 index 1e0577474..000000000 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBData2SqlStrategy.java +++ /dev/null @@ -1,33 +0,0 @@ -package ai.chat2db.server.web.api.controller.rdb.data.export.strategy; - -import ai.chat2db.server.domain.api.enums.ExportFileSuffix; -import ai.chat2db.spi.sql.Chat2DBContext; - -import java.io.ByteArrayOutputStream; -import java.io.OutputStreamWriter; -import java.io.PrintWriter; -import java.nio.charset.StandardCharsets; -import java.sql.Connection; -import java.sql.SQLException; - -/** - * @author: zgq - * @date: 2024年03月24日 12:50 - */ -public class ExportDBData2SqlStrategy extends ExportDBDataStrategy { - - public ExportDBData2SqlStrategy() { - suffix = ExportFileSuffix.SQL.getSuffix(); - contentType = "application/zip"; - } - - @Override - protected ByteArrayOutputStream exportData(Connection connection, String databaseName, String schemaName, String tableName) throws SQLException { - ByteArrayOutputStream byteOut = new ByteArrayOutputStream(); - try (PrintWriter writer = new PrintWriter(new OutputStreamWriter(byteOut, StandardCharsets.UTF_8))) { - String sql = Chat2DBContext.getDBManage().exportDatabaseData(connection, databaseName, schemaName, tableName); - writer.println(sql); - } - return byteOut; - } -} \ No newline at end of file diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBDataStrategy.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBDataStrategy.java deleted file mode 100644 index cb4a18209..000000000 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/export/strategy/ExportDBDataStrategy.java +++ /dev/null @@ -1,65 +0,0 @@ -package ai.chat2db.server.web.api.controller.rdb.data.export.strategy; - -import ai.chat2db.server.domain.api.enums.ExportFileSuffix; -import ai.chat2db.server.domain.api.param.datasource.DatabaseExportDataParam; -import ai.chat2db.spi.sql.Chat2DBContext; -import jakarta.servlet.ServletOutputStream; -import jakarta.servlet.http.HttpServletResponse; -import lombok.AllArgsConstructor; -import lombok.Getter; -import lombok.NoArgsConstructor; - -import java.io.ByteArrayOutputStream; -import java.sql.Connection; -import java.sql.SQLException; -import java.util.List; -import java.util.Objects; -import java.util.zip.ZipEntry; -import java.util.zip.ZipOutputStream; - -/** - * @author: zgq - * @date: 2024年03月24日 12:46 - */ -@Getter -@AllArgsConstructor -@NoArgsConstructor -public abstract class ExportDBDataStrategy { - - public String suffix; - public String contentType; - - public void doExport(DatabaseExportDataParam param, HttpServletResponse response) { - String databaseName = param.getDatabaseName(); - String schemaName = param.getSchemaName(); - setResponseHeaders(param, response); - try (ServletOutputStream outputStream = response.getOutputStream(); - ZipOutputStream zipOut = new ZipOutputStream(outputStream); - Connection connection = Chat2DBContext.getConnection();) { - List tableNames = Chat2DBContext.getMetaData().tableNames(connection, databaseName, schemaName, null); - tableNames.addAll(Chat2DBContext.getMetaData().viewNames(connection, databaseName, schemaName)); - for (String tableName : tableNames) { - String fileName = tableName + getSuffix(); - zipOut.putNextEntry(new ZipEntry(fileName)); - ByteArrayOutputStream byteOut = exportData(connection, databaseName, schemaName, tableName); - byteOut.writeTo(zipOut); - zipOut.closeEntry(); - byteOut.close(); - } - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - private void setResponseHeaders(DatabaseExportDataParam param, HttpServletResponse response) { - response.setContentType(contentType); - response.setHeader("Content-disposition", "attachment;filename*=utf-8''" + getFileName(param) + ExportFileSuffix.ZIP.getSuffix()); - } - - protected String getFileName(DatabaseExportDataParam param) { - return Objects.isNull(param.getSchemaName()) ? param.getDatabaseName() : param.getSchemaName(); - } - - protected abstract ByteArrayOutputStream exportData(Connection connection, String databaseName, String schemaName, String tableName) throws SQLException; - -} \ No newline at end of file diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/json/EasyJsonExportUtil.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/json/EasyJsonExportUtil.java new file mode 100644 index 000000000..e4ed27e16 --- /dev/null +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/json/EasyJsonExportUtil.java @@ -0,0 +1,67 @@ +package ai.chat2db.server.web.api.controller.rdb.data.json; + +import ai.chat2db.server.tools.base.excption.BusinessException; +import ai.chat2db.server.tools.common.model.rdb.data.option.json.ExportData2JsonOptions; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections.CollectionUtils; + +import java.io.IOException; +import java.io.PrintWriter; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * @author: zgq + * @date: 2024年04月26日 14:46 + */ +@Slf4j +public class EasyJsonExportUtil { + + + public static void write(List tableColumns, ExportData2JsonOptions exportDataOption, ResultSet resultSet, PrintWriter writer) throws SQLException { + List> data = EasyJsonExportUtil.getDataMap(resultSet, tableColumns); + ObjectMapper objectMapper = EasyJsonExportUtil.getObjectMapper(exportDataOption); + try { + String jsonString = objectMapper.writeValueAsString(data); + writer.println(jsonString); + } catch (IOException e) { + throw new BusinessException("data.export2Json.error", data.toArray(), e); + } + } + public static List> getDataMap(ResultSet resultSet, List filedNames) throws SQLException { + ResultSetMetaData metaData = resultSet.getMetaData(); + int columnCount = metaData.getColumnCount(); + List> data = new ArrayList<>(); + while (resultSet.next()) { + Map row = new LinkedHashMap<>(); + for (int i = 1; i <= columnCount; i++) { + String columnName = metaData.getColumnName(i); + if (CollectionUtils.isNotEmpty(filedNames) && !filedNames.contains(columnName)) { + continue; + } + row.put(columnName, resultSet.getObject(i)); + } + data.add(row); + } + return data; + } + + public static ObjectMapper getObjectMapper(ExportData2JsonOptions jsonExportDataOption) { + ObjectMapper objectMapper = new ObjectMapper(); + if (!jsonExportDataOption.getIsTimestamps()) { + String dataTimeFormat = jsonExportDataOption.getDataTimeFormat(); + log.info("configure dataTimeFormat:{}", dataTimeFormat); + objectMapper.setDateFormat(new SimpleDateFormat(dataTimeFormat)); + objectMapper.configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, false); + } + return objectMapper; + } +} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/json/JSONExporter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/json/JSONExporter.java new file mode 100644 index 000000000..1e44ec42a --- /dev/null +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/json/JSONExporter.java @@ -0,0 +1,64 @@ +package ai.chat2db.server.web.api.controller.rdb.data.json; + +import ai.chat2db.server.domain.api.enums.ExportFileSuffix; +import ai.chat2db.server.tools.common.model.rdb.data.option.AbstractExportDataOptions; +import ai.chat2db.server.tools.common.model.rdb.data.option.json.ExportData2JsonOptions; +import ai.chat2db.server.web.api.controller.rdb.data.AbstractDataFileExporter; +import ai.chat2db.server.web.api.controller.rdb.data.DataFileExporter; +import ai.chat2db.server.web.api.controller.rdb.data.sql.EasySqlBuilder; +import jakarta.servlet.http.HttpServletResponse; +import lombok.extern.slf4j.Slf4j; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStreamWriter; +import java.io.PrintWriter; +import java.nio.charset.StandardCharsets; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.List; + +/** + * 功能描述 + * + * @author: zgq + * @date: 2024年04月26日 14:20 + */ +@Slf4j +public class JSONExporter extends AbstractDataFileExporter implements DataFileExporter { + + public JSONExporter() { + suffix = ExportFileSuffix.JSON.getSuffix(); + contentType = "application/json"; + } + + @Override + protected void doTableDataExport(HttpServletResponse response, Connection connection, String databaseName, + String schemaName, String tableName, List tableColumns, + AbstractExportDataOptions exportDataOption) throws SQLException { + log.info("Export File Format JSON file"); + String sql = EasySqlBuilder.buildQuerySql(databaseName, schemaName, tableName); + try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { + EasyJsonExportUtil.write(tableColumns, (ExportData2JsonOptions) exportDataOption, resultSet, response.getWriter()); + } catch (IOException e) { + throw new RuntimeException(e); + } + + } + + + @Override + protected ByteArrayOutputStream doTableDataExport(Connection connection, String databaseName, String schemaName, + String tableName, List tableColumns, + AbstractExportDataOptions exportDataOption) throws SQLException { + log.info("Export File Format JSON file"); + ByteArrayOutputStream byteOut = new ByteArrayOutputStream(); + String sql = EasySqlBuilder.buildQuerySql(databaseName, schemaName, tableName); + try (ResultSet resultSet = connection.createStatement().executeQuery(sql); + PrintWriter writer = new PrintWriter(new OutputStreamWriter(byteOut, StandardCharsets.UTF_8))) { + EasyJsonExportUtil.write(tableColumns, (ExportData2JsonOptions) exportDataOption, resultSet, writer); + } + return byteOut; + } +} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/json/JSONImportExportFactory.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/json/JSONImportExportFactory.java new file mode 100644 index 000000000..db2a63782 --- /dev/null +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/json/JSONImportExportFactory.java @@ -0,0 +1,23 @@ +package ai.chat2db.server.web.api.controller.rdb.data.json; + +import ai.chat2db.server.web.api.controller.rdb.data.DataFileExporter; +import ai.chat2db.server.web.api.controller.rdb.data.DataFileImportExportFactory; +import ai.chat2db.server.web.api.controller.rdb.data.DataFileImporter; + +/** + * @author: zgq + * @date: 2024年04月26日 14:22 + */ +public class JSONImportExportFactory implements DataFileImportExportFactory { + + @Override + public DataFileImporter createImporter() { + return new JSONImporter(); + } + + + @Override + public DataFileExporter createExporter() { + return new JSONExporter(); + } +} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/json/JSONImporter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/json/JSONImporter.java new file mode 100644 index 000000000..89c9c5493 --- /dev/null +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/json/JSONImporter.java @@ -0,0 +1,114 @@ +package ai.chat2db.server.web.api.controller.rdb.data.json; + +import ai.chat2db.server.tools.base.excption.BusinessException; +import ai.chat2db.server.tools.common.model.rdb.data.option.AbstractImportDataOptions; +import ai.chat2db.server.tools.common.model.rdb.data.option.json.ImportJsonDataOptions; +import ai.chat2db.server.web.api.controller.rdb.data.AbstractDataFileImporter; +import ai.chat2db.server.web.api.controller.rdb.data.DataFileImporter; +import ai.chat2db.server.web.api.controller.rdb.data.sql.EasySqlBuilder; +import ai.chat2db.server.web.api.controller.rdb.data.util.EasyBatchSqlExecutor; +import cn.hutool.core.date.DatePattern; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.jetbrains.annotations.NotNull; +import org.springframework.web.multipart.MultipartFile; + +import java.sql.Connection; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Objects; + +/** + * @author: zgq + * @date: 2024年04月26日 14:20 + */ +@Slf4j +public class JSONImporter extends AbstractDataFileImporter implements DataFileImporter { + + @Override + protected void doImportData(Connection connection, String databaseName, String schemaName, String tableName, List tableColumns, + List fileColumns, AbstractImportDataOptions importDataOption, MultipartFile file) { + log.info("import JSON data file"); + String rootNodeName = ((ImportJsonDataOptions) importDataOption).getRootNodeName(); + String dataTimeFormat = ((ImportJsonDataOptions) importDataOption).getDataTimeFormat(); + Integer dataStartRowNum = ((ImportJsonDataOptions) importDataOption).getDataStartRowNum(); + Integer dataEndRowNum = ((ImportJsonDataOptions) importDataOption).getDataEndRowNum(); + int limitRowSize = dataEndRowNum - dataStartRowNum + 1; + List sqlCacheList = new ArrayList<>(BATCH_SIZE); + int recordCount = 0; + ObjectMapper objectMapper = new ObjectMapper(); + StringBuilder sqlBuilder = new StringBuilder(); + try { + JsonNode jsonNode = objectMapper.readTree(file.getInputStream()); + jsonNode = getJsonNode(rootNodeName, jsonNode); + Iterator records = jsonNode.elements(); + List values = new ArrayList<>(); + while (records.hasNext() && recordCount++ < limitRowSize) { + JsonNode recordNode = records.next(); + iteratorValues(fileColumns, dataTimeFormat, values, recordNode); + EasySqlBuilder.buildInsert(databaseName, schemaName, tableName, true, fileColumns, sqlBuilder); + EasySqlBuilder.buildInsertValues(values, sqlBuilder); + sqlBuilder.append(";"); + values.clear(); + sqlCacheList.add(sqlBuilder.toString()); + sqlBuilder.setLength(0); + if (sqlCacheList.size() >= BATCH_SIZE) { + EasyBatchSqlExecutor.executeBatchInsert(connection, sqlCacheList); + } + } + if (sqlCacheList.size() > 0) { + EasyBatchSqlExecutor.executeBatchInsert(connection, sqlCacheList); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + + } + + private void iteratorValues(List fileColumns, String dataTimeFormat, + List values, JsonNode recordNode) throws Exception { + for (String columnName : fileColumns) { + JsonNode columnValueNode = recordNode.get(columnName); + if (Objects.isNull(columnValueNode)) { + values.add("NULL"); + } else { + String value = columnValueNode.asText(); + if (isValidDate(value, dataTimeFormat)) { + value = formatDate(value, dataTimeFormat); + } + values.add(value); + } + } + } + + @NotNull + private JsonNode getJsonNode(String rootNodeName, JsonNode jsonNode) { + if (StringUtils.isNotBlank(rootNodeName) && jsonNode.has(rootNodeName)) { + jsonNode = jsonNode.get(rootNodeName); + } + if (!jsonNode.isArray() || jsonNode.size() <= 0) { + throw new BusinessException("jsonFile.parse.error"); + } + return jsonNode; + } + + private static boolean isValidDate(String dateString, String dataTimeFormat) { + try { + SimpleDateFormat sdf = new SimpleDateFormat(dataTimeFormat); + sdf.parse(dateString); + return true; + } catch (Exception e) { + return false; + } + } + + private static String formatDate(String dateString, String dataTimeFormat) throws Exception { + SimpleDateFormat inputFormat = new SimpleDateFormat(dataTimeFormat); + SimpleDateFormat outputFormat = new SimpleDateFormat(DatePattern.NORM_DATETIME_PATTERN); + return outputFormat.format(inputFormat.parse(dateString)); + } +} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/observer/LoggingObserver.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/observer/LoggingObserver.java new file mode 100644 index 000000000..7f9e325d9 --- /dev/null +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/observer/LoggingObserver.java @@ -0,0 +1,37 @@ +package ai.chat2db.server.web.api.controller.rdb.data.observer; + +import cn.hutool.core.date.DatePattern; +import lombok.Getter; + +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.List; + +@Getter +public class LoggingObserver implements Observer { + private final List logs = new ArrayList<>(); + private int errorCounter = 0; + + @Override + public void notifyInfo(String log) { + logs.add(String.format("[%s] [Info]:%s", getTime(), log)); + } + + + @Override + public void notifyError(String log) { + errorCounter++; + logs.add("[" + getTime() + "] [Error]:execute sql failed, error sql:"+log+""); + } + + public String getLogs() { + return String.join("\n", logs); + } + + private String getTime() { + LocalDateTime now = LocalDateTime.now(); + DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SS"); + return now.format(dateTimeFormatter); + } +} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/observer/Observer.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/observer/Observer.java new file mode 100644 index 000000000..4ad55ce9f --- /dev/null +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/observer/Observer.java @@ -0,0 +1,6 @@ +package ai.chat2db.server.web.api.controller.rdb.data.observer; + +public interface Observer { + void notifyInfo(String log); + void notifyError(String log); +} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/sql/EasySqlBuilder.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/sql/EasySqlBuilder.java new file mode 100644 index 000000000..33cb60c9c --- /dev/null +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/sql/EasySqlBuilder.java @@ -0,0 +1,203 @@ +package ai.chat2db.server.web.api.controller.rdb.data.sql; + +import ai.chat2db.server.tools.common.model.rdb.data.option.AbstractExportDataOptions; +import ai.chat2db.server.tools.common.model.rdb.data.option.sql.BaseExportData2SqlOptions; +import ai.chat2db.spi.util.ResultSetUtils; +import jakarta.validation.constraints.NotBlank; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; + +import java.io.PrintWriter; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; + +/** + * @author: zgq + * @date: 2024年04月26日 12:46 + */ +@Slf4j +public class EasySqlBuilder { + + public static String buildQuerySql(String databaseName, String schemaName, String tableName) { + StringBuilder sqlBuilder = new StringBuilder("SELECT * FROM "); + buildTableName(databaseName, schemaName, tableName, sqlBuilder); + return sqlBuilder.toString(); + } + + private static void buildTableName(String databaseName, String schemaName, String tableName, StringBuilder sqlBuilder) { + if (StringUtils.isNotBlank(databaseName)) { + sqlBuilder.append(databaseName).append("."); + } + + if (StringUtils.isNotBlank(schemaName)) { + sqlBuilder.append(schemaName).append("."); + } + + if (StringUtils.isNotBlank(tableName)) { + sqlBuilder.append(tableName); + } + } + + public static void exportData2Sql(String databaseName, String schemaName, String tableName, List tableColumns, AbstractExportDataOptions exportDataOption, + ResultSet resultSet, PrintWriter writer) throws SQLException { + ResultSetMetaData metaData = resultSet.getMetaData(); + List headList = ResultSetUtils.getRsHeader(resultSet); + if (tableColumns.size() != headList.size()) { + headList = tableColumns; + } + String sqlType = ((BaseExportData2SqlOptions) exportDataOption).getSqlType(); + switch (sqlType) { + case "single" -> { + log.info("Exporting single insert SQL for table:" + tableName); + buildSingleInsert(writer, resultSet, metaData, databaseName, schemaName, tableName, headList, exportDataOption); + } + case "multi" -> { + log.info("Exporting multi insert SQL for table: " + tableName); + buildMultiInsert(writer, resultSet, metaData, databaseName, schemaName, tableName, headList, exportDataOption); + } + case "update" -> { + log.info("Exporting update SQL for table: " + tableName); + buildUpdateStatement(writer, resultSet, metaData, databaseName, schemaName, tableName, headList, exportDataOption); + } + default -> throw new IllegalStateException("Unexpected value: " + sqlType); + } + } + + public static void buildUpdateStatement(PrintWriter writer, ResultSet resultSet, ResultSetMetaData metaData, + String databaseName, String schemaName, String tableName, List headList, AbstractExportDataOptions exportDataOption) throws SQLException { + String updateCondition = ((BaseExportData2SqlOptions) exportDataOption).getUpdateCondition(); + StringBuilder sqlBuilder = new StringBuilder(); + while (resultSet.next()) { + EasySqlBuilder.buildUpdate(databaseName, schemaName, tableName, sqlBuilder); + int columnCount = 0; + for (int i = 1; i <= metaData.getColumnCount(); i++) { + String columnName = metaData.getColumnName(i); + if (!headList.contains(columnName) || Objects.equals(updateCondition, columnName)) { + continue; + } + columnCount++; + String value = resultSet.getString(i); + if (Objects.isNull(value)) { + sqlBuilder.append(columnName).append(" = NULL"); + } else { + sqlBuilder.append(columnName).append(" = '").append(value).append("'"); + } + + if (columnCount < headList.size() - 1) { + sqlBuilder.append(", "); + + } + } + EasySqlBuilder.buildUpdateConditions(updateCondition, resultSet.getString(updateCondition), sqlBuilder); + } + writer.println(sqlBuilder); + } + + public static void buildMultiInsert(PrintWriter writer, ResultSet resultSet, ResultSetMetaData metaData, + String databaseName, String schemaName, String tableName, List headList, AbstractExportDataOptions exportDataOption) throws SQLException { + Boolean containsHeader = ((BaseExportData2SqlOptions) exportDataOption).getContainsHeader(); + StringBuilder sqlBuilder = new StringBuilder(); + List values = new ArrayList<>(); + EasySqlBuilder.buildInsert(databaseName, schemaName, tableName, containsHeader, headList, sqlBuilder); + while (resultSet.next()) { + int columnCount = 0; + for (int i = 1; i <= metaData.getColumnCount(); i++) { + String columnName = metaData.getColumnName(i); + if (!headList.contains(columnName)) { + continue; + } + columnCount++; + String value = resultSet.getString(i); + if (Objects.isNull(value)) { + values.add(null); + } else { + values.add(value); + } + if (columnCount == headList.size()) { + break; + } + } + EasySqlBuilder.buildInsertValues(values, sqlBuilder); + if (resultSet.isLast()) { + sqlBuilder.append(";\n"); + break; + } else { + sqlBuilder.append(",\n"); + } + values.clear(); + } + writer.println(sqlBuilder); + } + + public static void buildSingleInsert(PrintWriter writer, ResultSet resultSet, ResultSetMetaData metaData, + String databaseName, String schemaName, String tableName, List headList, AbstractExportDataOptions exportDataOption) throws SQLException { + Boolean containsHeader = ((BaseExportData2SqlOptions) exportDataOption).getContainsHeader(); + StringBuilder sqlBuilder = new StringBuilder(); + List values = new ArrayList<>(); + while (resultSet.next()) { + int columnCount = 0; + for (int i = 1; i <= metaData.getColumnCount(); i++) { + String columnName = metaData.getColumnName(i); + if (!headList.contains(columnName)) { + continue; + } + columnCount++; + String value = resultSet.getString(i); + if (Objects.isNull(value)) { + values.add(null); + } else { + values.add(value); + } + if (columnCount == headList.size()) { + break; + } + } + EasySqlBuilder.buildInsert(databaseName, schemaName, tableName, containsHeader, headList, sqlBuilder); + EasySqlBuilder.buildInsertValues(values, sqlBuilder); + sqlBuilder.append(";\n"); + values.clear(); + } + writer.println(sqlBuilder); + } + + public static void buildInsert(String databaseName, String schemaName, @NotBlank String tableName, boolean containsHeader, List fieldNames, StringBuilder sqlBuilder) { + sqlBuilder.append("INSERT INTO "); + buildTableName(databaseName, schemaName, tableName, sqlBuilder); + + if (containsHeader) { + sqlBuilder.append(" ").append(buildColumns(fieldNames)); + } + sqlBuilder.append(" VALUES "); + } + + + public static void buildInsertValues(List values, StringBuilder sqlBuilder) { + sqlBuilder.append(buildValues(values)); + } + + public static void buildUpdate(String databaseName, String schemaName, @NotBlank String tableName, StringBuilder sqlBuilder) { + sqlBuilder.append("UPDATE "); + buildTableName(databaseName, schemaName, tableName, sqlBuilder); + sqlBuilder.append(" SET "); + } + + public static void buildUpdateConditions(String updateCondition, String value, StringBuilder sqlBuilder) { + sqlBuilder.append(" WHERE ").append(updateCondition).append(" = ").append("'").append(value).append("'").append(";\n"); + } + + public static String buildValues(List list) { + return list.stream() + .map(s -> s == null ? null : "'" + s + "'") + .collect(Collectors.joining(",", "(", ")")); + } + + public static String buildColumns(List headers) { + return headers.stream() + .collect(Collectors.joining(",", "(", ")")); + } +} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/sql/SQLExporter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/sql/SQLExporter.java new file mode 100644 index 000000000..6da8c5b6d --- /dev/null +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/sql/SQLExporter.java @@ -0,0 +1,60 @@ +package ai.chat2db.server.web.api.controller.rdb.data.sql; + +import ai.chat2db.server.domain.api.enums.ExportFileSuffix; +import ai.chat2db.server.tools.common.model.rdb.data.option.AbstractExportDataOptions; +import ai.chat2db.server.web.api.controller.rdb.data.AbstractDataFileExporter; +import ai.chat2db.server.web.api.controller.rdb.data.DataFileExporter; +import jakarta.servlet.http.HttpServletResponse; +import lombok.extern.slf4j.Slf4j; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStreamWriter; +import java.io.PrintWriter; +import java.nio.charset.StandardCharsets; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.List; + +/** + * @author: zgq + * @date: 2024年04月26日 15:33 + */ +@Slf4j +public class SQLExporter extends AbstractDataFileExporter implements DataFileExporter { + + public SQLExporter() { + suffix = ExportFileSuffix.SQL.getSuffix(); + contentType = "text/sql"; + } + + @Override + protected void doTableDataExport(HttpServletResponse response, Connection connection, + String databaseName, String schemaName, + String tableName, List tableColumns, + AbstractExportDataOptions exportDataOption) throws SQLException { + String sql = EasySqlBuilder.buildQuerySql(databaseName, schemaName, tableName); + log.info("Export File Format SQL file"); + try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { + PrintWriter writer = response.getWriter(); + EasySqlBuilder.exportData2Sql(databaseName,schemaName,tableName, tableColumns, exportDataOption, resultSet, writer); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + protected ByteArrayOutputStream doTableDataExport(Connection connection, String databaseName, String schemaName, + String tableName, List tableColumns, + AbstractExportDataOptions exportDataOption) throws SQLException { + log.info("Export File Format SQL file"); + ByteArrayOutputStream byteOut = new ByteArrayOutputStream(); + String sql = EasySqlBuilder.buildQuerySql(databaseName, schemaName, tableName); + try (ResultSet resultSet = connection.createStatement().executeQuery(sql); + PrintWriter writer = new PrintWriter(new OutputStreamWriter(byteOut, StandardCharsets.UTF_8))) { + EasySqlBuilder.exportData2Sql(databaseName, schemaName, tableName, tableColumns, exportDataOption, resultSet, writer); + } + return byteOut; + } +} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/sql/SQLImportExportFactory.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/sql/SQLImportExportFactory.java new file mode 100644 index 000000000..a7795ebcb --- /dev/null +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/sql/SQLImportExportFactory.java @@ -0,0 +1,29 @@ +package ai.chat2db.server.web.api.controller.rdb.data.sql; + +import ai.chat2db.server.web.api.controller.rdb.data.DataFileExporter; +import ai.chat2db.server.web.api.controller.rdb.data.DataFileImportExportFactory; +import ai.chat2db.server.web.api.controller.rdb.data.DataFileImporter; + +/** + * 功能描述 + * + * @author: zgq + * @date: 2024年04月26日 15:32 + */ +public class SQLImportExportFactory implements DataFileImportExportFactory { + /** + * @return + */ + @Override + public DataFileImporter createImporter() { + return new SQLImporter(); + } + + /** + * @return + */ + @Override + public DataFileExporter createExporter() { + return new SQLExporter(); + } +} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/sql/SQLImporter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/sql/SQLImporter.java new file mode 100644 index 000000000..fd5558b70 --- /dev/null +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/sql/SQLImporter.java @@ -0,0 +1,25 @@ +package ai.chat2db.server.web.api.controller.rdb.data.sql; + +import ai.chat2db.server.tools.common.model.rdb.data.option.AbstractImportDataOptions; +import ai.chat2db.server.web.api.controller.rdb.data.AbstractDataFileImporter; +import ai.chat2db.server.web.api.controller.rdb.data.DataFileImporter; +import org.springframework.web.multipart.MultipartFile; + +import java.sql.Connection; +import java.util.List; + +/** + * 功能描述 + * + * @author: zgq + * @date: 2024年04月26日 15:33 + */ +public class SQLImporter extends AbstractDataFileImporter implements DataFileImporter { + + @Override + protected void doImportData(Connection connection, String databaseName, String schemaName, String tableName, + List tableColumns, List fileColumns, + AbstractImportDataOptions importDataOption, MultipartFile file) { + + } +} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/util/EasyBatchSqlExecutor.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/util/EasyBatchSqlExecutor.java new file mode 100644 index 000000000..bd14ac01f --- /dev/null +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/util/EasyBatchSqlExecutor.java @@ -0,0 +1,46 @@ +package ai.chat2db.server.web.api.controller.rdb.data.util; + +import ai.chat2db.server.web.api.controller.rdb.data.DataFileFactoryProducer; + +import java.sql.BatchUpdateException; +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.List; + +/** + * @author: zgq + * @date: 2024年05月08日 12:30 + */ +public class EasyBatchSqlExecutor { + public static void executeBatchInsert(Connection connection, List sqlCacheList) { + try (Statement stmt = connection.createStatement()) { + for (String sql : sqlCacheList) { + stmt.addBatch(sql); + } + connection.setAutoCommit(false); + try { + stmt.executeBatch(); + connection.commit(); + } catch (BatchUpdateException e) { + connection.rollback(); + handleErrorSql(stmt,sqlCacheList); + } + stmt.clearBatch(); + sqlCacheList.clear(); + } catch (SQLException e) { + throw new RuntimeException(e); + } + } + + public static void handleErrorSql(Statement stmt, List sqlCacheList) { + for (String sql : sqlCacheList) { + try { + stmt.execute(sql); + } catch (SQLException ex) { + DataFileFactoryProducer.notifyError(sql + "error msg:" + ex.getMessage() + ""); + } + } + } + +} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xls/XLSExporter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xls/XLSExporter.java new file mode 100644 index 000000000..14e541f6c --- /dev/null +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xls/XLSExporter.java @@ -0,0 +1,22 @@ +package ai.chat2db.server.web.api.controller.rdb.data.xls; + +import ai.chat2db.server.domain.api.enums.ExportFileSuffix; +import ai.chat2db.server.web.api.controller.rdb.data.DataFileExporter; +import ai.chat2db.server.web.api.controller.rdb.data.xlsx.BaseExcelExporter; +import com.alibaba.excel.support.ExcelTypeEnum; + +/** + * @author: zgq + * @date: 2024年04月29日 21:48 + */ +public class XLSExporter extends BaseExcelExporter implements DataFileExporter { + + public XLSExporter() { + suffix = ExportFileSuffix.XLS.getSuffix(); + contentType = "application/vnd.ms-excel"; + } + @Override + protected ExcelTypeEnum getExcelType() { + return ExcelTypeEnum.XLS; + } +} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xls/XLSImportExportFactory.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xls/XLSImportExportFactory.java new file mode 100644 index 000000000..e95c10855 --- /dev/null +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xls/XLSImportExportFactory.java @@ -0,0 +1,24 @@ +package ai.chat2db.server.web.api.controller.rdb.data.xls; + +import ai.chat2db.server.web.api.controller.rdb.data.DataFileExporter; +import ai.chat2db.server.web.api.controller.rdb.data.DataFileImportExportFactory; +import ai.chat2db.server.web.api.controller.rdb.data.DataFileImporter; + +/** + * @author: zgq + * @date: 2024年04月29日 21:48 + */ +public class XLSImportExportFactory implements DataFileImportExportFactory { + + + @Override + public DataFileImporter createImporter() { + return new XLSImporter(); + } + + + @Override + public DataFileExporter createExporter() { + return new XLSExporter(); + } +} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xls/XLSImporter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xls/XLSImporter.java new file mode 100644 index 000000000..bca4ba4a1 --- /dev/null +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xls/XLSImporter.java @@ -0,0 +1,17 @@ +package ai.chat2db.server.web.api.controller.rdb.data.xls; + +import ai.chat2db.server.web.api.controller.rdb.data.DataFileImporter; +import ai.chat2db.server.web.api.controller.rdb.data.xlsx.BaseExcelImporter; +import com.alibaba.excel.support.ExcelTypeEnum; + +/** + * @author: zgq + * @date: 2024年04月29日 21:48 + */ +public class XLSImporter extends BaseExcelImporter implements DataFileImporter { + + @Override + protected ExcelTypeEnum getExcelType() { + return ExcelTypeEnum.XLS; + } +} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/BaseExcelExporter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/BaseExcelExporter.java new file mode 100644 index 000000000..e8472552e --- /dev/null +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/BaseExcelExporter.java @@ -0,0 +1,65 @@ +package ai.chat2db.server.web.api.controller.rdb.data.xlsx; + +import ai.chat2db.server.tools.common.model.rdb.data.option.AbstractExportDataOptions; +import ai.chat2db.server.web.api.controller.rdb.data.AbstractDataFileExporter; +import ai.chat2db.server.web.api.controller.rdb.data.sql.EasySqlBuilder; +import ai.chat2db.spi.util.ResultSetUtils; +import com.alibaba.excel.support.ExcelTypeEnum; +import jakarta.servlet.ServletOutputStream; +import jakarta.servlet.http.HttpServletResponse; +import lombok.extern.slf4j.Slf4j; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.List; + +/** + * @author: zgq + * @date: 2024年04月27日 15:23 + */ +@Slf4j +public abstract class BaseExcelExporter extends AbstractDataFileExporter { + + + @Override + protected void doTableDataExport(HttpServletResponse response, Connection connection, String databaseName, + String schemaName, String tableName, List tableColumns, + AbstractExportDataOptions exportDataOption) throws SQLException { + ExcelTypeEnum excelType = getExcelType(); + log.info("Export File Format {} file", excelType.name()); + String sql = EasySqlBuilder.buildQuerySql(databaseName, schemaName, tableName); + try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { + ServletOutputStream outputStream = response.getOutputStream(); + EasyExcelExportUtil.write(outputStream, EasyExcelExportUtil.getDataList(resultSet, tableColumns), + tableName, ResultSetUtils.getRsHeader(resultSet), + tableColumns, excelType, exportDataOption); + } catch (IOException e) { + throw new RuntimeException(e); + } + + } + + + @Override + protected ByteArrayOutputStream doTableDataExport(Connection connection, String databaseName, String schemaName, + String tableName, List tableColumns, + AbstractExportDataOptions exportDataOption) throws SQLException { + ExcelTypeEnum excelType = getExcelType(); + log.info("Export File Format {} file", excelType.name()); + ByteArrayOutputStream byteOut = new ByteArrayOutputStream(); + String sql = EasySqlBuilder.buildQuerySql(databaseName, schemaName, tableName); + try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { + EasyExcelExportUtil.write(byteOut, EasyExcelExportUtil.getDataList(resultSet, tableColumns), + tableName, ResultSetUtils.getRsHeader(resultSet), + tableColumns, excelType, exportDataOption); + return byteOut; + } + } + + + protected abstract ExcelTypeEnum getExcelType(); + +} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/BaseExcelImporter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/BaseExcelImporter.java new file mode 100644 index 000000000..af578d356 --- /dev/null +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/BaseExcelImporter.java @@ -0,0 +1,40 @@ +package ai.chat2db.server.web.api.controller.rdb.data.xlsx; + +import ai.chat2db.server.tools.common.model.rdb.data.option.AbstractImportDataOptions; +import ai.chat2db.server.tools.common.model.rdb.data.option.BaseImportExcelDataOptions; +import ai.chat2db.server.web.api.controller.rdb.data.AbstractDataFileImporter; +import ai.chat2db.server.web.api.controller.rdb.data.DataFileFactoryProducer; +import com.alibaba.excel.EasyExcel; +import com.alibaba.excel.support.ExcelTypeEnum; +import lombok.extern.slf4j.Slf4j; +import org.springframework.web.multipart.MultipartFile; + +import java.io.IOException; +import java.sql.Connection; +import java.util.List; + +/** + * @author: zgq + * @date: 2024年04月27日 11:16 + */ +@Slf4j +public abstract class BaseExcelImporter extends AbstractDataFileImporter { + @Override + protected void doImportData(Connection connection, String databaseName, String schemaName, String tableName, + List tableColumns, List fileColumns, + AbstractImportDataOptions importDataOption, MultipartFile file) throws IOException { + ExcelTypeEnum excelType = getExcelType(); + NoModelDataListener noModelDataListener = new NoModelDataListener(databaseName,schemaName, tableName, tableColumns, + fileColumns, importDataOption, connection); + Integer headerRowNum = ((BaseImportExcelDataOptions) importDataOption).getHeaderRowNum(); + DataFileFactoryProducer.notifyInfo(String.format("import %s data file start", excelType.name())); + EasyExcel.read(file.getInputStream(), noModelDataListener) + .excelType(excelType) + .sheet() + .headRowNumber(headerRowNum) + .doRead(); + + } + + protected abstract ExcelTypeEnum getExcelType(); +} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/EasyExcelExportUtil.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/EasyExcelExportUtil.java new file mode 100644 index 000000000..ab571fecb --- /dev/null +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/EasyExcelExportUtil.java @@ -0,0 +1,66 @@ +package ai.chat2db.server.web.api.controller.rdb.data.xlsx; + +import ai.chat2db.server.tools.common.model.rdb.data.option.AbstractExportDataOptions; +import ai.chat2db.server.tools.common.model.rdb.data.option.BaseExportDataOptions; +import com.alibaba.excel.EasyExcel; +import com.alibaba.excel.support.ExcelTypeEnum; +import com.alibaba.excel.write.builder.ExcelWriterSheetBuilder; +import org.jetbrains.annotations.NotNull; + +import java.io.OutputStream; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +/** + * @author: zgq + * @date: 2024年04月26日 13:31 + */ +public class EasyExcelExportUtil { + + public static void write(OutputStream out, List> dataList, String fileName, + List srcHeaders,List targetHeaders ,ExcelTypeEnum type, + AbstractExportDataOptions exportDataOption) { + Boolean containsHeader = ((BaseExportDataOptions) exportDataOption).getContainsHeader(); + ExcelWriterSheetBuilder excelWriterSheetBuilder = EasyExcel.write(out).excelType(type).sheet(fileName); + if (containsHeader) { + if (srcHeaders.size() != targetHeaders.size()) { + excelWriterSheetBuilder.head(getListHeadList(targetHeaders)); + } else { + excelWriterSheetBuilder.head(getListHeadList(srcHeaders)); + } + } + excelWriterSheetBuilder.doWrite(dataList); + + } + + @NotNull + public static List> getListHeadList(List headers) { + return headers + .stream() + .map(Collections::singletonList) + .collect(Collectors.toList()); + } + + @NotNull + public static List> getDataList(ResultSet resultSet, List fileNames) throws SQLException { + ResultSetMetaData metaData = resultSet.getMetaData(); + int columnCount = metaData.getColumnCount(); + List> dataList = new ArrayList<>(); + while (resultSet.next()) { + List row = new ArrayList<>(); + for (int i = 1; i <= columnCount; i++) { + if (fileNames.size() != columnCount && !fileNames.contains(metaData.getColumnName(i))) { + continue; + } + row.add(resultSet.getString(i)); + } + dataList.add(row); + } + return dataList; + } +} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/NoModelDataListener.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/NoModelDataListener.java new file mode 100644 index 000000000..896f367c3 --- /dev/null +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/NoModelDataListener.java @@ -0,0 +1,110 @@ +package ai.chat2db.server.web.api.controller.rdb.data.xlsx; + +import ai.chat2db.server.tools.common.model.rdb.data.option.AbstractImportDataOptions; +import ai.chat2db.server.tools.common.model.rdb.data.option.BaseImportExcelDataOptions; +import ai.chat2db.server.web.api.controller.rdb.data.DataFileFactoryProducer; +import ai.chat2db.server.web.api.controller.rdb.data.sql.EasySqlBuilder; +import ai.chat2db.server.web.api.controller.rdb.data.util.EasyBatchSqlExecutor; +import com.alibaba.excel.context.AnalysisContext; +import com.alibaba.excel.event.AnalysisEventListener; +import com.alibaba.excel.metadata.data.ReadCellData; +import com.alibaba.excel.util.ConverterUtils; +import com.alibaba.excel.util.ListUtils; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; + +import java.sql.BatchUpdateException; +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +@Slf4j +public class NoModelDataListener extends AnalysisEventListener> { + + private final String databaseName; + private final String schemaName; + private final String tableName; + private final List tableColumns; + private final List fileColumns; + private final Connection connection; + private final int BATCH_SIZE = 100; + private final List sqlCacheList = ListUtils.newArrayListWithExpectedSize(BATCH_SIZE); + private final List columnIndexList; + private int recordIndex = 0; + private final int limitRowSize; + + + public NoModelDataListener(String databaseName, String schemaName, String tableName, List tableColumns, + List fileColumns, AbstractImportDataOptions importDataOption, Connection connection) { + this.databaseName = databaseName; + this.schemaName = schemaName; + this.tableName = tableName; + this.tableColumns = tableColumns; + this.fileColumns = fileColumns; + this.connection = connection; + this.columnIndexList = ListUtils.newArrayListWithExpectedSize(fileColumns.size()); + this.limitRowSize = ((BaseImportExcelDataOptions) importDataOption).getDataEndRowNum() + - ((BaseImportExcelDataOptions) importDataOption).getDataStartRowNum() + 1; + + } + + @Override + public boolean hasNext(AnalysisContext context) { + if (recordIndex++ >= limitRowSize) { + DataFileFactoryProducer.notifyInfo(String.format("read %s records，stopping reading", limitRowSize)); + if (sqlCacheList.size() > 0) { + EasyBatchSqlExecutor.executeBatchInsert(connection, sqlCacheList); + } + return false; + } + return super.hasNext(context); + } + + @Override + public void invokeHead(Map> headMap, AnalysisContext context) { + Map stringMap = ConverterUtils.convertToStringMap(headMap, context); + for (Map.Entry entry : stringMap.entrySet()) { + String value = entry.getValue(); + if (fileColumns.contains(value)) { + Integer key = entry.getKey(); + columnIndexList.add(key); + } + } + } + + @Override + public void invoke(Map data, AnalysisContext context) { + List values = new ArrayList<>(); + for (int i = 0; i < data.size(); i++) { + if (!columnIndexList.contains(i)) { + continue; + } + String value = data.get(i); + if (Objects.isNull(value)) { + values.add(null); + } else { + values.add(value); + } + } + String sql = String.format("INSERT INTO %s.%s %s VALUES %s", + StringUtils.isBlank(databaseName) ? schemaName : databaseName, tableName, + EasySqlBuilder.buildColumns(tableColumns), + EasySqlBuilder.buildValues(values)); + values.clear(); + sqlCacheList.add(sql); + if (sqlCacheList.size() >= BATCH_SIZE) { + EasyBatchSqlExecutor.executeBatchInsert(connection, sqlCacheList); + } + } + @Override + public void doAfterAllAnalysed(AnalysisContext context) { + if (sqlCacheList.size() > 0) { + EasyBatchSqlExecutor.executeBatchInsert(connection, sqlCacheList); + } + } +} + diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/XLSXExporter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/XLSXExporter.java new file mode 100644 index 000000000..5621edc52 --- /dev/null +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/XLSXExporter.java @@ -0,0 +1,22 @@ +package ai.chat2db.server.web.api.controller.rdb.data.xlsx; + +import ai.chat2db.server.domain.api.enums.ExportFileSuffix; +import ai.chat2db.server.web.api.controller.rdb.data.DataFileExporter; +import com.alibaba.excel.support.ExcelTypeEnum; + +/** + * @author: zgq + * @date: 2024年04月26日 14:05 + */ +public class XLSXExporter extends BaseExcelExporter implements DataFileExporter { + + public XLSXExporter() { + suffix = ExportFileSuffix.XLSX.getSuffix(); + contentType = "application/vnd.ms-excel"; + } + + @Override + protected ExcelTypeEnum getExcelType() { + return ExcelTypeEnum.XLSX; + } +} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/XLSXImportExportFactory.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/XLSXImportExportFactory.java new file mode 100644 index 000000000..4031fad51 --- /dev/null +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/XLSXImportExportFactory.java @@ -0,0 +1,23 @@ +package ai.chat2db.server.web.api.controller.rdb.data.xlsx; + +import ai.chat2db.server.web.api.controller.rdb.data.DataFileExporter; +import ai.chat2db.server.web.api.controller.rdb.data.DataFileImportExportFactory; +import ai.chat2db.server.web.api.controller.rdb.data.DataFileImporter; + +/** + * @author: zgq + * @date: 2024年04月26日 14:17 + */ +public class XLSXImportExportFactory implements DataFileImportExportFactory { + + @Override + public DataFileImporter createImporter() { + return new XLSXImporter(); + } + + + @Override + public DataFileExporter createExporter() { + return new XLSXExporter(); + } +} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/XLSXImporter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/XLSXImporter.java new file mode 100644 index 000000000..d2c15e8af --- /dev/null +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/data/xlsx/XLSXImporter.java @@ -0,0 +1,16 @@ +package ai.chat2db.server.web.api.controller.rdb.data.xlsx; + +import ai.chat2db.server.web.api.controller.rdb.data.DataFileImporter; +import com.alibaba.excel.support.ExcelTypeEnum; + +/** + * @author: zgq + * @date: 2024年04月26日 14:19 + */ +public class XLSXImporter extends BaseExcelImporter implements DataFileImporter { + + @Override + protected ExcelTypeEnum getExcelType() { + return ExcelTypeEnum.XLSX; + } +} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/doc/export/ExportExcelService.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/doc/export/ExportExcelService.java index 79b97a271..21f744dbd 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/doc/export/ExportExcelService.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/doc/export/ExportExcelService.java @@ -29,7 +29,7 @@ public class ExportExcelService extends DatabaseExportService { public ExportExcelService() { exportTypeEnum = ExportTypeEnum.EXCEL; - suffix = ExportFileSuffix.EXCEL.getSuffix(); + suffix = ExportFileSuffix.XLSX.getSuffix(); contentType = "text/csv"; } diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/factory/ExportDBDataStrategyFactory.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/factory/ExportDBDataStrategyFactory.java deleted file mode 100644 index 638557b17..000000000 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/factory/ExportDBDataStrategyFactory.java +++ /dev/null @@ -1,31 +0,0 @@ -package ai.chat2db.server.web.api.controller.rdb.factory; - -import ai.chat2db.server.domain.api.enums.ExportTypeEnum; -import ai.chat2db.server.web.api.controller.rdb.data.export.strategy.*; -import lombok.SneakyThrows; - -import java.util.Map; - -/** - * @author: zgq - * @date: 2024年03月24日 12:53 - */ -public class ExportDBDataStrategyFactory { - - public static final Map> SERVICE_MAP = Map.of( - ExportTypeEnum.SQL.getCode(), ExportDBData2SqlStrategy.class, - ExportTypeEnum.CSV.getCode(), ExportDBData2CsvStrategy.class, - ExportTypeEnum.EXCEL.getCode(), ExportDBData2ExcelStrategy.class, - ExportTypeEnum.JSON.getCode(), ExportDBData2JsonStrategy.class - ); - - @SneakyThrows - public static Class get(String type) { - Class dataResult = SERVICE_MAP.get(type); - if (dataResult == null) { - throw new ClassNotFoundException("no ExportUI was found"); - } else { - return dataResult; - } - } -} \ No newline at end of file diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/request/DatabaseExportDataRequest.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/request/DatabaseExportDataRequest.java index 41686b503..9a6a0c35f 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/request/DatabaseExportDataRequest.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/request/DatabaseExportDataRequest.java @@ -1,11 +1,15 @@ package ai.chat2db.server.web.api.controller.rdb.request; +import ai.chat2db.server.tools.common.model.rdb.data.option.AbstractExportDataOptions; +import ai.chat2db.server.tools.common.model.rdb.data.option.table.BaseTableOptions; import ai.chat2db.server.web.api.controller.data.source.request.DataSourceBaseRequest; import jakarta.validation.constraints.NotNull; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; +import java.util.List; + /** * @author: zgq * @date: 2024年03月24日 12:36 @@ -15,5 +19,7 @@ @NoArgsConstructor public class DatabaseExportDataRequest extends DataSourceBaseRequest { @NotNull - private String exportType; + private AbstractExportDataOptions exportDataOption; + private List exportTableOptions; + } \ No newline at end of file diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/request/DatabaseImportDataRequest.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/request/DatabaseImportDataRequest.java new file mode 100644 index 000000000..b2f29e3b7 --- /dev/null +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/request/DatabaseImportDataRequest.java @@ -0,0 +1,20 @@ +package ai.chat2db.server.web.api.controller.rdb.request; + +import ai.chat2db.server.tools.common.model.rdb.data.option.AbstractImportDataOptions; +import ai.chat2db.server.tools.common.model.rdb.data.option.table.ImportTableOptions; +import ai.chat2db.server.web.api.controller.data.source.request.DataSourceBaseRequest; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * @author: zgq + * @date: 2024年04月23日 13:50 + */ +@Data +@AllArgsConstructor +@NoArgsConstructor +public class DatabaseImportDataRequest extends DataSourceBaseRequest { + private ImportTableOptions importTableOption; + private AbstractImportDataOptions importDataOption; +} diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/task/biz/TaskBizService.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/task/biz/TaskBizService.java index 2623c580e..6230af2cc 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/task/biz/TaskBizService.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/task/biz/TaskBizService.java @@ -217,7 +217,7 @@ private File createTempFile(String tableName, String exportType) { } else if (ExportTypeEnum.INSERT.getCode().equals(exportType)) { return FileUtil.createTempFile(fileName, ".sql", true); } else if (ExportTypeEnum.EXCEL.getCode().equals(exportType)) { - return FileUtil.createTempFile(fileName, ExportFileSuffix.EXCEL.getSuffix(), true); + return FileUtil.createTempFile(fileName, ExportFileSuffix.XLSX.getSuffix(), true); } else if (ExportTypeEnum.MARKDOWN.getCode().equals(exportType)) { return FileUtil.createTempFile(fileName, ExportFileSuffix.MARKDOWN.getSuffix(), true); } else if (ExportTypeEnum.WORD.getCode().equals(exportType)) {