From feb976ac16b593dcd851b2fb4f0d5a6c4083f46d Mon Sep 17 00:00:00 2001 From: zgq <203083679@qq.com> Date: Sat, 20 Apr 2024 12:17:38 +0800 Subject: [PATCH 1/4] fix clickhouse schemas & tables query fix pgsql tables query --- .../plugin/clickhouse/ClickHouseMetaData.java | 15 ++++++- .../plugin/postgresql/PostgreSQLMetaData.java | 7 ++- .../domain/core/impl/TableServiceImpl.java | 44 ++++++++----------- 3 files changed, 38 insertions(+), 28 deletions(-) diff --git a/chat2db-server/chat2db-plugins/chat2db-clickhouse/src/main/java/ai/chat2db/plugin/clickhouse/ClickHouseMetaData.java b/chat2db-server/chat2db-plugins/chat2db-clickhouse/src/main/java/ai/chat2db/plugin/clickhouse/ClickHouseMetaData.java index b13ce6f9f..991183f51 100644 --- a/chat2db-server/chat2db-plugins/chat2db-clickhouse/src/main/java/ai/chat2db/plugin/clickhouse/ClickHouseMetaData.java +++ b/chat2db-server/chat2db-plugins/chat2db-clickhouse/src/main/java/ai/chat2db/plugin/clickhouse/ClickHouseMetaData.java @@ -41,10 +41,24 @@ public class ClickHouseMetaData extends DefaultMetaService implements MetaData { = "SELECT create_table_query from system.`tables` WHERE `database`='%s' and name='%s'"; private List systemDatabases = Arrays.asList("information_schema", "system"); public static final String FUNCTION_SQL = "SELECT name,create_query as ddl from system.functions where origin='SQLUserDefined'"; + private static String SELECT_TABLE_SQL = "SELECT name,comment from system.`tables` WHERE `database`='%s'"; public static String format(String tableName) { return "`" + tableName + "`"; } + @Override + public List tables(Connection connection, String databaseName, String schemaName, String tableName) { + return SQLExecutor.getInstance().execute(connection, String.format(SELECT_TABLE_SQL, schemaName), resultSet -> { + ArrayList
tables = new ArrayList<>(); + while (resultSet.next()) { + Table table = new Table(); + table.setName(resultSet.getString("name")); + table.setComment(resultSet.getString("comment")); + tables.add(table); + } + return tables; + }); + } @Override public List functions(Connection connection, String databaseName, String schemaName) { @@ -299,7 +313,6 @@ public TableMeta getTableMeta(String databaseName, String schemaName, String tab @Override public String getMetaDataName(String... names) { return Arrays.stream(names) - .skip(1) // 跳过第一个名称 .filter(StringUtils::isNotBlank) .map(name -> "`" + name + "`") .collect(Collectors.joining(".")); diff --git a/chat2db-server/chat2db-plugins/chat2db-postgresql/src/main/java/ai/chat2db/plugin/postgresql/PostgreSQLMetaData.java b/chat2db-server/chat2db-plugins/chat2db-postgresql/src/main/java/ai/chat2db/plugin/postgresql/PostgreSQLMetaData.java index 4a05533ab..bb564e5b5 100644 --- a/chat2db-server/chat2db-plugins/chat2db-postgresql/src/main/java/ai/chat2db/plugin/postgresql/PostgreSQLMetaData.java +++ b/chat2db-server/chat2db-plugins/chat2db-postgresql/src/main/java/ai/chat2db/plugin/postgresql/PostgreSQLMetaData.java @@ -71,7 +71,12 @@ public List schemas(Connection connection, String databaseName) { }); return SortUtils.sortSchema(schemas, systemSchemas); } - + @Override + public List
tables(Connection connection, String databaseName, String schemaName, String tableName) { + return SQLExecutor.getInstance().tables(connection, StringUtils.isEmpty(databaseName) ? null : databaseName, + StringUtils.isEmpty(schemaName) ? null : schemaName, tableName, + new String[]{"TABLE", "SYSTEM TABLE","PARTITIONED TABLE"}); + } private static final String SELECT_TABLE_INDEX = "SELECT tmp.INDISPRIMARY AS Index_primary, tmp.TABLE_SCHEM, tmp.TABLE_NAME, tmp.NON_UNIQUE, tmp.INDEX_QUALIFIER, tmp.INDEX_NAME AS Key_name, tmp.indisclustered, tmp.ORDINAL_POSITION AS Seq_in_index, TRIM ( BOTH '\"' FROM pg_get_indexdef ( tmp.CI_OID, tmp.ORDINAL_POSITION, FALSE ) ) AS Column_name,CASE tmp.AM_NAME WHEN 'btree' THEN CASE tmp.I_INDOPTION [ tmp.ORDINAL_POSITION - 1 ] & 1 :: SMALLINT WHEN 1 THEN 'D' ELSE'A' END ELSE NULL END AS Collation, tmp.CARDINALITY, tmp.PAGES, tmp.FILTER_CONDITION , tmp.AM_NAME AS Index_method, tmp.DESCRIPTION AS Index_comment FROM ( SELECT n.nspname AS TABLE_SCHEM, ct.relname AS TABLE_NAME, NOT i.indisunique AS NON_UNIQUE, NULL AS INDEX_QUALIFIER, ci.relname AS INDEX_NAME,i.INDISPRIMARY , i.indisclustered , ( information_schema._pg_expandarray ( i.indkey ) ).n AS ORDINAL_POSITION, ci.reltuples AS CARDINALITY, ci.relpages AS PAGES, pg_get_expr ( i.indpred, i.indrelid ) AS FILTER_CONDITION, ci.OID AS CI_OID, i.indoption AS I_INDOPTION, am.amname AS AM_NAME , d.description FROM pg_class ct JOIN pg_namespace n ON ( ct.relnamespace = n.OID ) JOIN pg_index i ON ( ct.OID = i.indrelid ) JOIN pg_class ci ON ( ci.OID = i.indexrelid ) JOIN pg_am am ON ( ci.relam = am.OID ) left outer join pg_description d on i.indexrelid = d.objoid WHERE n.nspname = '%s' AND ct.relname = '%s' ) AS tmp ;"; private static String ROUTINES_SQL = "SELECT p.proname, p.prokind, pg_catalog.pg_get_functiondef(p.oid) as \"code\" FROM pg_catalog.pg_proc p where p.prokind = '%s' and p.proname='%s'"; diff --git a/chat2db-server/chat2db-server-domain/chat2db-server-domain-core/src/main/java/ai/chat2db/server/domain/core/impl/TableServiceImpl.java b/chat2db-server/chat2db-server-domain/chat2db-server-domain-core/src/main/java/ai/chat2db/server/domain/core/impl/TableServiceImpl.java index 5e0282efc..3698137de 100644 --- a/chat2db-server/chat2db-server-domain/chat2db-server-domain-core/src/main/java/ai/chat2db/server/domain/core/impl/TableServiceImpl.java +++ b/chat2db-server/chat2db-server-domain/chat2db-server-domain-core/src/main/java/ai/chat2db/server/domain/core/impl/TableServiceImpl.java @@ -420,29 +420,24 @@ public ListResult queryTables(TablePageQueryParam param) { private long addDBCache(Long dataSourceId, String databaseName, String schemaName, long version) { String key = getTableKey(dataSourceId, databaseName, schemaName); - Connection connection = Chat2DBContext.getConnection(); - long n = 0; - try (ResultSet resultSet = connection.getMetaData().getTables(databaseName, schemaName, null, - new String[]{"TABLE", "SYSTEM TABLE"})) { - List cacheDOS = new ArrayList<>(); - while (resultSet.next()) { - TableCacheDO tableCacheDO = new TableCacheDO(); - tableCacheDO.setDatabaseName(databaseName); - tableCacheDO.setSchemaName(schemaName); - tableCacheDO.setTableName(resultSet.getString("TABLE_NAME")); - tableCacheDO.setExtendInfo(resultSet.getString("REMARKS")); - tableCacheDO.setDataSourceId(dataSourceId); - tableCacheDO.setVersion(version); - tableCacheDO.setKey(key); - cacheDOS.add(tableCacheDO); - if (cacheDOS.size() >= 500) { - getTableCacheMapper().batchInsert(cacheDOS); - cacheDOS = new ArrayList<>(); - } - n++; - } - if (!CollectionUtils.isEmpty(cacheDOS)) { + MetaData metaData = Chat2DBContext.getMetaData(); + List
tables = metaData.tables(connection, databaseName, schemaName, null); + if (tables.isEmpty()) { + return 0; + } + List cacheDOS = tables.stream().map(table -> { + TableCacheDO tableCacheDO = new TableCacheDO(); + tableCacheDO.setDatabaseName(databaseName); + tableCacheDO.setSchemaName(schemaName); + tableCacheDO.setTableName(table.getName()); + tableCacheDO.setExtendInfo(table.getComment()); + tableCacheDO.setDataSourceId(dataSourceId); + tableCacheDO.setVersion(version); + tableCacheDO.setKey(key); + return tableCacheDO; + }).collect(Collectors.toList()); + if (!CollectionUtils.isEmpty(cacheDOS)) { getTableCacheMapper().batchInsert(cacheDOS); } LambdaQueryWrapper q = new LambdaQueryWrapper(); @@ -455,10 +450,7 @@ private long addDBCache(Long dataSourceId, String databaseName, String schemaNam q.eq(TableCacheDO::getSchemaName, schemaName); } getTableCacheMapper().delete(q); - } catch (SQLException e) { - throw new RuntimeException(e); - } - return n; + return tables.size(); } private Long getLock(Long dataSourceId, String databaseName, String schemaName, TableCacheVersionDO versionDO) { From 2277eaa31195293cecd9d3dfa058e2145bdaff64 Mon Sep 17 00:00:00 2001 From: zgq <203083679@qq.com> Date: Sat, 20 Apr 2024 13:47:54 +0800 Subject: [PATCH 2/4] fix --- .../ai/chat2db/plugin/clickhouse/ClickHouseMetaData.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/chat2db-server/chat2db-plugins/chat2db-clickhouse/src/main/java/ai/chat2db/plugin/clickhouse/ClickHouseMetaData.java b/chat2db-server/chat2db-plugins/chat2db-clickhouse/src/main/java/ai/chat2db/plugin/clickhouse/ClickHouseMetaData.java index 991183f51..a71b198e2 100644 --- a/chat2db-server/chat2db-plugins/chat2db-clickhouse/src/main/java/ai/chat2db/plugin/clickhouse/ClickHouseMetaData.java +++ b/chat2db-server/chat2db-plugins/chat2db-clickhouse/src/main/java/ai/chat2db/plugin/clickhouse/ClickHouseMetaData.java @@ -38,10 +38,10 @@ public class ClickHouseMetaData extends DefaultMetaService implements MetaData { = "SELECT TRIGGER_NAME FROM INFORMATION_SCHEMA.TRIGGERS where TRIGGER_SCHEMA = '%s';"; private static String SELECT_TABLE_COLUMNS = "select * from `system`.columns where table ='%s' and database='%s';"; private static String VIEW_SQL - = "SELECT create_table_query from system.`tables` WHERE `database`='%s' and name='%s'"; + = "SELECT create_table_query from system.`tables` WHERE `database`='%s' and name='%s' and engine='View'"; private List systemDatabases = Arrays.asList("information_schema", "system"); public static final String FUNCTION_SQL = "SELECT name,create_query as ddl from system.functions where origin='SQLUserDefined'"; - private static String SELECT_TABLE_SQL = "SELECT name,comment from system.`tables` WHERE `database`='%s'"; + private static String SELECT_TABLE_SQL = "SELECT name,comment from system.tables WHERE engine !='View' and database='%s'"; public static String format(String tableName) { return "`" + tableName + "`"; @@ -232,7 +232,7 @@ private void setColumnSize(TableColumn column, String columnType) { @Override public Table view(Connection connection, String databaseName, String schemaName, String viewName) { - String sql = String.format(VIEW_SQL, databaseName, viewName); + String sql = String.format(VIEW_SQL, schemaName, viewName); return SQLExecutor.getInstance().execute(connection, sql, resultSet -> { Table table = new Table(); table.setDatabaseName(databaseName); From b9ed69b615205846fe602460dcb4f29c728490a0 Mon Sep 17 00:00:00 2001 From: zgq <203083679@qq.com> Date: Sat, 20 Apr 2024 19:04:53 +0800 Subject: [PATCH 3/4] fix clickhouse table ddl query --- .../ai/chat2db/plugin/clickhouse/ClickHouseMetaData.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/chat2db-server/chat2db-plugins/chat2db-clickhouse/src/main/java/ai/chat2db/plugin/clickhouse/ClickHouseMetaData.java b/chat2db-server/chat2db-plugins/chat2db-clickhouse/src/main/java/ai/chat2db/plugin/clickhouse/ClickHouseMetaData.java index a71b198e2..d44174e1f 100644 --- a/chat2db-server/chat2db-plugins/chat2db-clickhouse/src/main/java/ai/chat2db/plugin/clickhouse/ClickHouseMetaData.java +++ b/chat2db-server/chat2db-plugins/chat2db-clickhouse/src/main/java/ai/chat2db/plugin/clickhouse/ClickHouseMetaData.java @@ -95,11 +95,10 @@ public List databases(Connection connection) { @Override public String tableDDL(Connection connection, @NotEmpty String databaseName, String schemaName, @NotEmpty String tableName) { - String sql = "SHOW CREATE TABLE " + format(databaseName) + "." - + format(tableName); + String sql =String.format("show create table %s.%s", schemaName, tableName); return SQLExecutor.getInstance().execute(connection, sql, resultSet -> { if (resultSet.next()) { - return resultSet.getString("Create Table"); + return resultSet.getString("statement"); } return null; }); From 2ab9b194e183bbb2819afa03fed87faa4173e113 Mon Sep 17 00:00:00 2001 From: zgq <203083679@qq.com> Date: Sat, 20 Apr 2024 20:49:00 +0800 Subject: [PATCH 4/4] fix clickhouse tables query --- .../plugin/clickhouse/ClickHouseMetaData.java | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/chat2db-server/chat2db-plugins/chat2db-clickhouse/src/main/java/ai/chat2db/plugin/clickhouse/ClickHouseMetaData.java b/chat2db-server/chat2db-plugins/chat2db-clickhouse/src/main/java/ai/chat2db/plugin/clickhouse/ClickHouseMetaData.java index d44174e1f..2cb99ba94 100644 --- a/chat2db-server/chat2db-plugins/chat2db-clickhouse/src/main/java/ai/chat2db/plugin/clickhouse/ClickHouseMetaData.java +++ b/chat2db-server/chat2db-plugins/chat2db-clickhouse/src/main/java/ai/chat2db/plugin/clickhouse/ClickHouseMetaData.java @@ -40,29 +40,29 @@ public class ClickHouseMetaData extends DefaultMetaService implements MetaData { private static String VIEW_SQL = "SELECT create_table_query from system.`tables` WHERE `database`='%s' and name='%s' and engine='View'"; private List systemDatabases = Arrays.asList("information_schema", "system"); - public static final String FUNCTION_SQL = "SELECT name,create_query as ddl from system.functions where origin='SQLUserDefined'"; - private static String SELECT_TABLE_SQL = "SELECT name,comment from system.tables WHERE engine !='View' and database='%s'"; + private static String FUNCTION_SQL = "SELECT name,create_query as ddl from system.functions where name='%s'"; + private static String FUNCTIONS_SQL = "SELECT name from system.functions where origin='%s'"; public static String format(String tableName) { return "`" + tableName + "`"; } @Override public List
tables(Connection connection, String databaseName, String schemaName, String tableName) { - return SQLExecutor.getInstance().execute(connection, String.format(SELECT_TABLE_SQL, schemaName), resultSet -> { - ArrayList
tables = new ArrayList<>(); - while (resultSet.next()) { - Table table = new Table(); - table.setName(resultSet.getString("name")); - table.setComment(resultSet.getString("comment")); - tables.add(table); - } - return tables; - }); + return SQLExecutor.getInstance().tables(connection, databaseName, + schemaName, tableName, + new String[]{"TABLE", "SYSTEM TABLE", + "REMOTE TABLE","DICTIONARY"}); } @Override public List functions(Connection connection, String databaseName, String schemaName) { - return SQLExecutor.getInstance().execute(connection, FUNCTION_SQL, resultSet -> { + String sql ; + if (systemDatabases.contains(schemaName)) { + sql=String.format(FUNCTIONS_SQL,"System"); + }else { + sql = String.format(FUNCTIONS_SQL, "SQLUserDefined"); + } + return SQLExecutor.getInstance().execute(connection, sql, resultSet -> { List functions = new ArrayList<>(); while (resultSet.next()) { Function function = new Function(); @@ -107,7 +107,7 @@ public String tableDDL(Connection connection, @NotEmpty String databaseName, Str @Override public Function function(Connection connection, @NotEmpty String databaseName, String schemaName, String functionName) { - return SQLExecutor.getInstance().execute(connection, FUNCTION_SQL, resultSet -> { + return SQLExecutor.getInstance().execute(connection, String.format(FUNCTION_SQL, functionName), resultSet -> { Function function = new Function(); function.setDatabaseName(databaseName); function.setSchemaName(schemaName);