diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 224046eb..7f6e90c0 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -25,18 +25,116 @@ jobs: run: make test WITH_RACE=1 - uses: codecov/codecov-action@v1 - integration-test: + integration-test-mysql-5735: runs-on: ubuntu-latest timeout-minutes: 15 strategy: fail-fast: true + services: + mysql: + image: mysql:5.7.35 + env: + MYSQL_ALLOW_EMPTY_PASSWORD: yes + ports: + - 3306:3306 + options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3 steps: - uses: actions/checkout@v2 +<<<<<<< HEAD +======= + - name: Shutdown Ubuntu MySQL (SUDO) + run: sudo service mysql stop # Shutdown the Default MySQL, "sudo" is necessary, please not remove it + - name: Set up Go 1.16 + uses: actions/setup-go@v2 + with: + go-version: 1.16 + - uses: actions/cache@v2 + with: + path: ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- + - name: Get dependencies + run: go mod download + - name: Download dependencies + run: sh install.sh + - name: Integration test + run: make integration_test + - name: Set up tmate session + if: ${{ failure() }} + uses: mxschmitt/action-tmate@v3 + + integration-test-mysql-8026: + runs-on: ubuntu-latest + timeout-minutes: 15 + strategy: + fail-fast: true + services: + mysql: + image: mysql:8.0.26 + env: + MYSQL_ALLOW_EMPTY_PASSWORD: yes + ports: + - 3306:3306 + options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3 + steps: + - uses: actions/checkout@v2 + - name: Shutdown Ubuntu MySQL (SUDO) + run: sudo service mysql stop # Shutdown the Default MySQL, "sudo" is necessary, please not remove it + - name: Set up Go 1.16 + uses: actions/setup-go@v2 + with: + go-version: 1.16 + - uses: actions/cache@v2 + with: + path: ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- + - name: Get dependencies + run: go mod download + - name: Download dependencies + run: sh install.sh + - name: Integration test + run: make integration_test + - name: Set up tmate session + if: ${{ failure() }} + uses: mxschmitt/action-tmate@v3 + + integration-test-mysql-8022: + runs-on: ubuntu-latest + timeout-minutes: 15 + strategy: + fail-fast: true + services: + mysql: + image: mysql:8.0.22 + env: + MYSQL_ALLOW_EMPTY_PASSWORD: yes + ports: + - 3306:3306 + options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3 + steps: + - uses: actions/checkout@v2 + - name: Shutdown Ubuntu MySQL (SUDO) + run: sudo service mysql stop # Shutdown the Default MySQL, "sudo" is necessary, please not remove it + - name: Set up Go 1.16 + uses: actions/setup-go@v2 + with: + go-version: 1.16 + - uses: actions/cache@v2 + with: + path: ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- + - name: Get dependencies + run: go mod download +>>>>>>> 4e012e5 (*: use `show full tables` in ListAllDatabasesTables (#325)) - name: Download dependencies run: sh install.sh - - name: Start MySQL - run: | - sudo systemctl start mysql.service - mysqladmin -uroot -proot password '' - name: Integration test run: make integration_test + - name: Set up tmate session + if: ${{ failure() }} + uses: mxschmitt/action-tmate@v3 diff --git a/Makefile b/Makefile index b8ac63f2..29d8614d 100644 --- a/Makefile +++ b/Makefile @@ -29,7 +29,7 @@ test: failpoint-enable integration_test: bins failpoint-enable bin/dumpling @make failpoint-disable - ./tests/run.sh + ./tests/run.sh $(CASE) tools: @echo "install tools..." diff --git a/tests/quote/data/quote-database-schema-create-mysql57.sql b/tests/quote/data/quote-database-schema-create-mysql57.sql new file mode 100755 index 00000000..a5df1c26 --- /dev/null +++ b/tests/quote/data/quote-database-schema-create-mysql57.sql @@ -0,0 +1,2 @@ +/*!40101 SET NAMES binary*/; +CREATE DATABASE `quo``te/database` /*!40100 DEFAULT CHARACTER SET latin1 */; diff --git a/tests/quote/data/quote-database.quote-table-schema-mysql57.sql b/tests/quote/data/quote-database.quote-table-schema-mysql57.sql new file mode 100755 index 00000000..b3c55dee --- /dev/null +++ b/tests/quote/data/quote-database.quote-table-schema-mysql57.sql @@ -0,0 +1,7 @@ +/*!40101 SET NAMES binary*/; +CREATE TABLE `quo``te/table` ( + `quo``te/col` int(11) NOT NULL, + `a` int(11) DEFAULT NULL, + `gen``id` int(11) GENERATED ALWAYS AS (`quo``te/col`) VIRTUAL, + PRIMARY KEY (`quo``te/col`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; diff --git a/tests/quote/data/quote-database.quote-table.000000000-mysql57.sql b/tests/quote/data/quote-database.quote-table.000000000-mysql57.sql new file mode 100755 index 00000000..5cee6b7b --- /dev/null +++ b/tests/quote/data/quote-database.quote-table.000000000-mysql57.sql @@ -0,0 +1,13 @@ +/*!40101 SET NAMES binary*/; +INSERT INTO `quo``te/table` (`quo``te/col`,`a`) VALUES +(0,10), +(1,9), +(2,8), +(3,7), +(4,6), +(5,5), +(6,4), +(7,3), +(8,2), +(9,1), +(10,0); diff --git a/tests/quote/run.sh b/tests/quote/run.sh index 232d153f..eafe61ed 100644 --- a/tests/quote/run.sh +++ b/tests/quote/run.sh @@ -1,13 +1,23 @@ -#!/bin/sh +#!/bin/bash # # Copyright 2020 PingCAP, Inc. Licensed under Apache-2.0. set -eu mkdir -p "$DUMPLING_OUTPUT_DIR"/data -cp "$DUMPLING_BASE_NAME/data/quote-database.quote-table.000000000.sql" "$DUMPLING_OUTPUT_DIR/data/quo\`te%2Fdatabase.quo\`te%2Ftable.000000000.sql" -cp "$DUMPLING_BASE_NAME/data/quote-database.quote-table-schema.sql" "$DUMPLING_OUTPUT_DIR/data/quo\`te%2Fdatabase.quo\`te%2Ftable-schema.sql" -cp "$DUMPLING_BASE_NAME/data/quote-database-schema-create.sql" "$DUMPLING_OUTPUT_DIR/data/quo\`te%2Fdatabase-schema-create.sql" + +mysql_version=$(echo "select version()" | mysql -uroot -h127.0.0.1 -P3306 | awk 'NR==2' | awk '{print $1}') +echo "current user mysql version is $mysql_version" +if [[ $mysql_version = 5* ]]; then + # there is a bug in mysql 5.x, see https://bugs.mysql.com/bug.php?id=96994, so we use different create db/table sql + cp "$DUMPLING_BASE_NAME/data/quote-database.quote-table.000000000-mysql57.sql" "$DUMPLING_OUTPUT_DIR/data/quo\`te%2Fdatabase.quo\`te%2Ftable.000000000.sql" + cp "$DUMPLING_BASE_NAME/data/quote-database.quote-table-schema-mysql57.sql" "$DUMPLING_OUTPUT_DIR/data/quo\`te%2Fdatabase.quo\`te%2Ftable-schema.sql" + cp "$DUMPLING_BASE_NAME/data/quote-database-schema-create-mysql57.sql" "$DUMPLING_OUTPUT_DIR/data/quo\`te%2Fdatabase-schema-create.sql" +else + cp "$DUMPLING_BASE_NAME/data/quote-database.quote-table.000000000.sql" "$DUMPLING_OUTPUT_DIR/data/quo\`te%2Fdatabase.quo\`te%2Ftable.000000000.sql" + cp "$DUMPLING_BASE_NAME/data/quote-database.quote-table-schema.sql" "$DUMPLING_OUTPUT_DIR/data/quo\`te%2Fdatabase.quo\`te%2Ftable-schema.sql" + cp "$DUMPLING_BASE_NAME/data/quote-database-schema-create.sql" "$DUMPLING_OUTPUT_DIR/data/quo\`te%2Fdatabase-schema-create.sql" +fi db="quo\`te/database" run_sql "drop database if exists \`quo\`\`te/database\`" diff --git a/tests/run.sh b/tests/run.sh index c0de699e..aa87234a 100755 --- a/tests/run.sh +++ b/tests/run.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # # Copyright 2020 PingCAP, Inc. Licensed under Apache-2.0. @@ -19,7 +19,6 @@ mkdir -p "$DUMPLING_TEST_DIR" PATH="tests/_utils:$PATH" . "tests/_utils/run_services" - file_should_exist bin/tidb-server file_should_exist bin/tidb-lightning file_should_exist bin/dumpling @@ -28,8 +27,9 @@ file_should_exist bin/sync_diff_inspector trap stop_services EXIT start_services -for script in tests/*/run.sh; do - echo "****************** Running test $script..." +run_case_by_fullpath() { + script="$1" + echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> Running test $script..." DUMPLING_BASE_NAME="$(dirname "$script")" export DUMPLING_BASE_NAME TEST_NAME="$(basename "$(dirname "$script")")" @@ -37,11 +37,24 @@ for script in tests/*/run.sh; do export DUMPLING_OUTPUT_DIR PATH="tests/_utils:$PATH" \ - sh "$script" - - echo "Cleaning up test output dir: $DUMPLING_OUTPUT_DIR" + bash "$script" + echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>TEST: $script Passed Cleaning up test output dir: $DUMPLING_OUTPUT_DIR" rm -rf "$DUMPLING_OUTPUT_DIR" - -done - -echo "Passed integration tests." +} + +if [ "$#" -ge 1 ]; then + test_case="$@" +else + test_case="*" +fi + +if [ "$test_case" == "*" ]; then + for script in tests/*/run.sh; do + run_case_by_fullpath "$script" + done +else + script="tests/$test_case/run.sh" + run_case_by_fullpath "$script" +fi + +echo "Passed integration tests." \ No newline at end of file diff --git a/tests/views/run.sh b/tests/views/run.sh index f0d8d001..b8695444 100644 --- a/tests/views/run.sh +++ b/tests/views/run.sh @@ -9,7 +9,8 @@ run_sql_file "$DUMPLING_BASE_NAME/data/views-schema-create.sql" export DUMPLING_TEST_DATABASE="views" run_sql "create table t (a bigint, b varchar(255))" -run_sql "set session collation_connection='utf8mb4_general_ci'; create definer = 'root'@'localhost' view v as select * from t;" +run_sql_file "$DUMPLING_BASE_NAME/data/views.v-schema-view.sql" + # insert 20 records to `t`. run_sql "insert into t values $(seq -s, 20 | sed 's/,*$//g' | sed 's/[0-9]*/(\0,"\0")/g')" diff --git a/v4/export/config.go b/v4/export/config.go index a5b99f7e..3af6fa2f 100644 --- a/v4/export/config.go +++ b/v4/export/config.go @@ -703,3 +703,16 @@ func adjustFileFormat(conf *Config) error { } return nil } + +func matchMysqlBugversion(info ServerInfo) bool { + // if 8.0.3 <= mysql8 version < 8.0.23 + // FLUSH TABLES WITH READ LOCK could block other sessions from executing SHOW TABLE STATUS. + // see more in https://dev.mysql.com/doc/relnotes/mysql/8.0/en/news-8-0-23.html + if info.ServerType != ServerTypeMySQL { + return false + } + currentVersion := info.ServerVersion + bugVersionStart := semver.New("8.0.2") + bugVersionEnd := semver.New("8.0.23") + return bugVersionStart.LessThan(*currentVersion) && currentVersion.LessThan(*bugVersionEnd) +} diff --git a/v4/export/config_test.go b/v4/export/config_test.go index 466492d5..d76245fa 100644 --- a/v4/export/config_test.go +++ b/v4/export/config_test.go @@ -3,7 +3,7 @@ package export import ( - "context" + tcontext "github.com/pingcap/dumpling/v4/context" . "github.com/pingcap/check" ) @@ -14,7 +14,24 @@ type testConfigSuite struct{} func (s *testConfigSuite) TestCreateExternalStorage(c *C) { mockConfig := defaultConfigForTest(c) - loc, err := mockConfig.createExternalStorage(context.Background()) + loc, err := mockConfig.createExternalStorage(tcontext.Background()) c.Assert(err, IsNil) c.Assert(loc.URI(), Matches, "file:.*") } + +func (s *testConfigSuite) TestMatchMysqlBugversion(c *C) { + cases := []struct { + serverInfo ServerInfo + expected bool + }{ + {ParseServerInfo(tcontext.Background(), "5.7.25-TiDB-3.0.6"), false}, + {ParseServerInfo(tcontext.Background(), "8.0.2"), false}, + {ParseServerInfo(tcontext.Background(), "8.0.3"), true}, + {ParseServerInfo(tcontext.Background(), "8.0.22"), true}, + {ParseServerInfo(tcontext.Background(), "8.0.23"), false}, + } + for _, x := range cases { + cmt := Commentf("server info %s", x.serverInfo) + c.Assert(x.expected, Equals, matchMysqlBugversion(x.serverInfo), cmt) + } +} diff --git a/v4/export/dump.go b/v4/export/dump.go index 1ad1ca61..bcb5915f 100755 --- a/v4/export/dump.go +++ b/v4/export/dump.go @@ -827,13 +827,34 @@ func extractTiDBRowIDFromDecodedKey(indexField, key string) (string, error) { return "", errors.Errorf("decoded key %s doesn't have %s field", key, indexField) } +func getListTableTypeByConf(conf *Config) listTableType { + // use listTableByShowTableStatus by default because it has better performance + listType := listTableByShowTableStatus + if conf.Consistency == consistencyTypeLock { + // for consistency lock, we need to build the tables to dump as soon as possible + listType = listTableByInfoSchema + } else if conf.Consistency == consistencyTypeFlush && matchMysqlBugversion(conf.ServerInfo) { + // For some buggy versions of mysql, we need a workaround to get a list of table names. + listType = listTableByShowFullTables + } + return listType +} + func prepareTableListToDump(tctx *tcontext.Context, conf *Config, db *sql.Conn) error { databases, err := prepareDumpingDatabases(conf, db) if err != nil { return err } +<<<<<<< HEAD conf.Tables, err = listAllTables(db, databases) +======= + tableTypes := []TableType{TableTypeBase} + if !conf.NoViews { + tableTypes = append(tableTypes, TableTypeView) + } + conf.Tables, err = ListAllDatabasesTables(tctx, db, databases, getListTableTypeByConf(conf), tableTypes...) +>>>>>>> 4e012e5 (*: use `show full tables` in ListAllDatabasesTables (#325)) if err != nil { return err } @@ -1015,16 +1036,16 @@ func detectServerInfo(d *Dumper) error { // resolveAutoConsistency is an initialization step of Dumper. func resolveAutoConsistency(d *Dumper) error { conf := d.conf - if conf.Consistency != "auto" { + if conf.Consistency != consistencyTypeAuto { return nil } switch conf.ServerInfo.ServerType { case ServerTypeTiDB: - conf.Consistency = "snapshot" + conf.Consistency = consistencyTypeSnapshot case ServerTypeMySQL, ServerTypeMariaDB: - conf.Consistency = "flush" + conf.Consistency = consistencyTypeFlush default: - conf.Consistency = "none" + conf.Consistency = consistencyTypeNone } return nil } diff --git a/v4/export/dump_test.go b/v4/export/dump_test.go index 64cccc29..ba131962 100644 --- a/v4/export/dump_test.go +++ b/v4/export/dump_test.go @@ -53,3 +53,70 @@ func (s *testSQLSuite) TestDumpBlock(c *C) { c.Assert(errors.ErrorEqual(d.dumpDatabases(writerCtx, conn, taskChan), context.Canceled), IsTrue) c.Assert(errors.ErrorEqual(wg.Wait(), writerErr), IsTrue) } +<<<<<<< HEAD +======= + +func (s *testSQLSuite) TestDumpTableMeta(c *C) { + db, mock, err := sqlmock.New() + c.Assert(err, IsNil) + defer db.Close() + + tctx, cancel := tcontext.Background().WithLogger(appLogger).WithCancel() + defer cancel() + conn, err := db.Conn(tctx) + c.Assert(err, IsNil) + conf := DefaultConfig() + conf.NoSchemas = true + + for serverType := ServerTypeUnknown; serverType < ServerTypeAll; serverType++ { + conf.ServerInfo.ServerType = ServerType(serverType) + hasImplicitRowID := false + mock.ExpectQuery("SHOW COLUMNS FROM"). + WillReturnRows(sqlmock.NewRows([]string{"Field", "Type", "Null", "Key", "Default", "Extra"}). + AddRow("id", "int(11)", "NO", "PRI", nil, "")) + if serverType == ServerTypeTiDB { + mock.ExpectExec("SELECT _tidb_rowid from"). + WillReturnResult(sqlmock.NewResult(0, 0)) + hasImplicitRowID = true + } + mock.ExpectQuery(fmt.Sprintf("SELECT \\* FROM `%s`.`%s`", database, table)). + WillReturnRows(sqlmock.NewRows([]string{"id"}).AddRow(1)) + meta, err := dumpTableMeta(conf, conn, database, &TableInfo{Type: TableTypeBase, Name: table}) + c.Assert(err, IsNil) + c.Assert(meta.DatabaseName(), Equals, database) + c.Assert(meta.TableName(), Equals, table) + c.Assert(meta.SelectedField(), Equals, "*") + c.Assert(meta.SelectedLen(), Equals, 1) + c.Assert(meta.ShowCreateTable(), Equals, "") + c.Assert(meta.HasImplicitRowID(), Equals, hasImplicitRowID) + } +} + +func (s *testSQLSuite) TestGetListTableTypeByConf(c *C) { + conf := defaultConfigForTest(c) + tctx := tcontext.Background().WithLogger(appLogger) + cases := []struct { + serverInfo ServerInfo + consistency string + expected listTableType + }{ + {ParseServerInfo(tctx, "5.7.25-TiDB-3.0.6"), consistencyTypeSnapshot, listTableByShowTableStatus}, + // no bug version + {ParseServerInfo(tctx, "8.0.2"), consistencyTypeLock, listTableByInfoSchema}, + {ParseServerInfo(tctx, "8.0.2"), consistencyTypeFlush, listTableByShowTableStatus}, + {ParseServerInfo(tctx, "8.0.23"), consistencyTypeNone, listTableByShowTableStatus}, + + // bug version + {ParseServerInfo(tctx, "8.0.3"), consistencyTypeLock, listTableByInfoSchema}, + {ParseServerInfo(tctx, "8.0.3"), consistencyTypeFlush, listTableByShowFullTables}, + {ParseServerInfo(tctx, "8.0.3"), consistencyTypeNone, listTableByShowTableStatus}, + } + + for _, x := range cases { + conf.Consistency = x.consistency + conf.ServerInfo = x.serverInfo + cmt := Commentf("server info %s consistency %s", x.serverInfo, x.consistency) + c.Assert(getListTableTypeByConf(conf), Equals, x.expected, cmt) + } +} +>>>>>>> 4e012e5 (*: use `show full tables` in ListAllDatabasesTables (#325)) diff --git a/v4/export/prepare_test.go b/v4/export/prepare_test.go index 28832543..0b869263 100644 --- a/v4/export/prepare_test.go +++ b/v4/export/prepare_test.go @@ -5,6 +5,7 @@ package export import ( "context" "fmt" + "strings" "github.com/DATA-DOG/go-sqlmock" . "github.com/pingcap/check" @@ -85,7 +86,11 @@ func (s *testPrepareSuite) TestListAllTables(c *C) { query := "SELECT table_schema,table_name FROM information_schema.tables WHERE table_type = (.*)" mock.ExpectQuery(query).WillReturnRows(rows) +<<<<<<< HEAD tables, err := listAllTables(conn, dbNames) +======= + tables, err := ListAllDatabasesTables(tctx, conn, dbNames, listTableByInfoSchema, TableTypeBase) +>>>>>>> 4e012e5 (*: use `show full tables` in ListAllDatabasesTables (#325)) c.Assert(err, IsNil) for d, t := range tables { @@ -101,6 +106,7 @@ func (s *testPrepareSuite) TestListAllTables(c *C) { data = NewDatabaseTables(). AppendTables("db", "t1"). AppendViews("db", "t2") +<<<<<<< HEAD query = "SELECT table_schema,table_name FROM information_schema.tables WHERE table_type = (.*)" mock.ExpectQuery(query).WillReturnRows(sqlmock.NewRows([]string{"table_schema", "table_name"}).AddRow("db", "t2")) tables, err = listAllViews(conn, []string{"db"}) @@ -108,6 +114,152 @@ func (s *testPrepareSuite) TestListAllTables(c *C) { c.Assert(len(tables), Equals, 1) c.Assert(len(tables["db"]), Equals, 1) c.Assert(tables["db"][0].Equals(data["db"][1]), IsTrue, Commentf("%v mismatch %v", tables["db"][0], data["db"][1])) +======= + query = "SELECT TABLE_SCHEMA,TABLE_NAME,TABLE_TYPE,AVG_ROW_LENGTH FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_TYPE='BASE TABLE' OR TABLE_TYPE='VIEW'" + mock.ExpectQuery(query).WillReturnRows(sqlmock.NewRows([]string{"TABLE_SCHEMA", "TABLE_NAME", "TABLE_TYPE", "AVG_ROW_LENGTH"}). + AddRow("db", "t1", TableTypeBaseStr, 1).AddRow("db", "t2", TableTypeViewStr, nil)) + tables, err = ListAllDatabasesTables(tctx, conn, []string{"db"}, listTableByInfoSchema, TableTypeBase, TableTypeView) + c.Assert(err, IsNil) + c.Assert(len(tables), Equals, 1) + c.Assert(len(tables["db"]), Equals, 2) + for i := 0; i < len(tables["db"]); i++ { + cmt := Commentf("%v mismatch: %v", tables["db"][i], data["db"][i]) + c.Assert(tables["db"][i].Equals(data["db"][i]), IsTrue, cmt) + } + + c.Assert(mock.ExpectationsWereMet(), IsNil) +} + +func (s *testPrepareSuite) TestListAllTablesByTableStatus(c *C) { + db, mock, err := sqlmock.New() + c.Assert(err, IsNil) + defer db.Close() + conn, err := db.Conn(context.Background()) + c.Assert(err, IsNil) + tctx := tcontext.Background().WithLogger(appLogger) + + // Test list all tables and skipping views. + data := NewDatabaseTables(). + AppendTables("db1", []string{"t1", "t2"}, []uint64{1, 2}). + AppendTables("db2", []string{"t3", "t4", "t5"}, []uint64{3, 4, 5}). + AppendViews("db3", "t6", "t7", "t8") + + query := "SHOW TABLE STATUS FROM `%s`" + showTableStatusColumnNames := []string{"Name", "Engine", "Version", "Row_format", "Rows", "Avg_row_length", "Data_length", "Max_data_length", "Index_length", "Data_free", "Auto_increment", "Create_time", "Update_time", "Check_time", "Collation", "Checksum", "Create_options", "Comment"} + dbNames := make([]databaseName, 0, len(data)) + for dbName, tableInfos := range data { + dbNames = append(dbNames, dbName) + rows := sqlmock.NewRows(showTableStatusColumnNames) + + for _, tbInfo := range tableInfos { + if tbInfo.Type == TableTypeBase { + rows.AddRow(tbInfo.Name, "InnoDB", 10, "Dynamic", 0, 0, 16384, 0, 0, 0, nil, "2021-07-08 03:04:07", nil, nil, "latin1_swedish_ci", nil, "", "") + } else { + rows.AddRow(tbInfo.Name, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, TableTypeView.String()) + } + } + mock.ExpectQuery(fmt.Sprintf(query, dbName)).WillReturnRows(rows) + } + + tables, err := ListAllDatabasesTables(tctx, conn, dbNames, listTableByShowTableStatus, TableTypeBase) + c.Assert(err, IsNil) + + for d, t := range tables { + expectedTbs, ok := data[d] + c.Assert(ok, IsTrue) + for i := 0; i < len(t); i++ { + cmt := Commentf("%v mismatch: %v", t[i], expectedTbs[i]) + c.Assert(t[i].Equals(expectedTbs[i]), IsTrue, cmt) + } + } + + // Test list all tables and not skipping views. + data = NewDatabaseTables(). + AppendTables("db", []string{"t1"}, []uint64{1}). + AppendViews("db", "t2") + mock.ExpectQuery(fmt.Sprintf(query, "db")).WillReturnRows(sqlmock.NewRows(showTableStatusColumnNames). + AddRow("t1", "InnoDB", 10, "Dynamic", 0, 1, 16384, 0, 0, 0, nil, "2021-07-08 03:04:07", nil, nil, "latin1_swedish_ci", nil, "", ""). + AddRow("t2", nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, TableTypeView.String())) + tables, err = ListAllDatabasesTables(tctx, conn, []string{"db"}, listTableByShowTableStatus, TableTypeBase, TableTypeView) + c.Assert(err, IsNil) + c.Assert(len(tables), Equals, 1) + c.Assert(len(tables["db"]), Equals, 2) + for i := 0; i < len(tables["db"]); i++ { + cmt := Commentf("%v mismatch: %v", tables["db"][i], data["db"][i]) + c.Assert(tables["db"][i].Equals(data["db"][i]), IsTrue, cmt) + } + + c.Assert(mock.ExpectationsWereMet(), IsNil) +} + +func (s *testPrepareSuite) TestListAllTablesByShowFullTables(c *C) { + db, mock, err := sqlmock.New() + c.Assert(err, IsNil) + defer db.Close() + conn, err := db.Conn(context.Background()) + c.Assert(err, IsNil) + tctx := tcontext.Background().WithLogger(appLogger) + + // Test list all tables and skipping views. + data := NewDatabaseTables(). + AppendTables("db1", []string{"t1", "t2"}, []uint64{1, 2}). + AppendTables("db2", []string{"t3", "t4", "t5"}, []uint64{3, 4, 5}). + AppendViews("db3", "t6", "t7", "t8") + + query := "SHOW FULL TABLES FROM `%s` WHERE TABLE_TYPE='BASE TABLE'" + dbNames := make([]databaseName, 0, len(data)) + for dbName, tableInfos := range data { + dbNames = append(dbNames, dbName) + columnNames := []string{strings.ToUpper(fmt.Sprintf("Tables_in_%s", dbName)), "TABLE_TYPE"} + rows := sqlmock.NewRows(columnNames) + for _, tbInfo := range tableInfos { + if tbInfo.Type == TableTypeBase { + rows.AddRow(tbInfo.Name, TableTypeBase.String()) + } else { + rows.AddRow(tbInfo.Name, TableTypeView.String()) + } + } + mock.ExpectQuery(fmt.Sprintf(query, dbName)).WillReturnRows(rows) + } + + tables, err := ListAllDatabasesTables(tctx, conn, dbNames, listTableByShowFullTables, TableTypeBase) + c.Assert(err, IsNil) + + for d, t := range tables { + expectedTbs, ok := data[d] + c.Assert(ok, IsTrue) + for i := 0; i < len(t); i++ { + cmt := Commentf("%v mismatch: %v", t[i], expectedTbs[i]) + c.Assert(t[i].Equals(expectedTbs[i]), IsTrue, cmt) + } + } + + // Test list all tables and not skipping views. + query = "SHOW FULL TABLES FROM `%s` WHERE TABLE_TYPE='BASE TABLE' OR TABLE_TYPE='VIEW'" + data = NewDatabaseTables(). + AppendTables("db", []string{"t1"}, []uint64{1}). + AppendViews("db", "t2") + for dbName, tableInfos := range data { + columnNames := []string{strings.ToUpper(fmt.Sprintf("Tables_in_%s", dbName)), "TABLE_TYPE"} + rows := sqlmock.NewRows(columnNames) + for _, tbInfo := range tableInfos { + if tbInfo.Type == TableTypeBase { + rows.AddRow(tbInfo.Name, TableTypeBase.String()) + } else { + rows.AddRow(tbInfo.Name, TableTypeView.String()) + } + } + mock.ExpectQuery(fmt.Sprintf(query, dbName)).WillReturnRows(rows) + } + tables, err = ListAllDatabasesTables(tctx, conn, []string{"db"}, listTableByShowFullTables, TableTypeBase, TableTypeView) + c.Assert(err, IsNil) + c.Assert(len(tables), Equals, 1) + c.Assert(len(tables["db"]), Equals, 2) + for i := 0; i < len(tables["db"]); i++ { + cmt := Commentf("%v mismatch: %v", tables["db"][i], data["db"][i]) + c.Assert(tables["db"][i].Equals(data["db"][i]), IsTrue, cmt) + } +>>>>>>> 4e012e5 (*: use `show full tables` in ListAllDatabasesTables (#325)) c.Assert(mock.ExpectationsWereMet(), IsNil) } diff --git a/v4/export/sql.go b/v4/export/sql.go index 655d9c2b..fb166a3e 100644 --- a/v4/export/sql.go +++ b/v4/export/sql.go @@ -18,7 +18,17 @@ import ( "go.uber.org/zap" ) -const orderByTiDBRowID = "ORDER BY `_tidb_rowid`" +const ( + orderByTiDBRowID = "ORDER BY `_tidb_rowid`" +) + +type listTableType int + +const ( + listTableByInfoSchema listTableType = iota + listTableByShowFullTables + listTableByShowTableStatus +) // ShowDatabases shows the databases of a database server. func ShowDatabases(db *sql.Conn) ([]string, error) { @@ -141,6 +151,7 @@ func RestoreCharset(w io.StringWriter) { } // ListAllDatabasesTables lists all the databases and tables from the database +<<<<<<< HEAD func ListAllDatabasesTables(db *sql.Conn, databaseNames []string, tableType TableType) (DatabaseTables, error) { var tableTypeStr string switch tableType { @@ -162,6 +173,83 @@ func ListAllDatabasesTables(db *sql.Conn, databaseNames []string, tableType Tabl var schema, table string if err := rows.Scan(&schema, &table); err != nil { return errors.Trace(err) +======= +// listTableByInfoSchema list tables by table information_schema in MySQL +// listTableByShowTableStatus has better performance than listTableByInfoSchema +// listTableByShowFullTables is used in mysql8 version [8.0.3,8.0.23), more details can be found in the comments of func matchMysqlBugversion +func ListAllDatabasesTables(tctx *tcontext.Context, db *sql.Conn, databaseNames []string, + listType listTableType, tableTypes ...TableType) (DatabaseTables, error) { // revive:disable-line:flag-parameter + dbTables := DatabaseTables{} + var ( + schema, table, tableTypeStr string + tableType TableType + avgRowLength uint64 + err error + ) + + tableTypeConditions := make([]string, len(tableTypes)) + for i, tableType := range tableTypes { + tableTypeConditions[i] = fmt.Sprintf("TABLE_TYPE='%s'", tableType) + } + switch listType { + case listTableByInfoSchema: + query := fmt.Sprintf("SELECT TABLE_SCHEMA,TABLE_NAME,TABLE_TYPE,AVG_ROW_LENGTH FROM INFORMATION_SCHEMA.TABLES WHERE %s", strings.Join(tableTypeConditions, " OR ")) + for _, schema := range databaseNames { + dbTables[schema] = make([]*TableInfo, 0) + } + if err = simpleQueryWithArgs(db, func(rows *sql.Rows) error { + var ( + sqlAvgRowLength sql.NullInt64 + err2 error + ) + if err2 = rows.Scan(&schema, &table, &tableTypeStr, &sqlAvgRowLength); err != nil { + return errors.Trace(err2) + } + tableType, err2 = ParseTableType(tableTypeStr) + if err2 != nil { + return errors.Trace(err2) + } + + if sqlAvgRowLength.Valid { + avgRowLength = uint64(sqlAvgRowLength.Int64) + } else { + avgRowLength = 0 + } + // only append tables to schemas in databaseNames + if _, ok := dbTables[schema]; ok { + dbTables[schema] = append(dbTables[schema], &TableInfo{table, avgRowLength, tableType}) + } + return nil + }, query); err != nil { + return nil, errors.Annotatef(err, "sql: %s", query) + } + case listTableByShowFullTables: + for _, schema = range databaseNames { + dbTables[schema] = make([]*TableInfo, 0) + query := fmt.Sprintf("SHOW FULL TABLES FROM `%s` WHERE %s", + escapeString(schema), strings.Join(tableTypeConditions, " OR ")) + if err = simpleQueryWithArgs(db, func(rows *sql.Rows) error { + var err2 error + if err2 = rows.Scan(&table, &tableTypeStr); err != nil { + return errors.Trace(err2) + } + tableType, err2 = ParseTableType(tableTypeStr) + if err2 != nil { + return errors.Trace(err2) + } + avgRowLength = 0 // can't get avgRowLength from the result of `show full tables` so hardcode to 0 here + dbTables[schema] = append(dbTables[schema], &TableInfo{table, avgRowLength, tableType}) + return nil + }, query); err != nil { + return nil, errors.Annotatef(err, "sql: %s", query) + } + } + default: + queryTemplate := "SHOW TABLE STATUS FROM `%s`" + selectedTableType := make(map[TableType]struct{}) + for _, tableType = range tableTypes { + selectedTableType[tableType] = struct{}{} +>>>>>>> 4e012e5 (*: use `show full tables` in ListAllDatabasesTables (#325)) } // only append tables to schemas in databaseNames