Skip to content

Commit

Permalink
refactor: optimize the error message of create table (4paradigm#3434)
Browse files Browse the repository at this point in the history
  • Loading branch information
@dl239
dl239 authored Aug 31, 2023
1 parent 6eff021 commit a013ba3
Show file tree
Hide file tree
Showing 12 changed files with 168 additions and 174 deletions.
42 changes: 21 additions & 21 deletions src/catalog/distribute_iterator_test.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -162,8 +162,8 @@ TEST_F(DistributeIteratorTest, AllInRemote) {
auto client2 = std::make_shared<openmldb::client::TabletClient>(endpoints[1], endpoints[1]);
ASSERT_EQ(client2->Init(), 0);
std::vector<::openmldb::api::TableMeta> metas = {CreateTableMeta(tid, 1), CreateTableMeta(tid, 4)};
ASSERT_TRUE(client1->CreateTable(metas[0]));
ASSERT_TRUE(client2->CreateTable(metas[1]));
ASSERT_TRUE(client1->CreateTable(metas[0]).OK());
ASSERT_TRUE(client2->CreateTable(metas[1]).OK());
std::map<uint32_t, std::shared_ptr<openmldb::client::TabletClient>> tablet_clients = {{1, client1}, {4, client2}};
FullTableIterator it(tid, {}, tablet_clients);
it.SeekToFirst();
Expand Down Expand Up @@ -205,8 +205,8 @@ TEST_F(DistributeIteratorTest, Hybrid) {
auto client2 = std::make_shared<openmldb::client::TabletClient>(endpoints[1], endpoints[1]);
ASSERT_EQ(client2->Init(), 0);
std::vector<::openmldb::api::TableMeta> metas = {CreateTableMeta(tid, 1), CreateTableMeta(tid, 4)};
ASSERT_TRUE(client1->CreateTable(metas[0]));
ASSERT_TRUE(client2->CreateTable(metas[1]));
ASSERT_TRUE(client1->CreateTable(metas[0]).OK());
ASSERT_TRUE(client2->CreateTable(metas[1]).OK());
std::map<uint32_t, std::shared_ptr<openmldb::client::TabletClient>> tablet_clients = {{1, client1}, {4, client2}};
FullTableIterator it(tid, tables, tablet_clients);
it.SeekToFirst();
Expand Down Expand Up @@ -265,8 +265,8 @@ TEST_F(DistributeIteratorTest, FullTableTraverseLimit) {
auto client2 = std::make_shared<openmldb::client::TabletClient>(endpoints[1], endpoints[1]);
ASSERT_EQ(client2->Init(), 0);
std::vector<::openmldb::api::TableMeta> metas = {CreateTableMeta(tid, 1), CreateTableMeta(tid, 4)};
ASSERT_TRUE(client1->CreateTable(metas[0]));
ASSERT_TRUE(client2->CreateTable(metas[1]));
ASSERT_TRUE(client1->CreateTable(metas[0]).OK());
ASSERT_TRUE(client2->CreateTable(metas[1]).OK());
std::map<uint32_t, std::shared_ptr<openmldb::client::TabletClient>> tablet_clients = {{1, client1}, {4, client2}};
FullTableIterator it(tid, tables, tablet_clients);
it.SeekToFirst();
Expand Down Expand Up @@ -317,7 +317,7 @@ TEST_F(DistributeIteratorTest, TraverseLimitSingle) {
auto client1 = std::make_shared<openmldb::client::TabletClient>(endpoints[0], endpoints[0]);
ASSERT_EQ(client1->Init(), 0);
std::vector<::openmldb::api::TableMeta> metas = {CreateTableMeta(tid, 0)};
ASSERT_TRUE(client1->CreateTable(metas[0]));
ASSERT_TRUE(client1->CreateTable(metas[0]).OK());
std::map<uint32_t, std::shared_ptr<openmldb::client::TabletClient>> tablet_clients = {{0, client1}};
for (int i = 0; i < 10; i++) {
std::string key = "card" + std::to_string(i);
Expand Down Expand Up @@ -355,8 +355,8 @@ TEST_F(DistributeIteratorTest, TraverseLimit) {
auto client2 = std::make_shared<openmldb::client::TabletClient>(endpoints[1], endpoints[1]);
ASSERT_EQ(client2->Init(), 0);
std::vector<::openmldb::api::TableMeta> metas = {CreateTableMeta(tid, 1), CreateTableMeta(tid, 3)};
ASSERT_TRUE(client1->CreateTable(metas[0]));
ASSERT_TRUE(client2->CreateTable(metas[1]));
ASSERT_TRUE(client1->CreateTable(metas[0]).OK());
ASSERT_TRUE(client2->CreateTable(metas[1]).OK());
std::map<uint32_t, std::shared_ptr<openmldb::client::TabletClient>> tablet_clients = {{1, client1}, {3, client2}};
std::map<uint32_t, uint32_t> cout_map = {{0, 0}, {1, 0}, {2, 0}, {3, 0}};
for (int i = 0; i < 100; i++) {
Expand Down Expand Up @@ -399,8 +399,8 @@ TEST_F(DistributeIteratorTest, WindowIterator) {
auto client2 = std::make_shared<openmldb::client::TabletClient>(endpoints[1], endpoints[1]);
ASSERT_EQ(client2->Init(), 0);
std::vector<::openmldb::api::TableMeta> metas = {CreateTableMeta(tid, 1), CreateTableMeta(tid, 3)};
ASSERT_TRUE(client1->CreateTable(metas[0]));
ASSERT_TRUE(client2->CreateTable(metas[1]));
ASSERT_TRUE(client1->CreateTable(metas[0]).OK());
ASSERT_TRUE(client2->CreateTable(metas[1]).OK());
std::map<uint32_t, std::shared_ptr<openmldb::client::TabletClient>> tablet_clients = {{1, client1}, {3, client2}};
for (int i = 0; i < 20; i++) {
std::string key = "card" + std::to_string(i);
Expand Down Expand Up @@ -462,7 +462,7 @@ TEST_F(DistributeIteratorTest, RemoteIterator) {
auto client1 = std::make_shared<openmldb::client::TabletClient>(endpoints[0], endpoints[0]);
ASSERT_EQ(client1->Init(), 0);
std::vector<::openmldb::api::TableMeta> metas = {CreateTableMeta(tid, 0)};
ASSERT_TRUE(client1->CreateTable(metas[0]));
ASSERT_TRUE(client1->CreateTable(metas[0]).OK());
std::map<uint32_t, std::shared_ptr<openmldb::client::TabletClient>> tablet_clients = {{0, client1}};
codec::SDKCodec codec(metas[0]);
int64_t now = 1999;
Expand Down Expand Up @@ -540,7 +540,7 @@ TEST_F(DistributeIteratorTest, RemoteIteratorSecondIndex) {
auto client1 = std::make_shared<openmldb::client::TabletClient>(endpoints[0], endpoints[0]);
ASSERT_EQ(client1->Init(), 0);
std::vector<::openmldb::api::TableMeta> metas = {CreateTableMeta(tid, 0)};
ASSERT_TRUE(client1->CreateTable(metas[0]));
ASSERT_TRUE(client1->CreateTable(metas[0]).OK());
std::map<uint32_t, std::shared_ptr<openmldb::client::TabletClient>> tablet_clients = {{0, client1}};
codec::SDKCodec codec(metas[0]);
uint64_t now = ::baidu::common::timer::get_micros() / 1000;
Expand Down Expand Up @@ -678,8 +678,8 @@ TEST_F(DistributeIteratorTest, MoreTsCnt) {
auto client2 = std::make_shared<openmldb::client::TabletClient>(endpoints[1], endpoints[1]);
ASSERT_EQ(client2->Init(), 0);
std::vector<::openmldb::api::TableMeta> metas = {CreateTableMeta(tid, 1), CreateTableMeta(tid, 3)};
ASSERT_TRUE(client1->CreateTable(metas[0]));
ASSERT_TRUE(client2->CreateTable(metas[1]));
ASSERT_TRUE(client1->CreateTable(metas[0]).OK());
ASSERT_TRUE(client2->CreateTable(metas[1]).OK());
std::map<uint32_t, std::shared_ptr<openmldb::client::TabletClient>> tablet_clients = {{1, client1}, {3, client2}};
std::map<uint32_t, uint32_t> cout_map = {{0, 0}, {1, 0}, {2, 0}, {3, 0}};
for (int i = 0; i < 50; i++) {
Expand Down Expand Up @@ -759,8 +759,8 @@ TEST_F(DistributeIteratorTest, TraverseSameTs) {
auto client2 = std::make_shared<openmldb::client::TabletClient>(endpoints[1], endpoints[1]);
ASSERT_EQ(client2->Init(), 0);
std::vector<::openmldb::api::TableMeta> metas = {CreateTableMeta(tid, 1), CreateTableMeta(tid, 3)};
ASSERT_TRUE(client1->CreateTable(metas[0]));
ASSERT_TRUE(client2->CreateTable(metas[1]));
ASSERT_TRUE(client1->CreateTable(metas[0]).OK());
ASSERT_TRUE(client2->CreateTable(metas[1]).OK());
std::map<uint32_t, std::shared_ptr<openmldb::client::TabletClient>> tablet_clients = {{1, client1}, {3, client2}};
std::map<uint32_t, uint32_t> cout_map = {{0, 0}, {1, 0}, {2, 0}, {3, 0}};
for (int i = 0; i < 20; i++) {
Expand Down Expand Up @@ -804,8 +804,8 @@ TEST_F(DistributeIteratorTest, WindowIteratorLimit) {
auto client2 = std::make_shared<openmldb::client::TabletClient>(endpoints[1], endpoints[1]);
ASSERT_EQ(client2->Init(), 0);
std::vector<::openmldb::api::TableMeta> metas = {CreateTableMeta(tid, 1), CreateTableMeta(tid, 3)};
ASSERT_TRUE(client1->CreateTable(metas[0]));
ASSERT_TRUE(client2->CreateTable(metas[1]));
ASSERT_TRUE(client1->CreateTable(metas[0]).OK());
ASSERT_TRUE(client2->CreateTable(metas[1]).OK());
std::map<uint32_t, std::shared_ptr<openmldb::client::TabletClient>> tablet_clients = {{1, client1}, {3, client2}};
for (int i = 0; i < 20; i++) {
std::string key = "card" + std::to_string(i);
Expand Down Expand Up @@ -921,8 +921,8 @@ TEST_F(DistributeIteratorTest, IteratorZero) {
auto client2 = std::make_shared<openmldb::client::TabletClient>(endpoints[1], endpoints[1]);
ASSERT_EQ(client2->Init(), 0);
std::vector<::openmldb::api::TableMeta> metas = {CreateTableMeta(tid, 1), CreateTableMeta(tid, 3)};
ASSERT_TRUE(client1->CreateTable(metas[0]));
ASSERT_TRUE(client2->CreateTable(metas[1]));
ASSERT_TRUE(client1->CreateTable(metas[0]).OK());
ASSERT_TRUE(client2->CreateTable(metas[1]).OK());
std::map<uint32_t, std::shared_ptr<openmldb::client::TabletClient>> tablet_clients = {{1, client1}, {3, client2}};
std::map<uint32_t, uint32_t> cout_map = {{0, 0}, {1, 0}, {2, 0}, {3, 0}};
int expect = 0;
Expand Down
68 changes: 7 additions & 61 deletions src/client/tablet_client.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -153,72 +153,18 @@ bool TabletClient::SQLBatchRequestQuery(const std::string& db, const std::string
return true;
}

bool TabletClient::CreateTable(const std::string& name, uint32_t tid, uint32_t pid, uint64_t abs_ttl, uint64_t lat_ttl,
bool leader, const std::vector<std::string>& endpoints,
const ::openmldb::type::TTLType& type, uint32_t seg_cnt, uint64_t term,
const ::openmldb::type::CompressType compress_type,
::openmldb::common::StorageMode storage_mode) {
::openmldb::api::CreateTableRequest request;
if (type == ::openmldb::type::kLatestTime) {
if (lat_ttl > FLAGS_latest_ttl_max) {
return false;
}
} else if (type == ::openmldb::type::TTLType::kAbsoluteTime) {
if (abs_ttl > FLAGS_absolute_ttl_max) {
return false;
}
} else {
if (abs_ttl > FLAGS_absolute_ttl_max || lat_ttl > FLAGS_latest_ttl_max) {
return false;
}
}
::openmldb::api::TableMeta* table_meta = request.mutable_table_meta();
table_meta->set_name(name);
table_meta->set_tid(tid);
table_meta->set_pid(pid);
table_meta->set_compress_type(compress_type);
table_meta->set_seg_cnt(seg_cnt);
table_meta->set_storage_mode(storage_mode);
if (leader) {
table_meta->set_mode(::openmldb::api::TableMode::kTableLeader);
table_meta->set_term(term);
} else {
table_meta->set_mode(::openmldb::api::TableMode::kTableFollower);
}
for (size_t i = 0; i < endpoints.size(); i++) {
table_meta->add_replicas(endpoints[i]);
}
::openmldb::common::ColumnDesc* column_desc = table_meta->add_column_desc();
column_desc->set_name("idx0");
column_desc->set_data_type(::openmldb::type::kString);
::openmldb::common::ColumnKey* index = table_meta->add_column_key();
index->set_index_name("idx0");
index->add_col_name("idx0");
::openmldb::common::TTLSt* ttl = index->mutable_ttl();
ttl->set_abs_ttl(abs_ttl);
ttl->set_lat_ttl(lat_ttl);
ttl->set_ttl_type(type);
// table_meta->set_ttl_type(type);
::openmldb::api::CreateTableResponse response;
bool ok = client_.SendRequest(&::openmldb::api::TabletServer_Stub::CreateTable, &request, &response,
FLAGS_request_timeout_ms * 2, 1);
if (ok && response.code() == 0) {
return true;
}
return false;
}

bool TabletClient::CreateTable(const ::openmldb::api::TableMeta& table_meta) {
base::Status TabletClient::CreateTable(const ::openmldb::api::TableMeta& table_meta) {
::openmldb::api::CreateTableRequest request;
::openmldb::api::TableMeta* table_meta_ptr = request.mutable_table_meta();
table_meta_ptr->CopyFrom(table_meta);
::openmldb::api::CreateTableResponse response;
bool ok = client_.SendRequest(&::openmldb::api::TabletServer_Stub::CreateTable, &request, &response,
FLAGS_request_timeout_ms * 2, 1);
if (ok && response.code() == 0) {
return true;
if (!client_.SendRequest(&::openmldb::api::TabletServer_Stub::CreateTable, &request, &response,
FLAGS_request_timeout_ms * 2, 1)) {
return {base::ReturnCode::kRPCError, "send request failed!"};
} else if (response.code() == 0) {
return {};
}
return false;
return {response.code(), response.msg()};
}

bool TabletClient::UpdateTableMetaForAddField(uint32_t tid, const std::vector<openmldb::common::ColumnDesc>& cols,
Expand Down
7 changes: 1 addition & 6 deletions src/client/tablet_client.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,12 +54,7 @@ class TabletClient : public Client {

int Init() override;

bool CreateTable(const std::string& name, uint32_t tid, uint32_t pid, uint64_t abs_ttl, uint64_t lat_ttl,
bool leader, const std::vector<std::string>& endpoints, const ::openmldb::type::TTLType& type,
uint32_t seg_cnt, uint64_t term, const ::openmldb::type::CompressType compress_type,
::openmldb::common::StorageMode storage_mode = ::openmldb::common::kMemory);

bool CreateTable(const ::openmldb::api::TableMeta& table_meta);
base::Status CreateTable(const ::openmldb::api::TableMeta& table_meta);

bool UpdateTableMetaForAddField(uint32_t tid, const std::vector<openmldb::common::ColumnDesc>& cols,
const openmldb::common::VersionPair& pair,
Expand Down
60 changes: 33 additions & 27 deletions src/nameserver/name_server_impl.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2182,9 +2182,9 @@ int NameServerImpl::SetPartitionInfo(TableInfo& table_info) {
return 0;
}

int NameServerImpl::CreateTableOnTablet(const std::shared_ptr<::openmldb::nameserver::TableInfo>& table_info,
bool is_leader, std::map<uint32_t, std::vector<std::string>>& endpoint_map,
uint64_t term) {
base::Status NameServerImpl::CreateTableOnTablet(const std::shared_ptr<::openmldb::nameserver::TableInfo>& table_info,
bool is_leader, uint64_t term,
std::map<uint32_t, std::vector<std::string>>* endpoint_map) {
::openmldb::type::CompressType compress_type = ::openmldb::type::CompressType::kNoCompress;
if (table_info->compress_type() == ::openmldb::type::kSnappy) {
compress_type = ::openmldb::type::CompressType::kSnappy;
Expand All @@ -2201,18 +2201,16 @@ int NameServerImpl::CreateTableOnTablet(const std::shared_ptr<::openmldb::namese
table_meta.set_key_entry_max_height(table_info->key_entry_max_height());
}
for (int idx = 0; idx < table_info->column_desc_size(); idx++) {
::openmldb::common::ColumnDesc* column_desc = table_meta.add_column_desc();
column_desc->CopyFrom(table_info->column_desc(idx));
table_meta.add_column_desc()->CopyFrom(table_info->column_desc(idx));
}
for (int idx = 0; idx < table_info->column_key_size(); idx++) {
::openmldb::common::ColumnKey* column_key = table_meta.add_column_key();
column_key->CopyFrom(table_info->column_key(idx));
table_meta.add_column_key()->CopyFrom(table_info->column_key(idx));
}
for (const auto& table_partition : table_info->table_partition()) {
::openmldb::common::TablePartition* partition = table_meta.add_table_partition();
auto partition = table_meta.add_table_partition();
partition->set_pid(table_partition.pid());
for (const auto& partition_meta : table_partition.partition_meta()) {
::openmldb::common::PartitionMeta* meta = partition->add_partition_meta();
auto meta = partition->add_partition_meta();
meta->set_endpoint(partition_meta.endpoint());
meta->set_is_leader(partition_meta.is_leader());
meta->set_is_alive(true);
Expand All @@ -2229,36 +2227,37 @@ int NameServerImpl::CreateTableOnTablet(const std::shared_ptr<::openmldb::namese
std::string endpoint = table_info->table_partition(idx).partition_meta(meta_idx).endpoint();
auto tablet_ptr = GetTablet(endpoint);
if (!tablet_ptr) {
PDLOG(WARNING, "endpoint[%s] can not find client", endpoint.c_str());
return -1;
PDLOG(WARNING, "endpoint[%s] cannot find client", endpoint.c_str());
return {base::ReturnCode::kServerConnError, absl::StrCat("endpoint ", endpoint, " cannot find client")};
}
if (is_leader) {
::openmldb::nameserver::TablePartition* table_partition = table_info->mutable_table_partition(idx);
::openmldb::nameserver::TermPair* term_pair = table_partition->add_term_offset();
auto table_partition = table_info->mutable_table_partition(idx);
auto term_pair = table_partition->add_term_offset();
term_pair->set_term(term);
term_pair->set_offset(0);
table_meta.set_mode(::openmldb::api::TableMode::kTableLeader);
table_meta.set_term(term);
for (const auto& e : endpoint_map[pid]) {
for (const auto& e : (*endpoint_map)[pid]) {
table_meta.add_replicas(e);
}
} else {
if (endpoint_map.find(pid) == endpoint_map.end()) {
endpoint_map.insert(std::make_pair(pid, std::vector<std::string>()));
auto iter = endpoint_map->find(pid);
if (iter == endpoint_map->end()) {
iter = endpoint_map->emplace(pid, std::vector<std::string>()).first;
}
endpoint_map[pid].push_back(endpoint);
iter->second.push_back(endpoint);
table_meta.set_mode(::openmldb::api::TableMode::kTableFollower);
}
if (!tablet_ptr->client_->CreateTable(table_meta)) {
PDLOG(WARNING, "create table failed. tid[%u] pid[%u] endpoint[%s]", table_info->tid(), pid,
endpoint.c_str());
return -1;
if (auto status = tablet_ptr->client_->CreateTable(table_meta); !status.OK()) {
PDLOG(WARNING, "create table failed. tid[%u] pid[%u] endpoint[%s] msg[%s]",
table_info->tid(), pid, endpoint.c_str(), status.GetMsg().c_str());
return status;
}
PDLOG(INFO, "create table success. tid[%u] pid[%u] endpoint[%s] idx[%d]", table_info->tid(), pid,
endpoint.c_str(), idx);
}
}
return 0;
return {};
}

int NameServerImpl::DropTableOnTablet(std::shared_ptr<::openmldb::nameserver::TableInfo> table_info) {
Expand Down Expand Up @@ -3758,11 +3757,18 @@ void NameServerImpl::CreateTableInternel(GeneralResponse& response,
std::shared_ptr<::openmldb::api::TaskInfo> task_ptr) {
std::map<uint32_t, std::vector<std::string>> endpoint_map;
do {
if (CreateTableOnTablet(table_info, false, endpoint_map, cur_term) < 0 ||
CreateTableOnTablet(table_info, true, endpoint_map, cur_term) < 0) {
response.set_code(::openmldb::base::ReturnCode::kCreateTableFailedOnTablet);
response.set_msg("create table failed on tablet");
PDLOG(WARNING, "create table failed. name[%s] tid[%u]", table_info->name().c_str(), tid);
auto status = CreateTableOnTablet(table_info, false, cur_term, &endpoint_map);
if (!status.OK()) {
base::SetResponseStatus(status, &response);
PDLOG(WARNING, "create table failed. name[%s] tid[%u] msg[%s]",
table_info->name().c_str(), tid, status.GetMsg().c_str());
break;
}
status = CreateTableOnTablet(table_info, true, cur_term, &endpoint_map);
if (!status.OK()) {
base::SetResponseStatus(status, &response);
PDLOG(WARNING, "create table failed. name[%s] tid[%u] msg[%s]",
table_info->name().c_str(), tid, status.GetMsg().c_str());
break;
}
if (!IsClusterMode()) {
Expand Down
4 changes: 2 additions & 2 deletions src/nameserver/name_server_impl.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -306,8 +306,8 @@ class NameServerImpl : public NameServer {
const ::openmldb::nameserver::TableInfo& table_info_local, uint32_t pid, int& code, // NOLINT
std::string& msg); // NOLINT

int CreateTableOnTablet(const std::shared_ptr<::openmldb::nameserver::TableInfo>& table_info, bool is_leader,
std::map<uint32_t, std::vector<std::string>>& endpoint_map, uint64_t term); // NOLINT
base::Status CreateTableOnTablet(const std::shared_ptr<::openmldb::nameserver::TableInfo>& table_info,
bool is_leader, uint64_t term, std::map<uint32_t, std::vector<std::string>>* endpoint_map);

void CheckZkClient();

Expand Down
Loading

0 comments on commit a013ba3

Please sign in to comment.