From 82098202767985b8cc4cf1b441b8eabb25cd7aeb Mon Sep 17 00:00:00 2001 From: zd-double Date: Mon, 28 Mar 2016 10:16:59 +0800 Subject: [PATCH 01/13] add rpc cookie --- Makefile | 2 +- sample/echo_cookie/Makefile | 84 ++++ sample/echo_cookie/client_async.cc | 87 +++++ sample/echo_cookie/client_http.sh | 10 + sample/echo_cookie/client_sync.cc | 79 ++++ .../echo_cookie/cookie_plugin/rpc_cookie.cc | 180 +++++++++ sample/echo_cookie/cookie_plugin/rpc_cookie.h | 106 +++++ sample/echo_cookie/cookie_plugin/serialize.cc | 361 ++++++++++++++++++ sample/echo_cookie/cookie_plugin/serialize.h | 346 +++++++++++++++++ sample/echo_cookie/echo_service.proto | 20 + sample/echo_cookie/server.cc | 126 ++++++ src/sofa/pbrpc/binary_rpc_request.cc | 13 + src/sofa/pbrpc/rpc_attachment.h | 43 +++ src/sofa/pbrpc/rpc_client_impl.cc | 20 + src/sofa/pbrpc/rpc_client_stream.h | 8 +- src/sofa/pbrpc/rpc_controller.cc | 20 + src/sofa/pbrpc/rpc_controller.h | 9 + src/sofa/pbrpc/rpc_controller_impl.h | 49 +++ src/sofa/pbrpc/rpc_message_header.h | 9 +- src/sofa/pbrpc/rpc_message_stream.h | 20 +- 20 files changed, 1583 insertions(+), 9 deletions(-) create mode 100644 sample/echo_cookie/Makefile create mode 100644 sample/echo_cookie/client_async.cc create mode 100755 sample/echo_cookie/client_http.sh create mode 100644 sample/echo_cookie/client_sync.cc create mode 100644 sample/echo_cookie/cookie_plugin/rpc_cookie.cc create mode 100644 sample/echo_cookie/cookie_plugin/rpc_cookie.h create mode 100644 sample/echo_cookie/cookie_plugin/serialize.cc create mode 100644 sample/echo_cookie/cookie_plugin/serialize.h create mode 100644 sample/echo_cookie/echo_service.proto create mode 100644 sample/echo_cookie/server.cc create mode 100644 src/sofa/pbrpc/rpc_attachment.h diff --git a/Makefile b/Makefile index e9c2587..c080f1d 100644 --- a/Makefile +++ b/Makefile @@ -44,7 +44,7 @@ PUB_INC=src/sofa/pbrpc/pbrpc.h src/sofa/pbrpc/closure_helper.h src/sofa/pbrpc/cl src/sofa/pbrpc/locks.h src/sofa/pbrpc/mutex_lock.h src/sofa/pbrpc/spin_lock.h \ src/sofa/pbrpc/fast_lock.h src/sofa/pbrpc/rw_lock.h src/sofa/pbrpc/scoped_locker.h \ src/sofa/pbrpc/condition_variable.h src/sofa/pbrpc/wait_event.h src/sofa/pbrpc/http.h \ - src/sofa/pbrpc/buffer.h src/sofa/pbrpc/buf_handle.h \ + src/sofa/pbrpc/buffer.h src/sofa/pbrpc/buf_handle.h src/sofa/pbrpc/rpc_attachment.h \ $(PROTO) $(PROTO_HEADER) #----------------------------------------------- diff --git a/sample/echo_cookie/Makefile b/sample/echo_cookie/Makefile new file mode 100644 index 0000000..3b2daf3 --- /dev/null +++ b/sample/echo_cookie/Makefile @@ -0,0 +1,84 @@ +# Copyright (c) 2014 Baidu.com, Inc. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. See the AUTHORS file for names of contributors. + +#----------------------------------------------- +## Sofa-pbrpc path containing `include'and `lib'. +## +## Check file exist: +## $(SOFA_PBRPC)/include/sofa/pbrpc/pbrpc.h +## $(SOFA_PBRPC)/lib/libsofa-pbrpc.a +## +SOFA_PBRPC=../../output +#----------------------------------------------- + +#----------------------------------------------- +# Uncomment exactly one of the lines labelled (A), (B), and (C) below +# to switch between compilation modes. +# +OPT ?= -O2 # (A) Production use (optimized mode) +# OPT ?= -g2 # (B) Debug mode, w/ full line-level debugging symbols +# OPT ?= -O2 -g2 # (C) Profiling mode: opt, but w/debugging symbols +#----------------------------------------------- + +#----------------------------------------------- +# !!! Do not change the following lines !!! +#----------------------------------------------- + +include ../../depends.mk + +CXX=g++ +INCPATH=-I. -I$(SOFA_PBRPC)/include -I$(PROTOBUF_DIR)/include \ + -I$(SNAPPY_DIR)/include -I$(ZLIB_DIR)/include +CXXFLAGS += $(OPT) -pipe -W -Wall -fPIC -D_GNU_SOURCE -D__STDC_LIMIT_MACROS $(INCPATH) + +LIBRARY=$(SOFA_PBRPC)/lib/libsofa-pbrpc.a $(PROTOBUF_DIR)/lib/libprotobuf.a $(SNAPPY_DIR)/lib/libsnappy.a +LDFLAGS += -L$(ZLIB_DIR)/lib -lpthread -lz + +UNAME_S := $(shell uname -s) +ifeq ($(UNAME_S),Linux) + LDFLAGS += -lrt +endif + +PROTO_SRC=echo_service.proto +PROTO_OBJ=$(patsubst %.proto,%.pb.o,$(PROTO_SRC)) +PROTO_OPTIONS=--proto_path=. --proto_path=$(SOFA_PBRPC)/include --proto_path=$(PROTOBUF_DIR)/include + +BIN=server client_sync client_async + +all: check_depends $(BIN) + +.PHONY: check_depends clean + +check_depends: + @if [ ! -f "$(PROTOBUF_DIR)/include/google/protobuf/message.h" ]; then echo "ERROR: need protobuf header"; exit 1; fi + @if [ ! -f "$(PROTOBUF_DIR)/lib/libprotobuf.a" ]; then echo "ERROR: need protobuf lib"; exit 1; fi + @if [ ! -f "$(PROTOBUF_DIR)/bin/protoc" ]; then echo "ERROR: need protoc binary"; exit 1; fi + @if [ ! -f "$(SNAPPY_DIR)/include/snappy.h" ]; then echo "ERROR: need snappy header"; exit 1; fi + @if [ ! -f "$(SNAPPY_DIR)/lib/libsnappy.a" ]; then echo "ERROR: need snappy lib"; exit 1; fi + @if [ ! -f "$(SOFA_PBRPC)/include/sofa/pbrpc/pbrpc.h" ]; then echo "ERROR: need sofa-pbrpc header"; exit 1; fi + @if [ ! -f "$(SOFA_PBRPC)/lib/libsofa-pbrpc.a" ]; then echo "ERROR: need sofa-pbrpc lib"; exit 1; fi + +clean: + @rm -f $(BIN) *.o *.pb.* + +rebuild: clean all + +server: $(PROTO_OBJ) server.o cookie_plugin/rpc_cookie.o cookie_plugin/serialize.o + $(CXX) $^ -o $@ $(LIBRARY) $(LDFLAGS) + +client_sync: $(PROTO_OBJ) client_sync.o cookie_plugin/rpc_cookie.o cookie_plugin/serialize.o + $(CXX) $^ -o $@ $(LIBRARY) $(LDFLAGS) + +client_async: $(PROTO_OBJ) client_async.o cookie_plugin/rpc_cookie.o cookie_plugin/serialize.o + $(CXX) $^ -o $@ $(LIBRARY) $(LDFLAGS) + +%.pb.o: %.pb.cc + $(CXX) $(CXXFLAGS) -c $< -o $@ + +%.pb.cc: %.proto + $(PROTOBUF_DIR)/bin/protoc $(PROTO_OPTIONS) --cpp_out=. $< + +%.o: %.cc $(PROTO_OBJ) + $(CXX) $(CXXFLAGS) -c $< -o $@ + diff --git a/sample/echo_cookie/client_async.cc b/sample/echo_cookie/client_async.cc new file mode 100644 index 0000000..6bfdf66 --- /dev/null +++ b/sample/echo_cookie/client_async.cc @@ -0,0 +1,87 @@ +// Copyright (c) 2014 Baidu.com, Inc. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. +// +// Author: qinzuoyan01@baidu.com (Qin Zuoyan) + +#include +#include +#include "cookie_plugin/rpc_cookie.h" +#include "echo_service.pb.h" + + +typedef sofa::pbrpc::shared_ptr CookiePtr; +sofa::pbrpc::RpcCookieManager cookie_manager; + +void EchoCallback(sofa::pbrpc::RpcController* cntl, + sofa::pbrpc::test::EchoRequest* request, + sofa::pbrpc::test::EchoResponse* response, + bool* callbacked) +{ + SLOG(NOTICE, "RemoteAddress=%s", cntl->RemoteAddress().c_str()); + SLOG(NOTICE, "IsRequestSent=%s", cntl->IsRequestSent() ? "true" : "false"); + if (cntl->IsRequestSent()) + { + SLOG(NOTICE, "LocalAddress=%s", cntl->LocalAddress().c_str()); + SLOG(NOTICE, "SentBytes=%ld", cntl->SentBytes()); + } + + if (cntl->Failed()) { + SLOG(ERROR, "request failed: %s", cntl->ErrorText().c_str()); + } + else { + SLOG(NOTICE, "request succeed: %s", response->message().c_str()); + CookiePtr cookie(new sofa::pbrpc::Cookie(&cookie_manager)); + cntl->GetResponseAttachment(cookie.get()); + std::string version; + cookie->Get("version", version); + SLOG(NOTICE, "cookie version : %s", version.c_str()); + cookie->Store(); + } + + delete cntl; + delete request; + delete response; + *callbacked = true; +} + +int main() +{ + SOFA_PBRPC_SET_LOG_LEVEL(NOTICE); + + // Define an rpc server. + sofa::pbrpc::RpcClientOptions client_options; + sofa::pbrpc::RpcClient rpc_client(client_options); + + // Define an rpc channel. + sofa::pbrpc::RpcChannelOptions channel_options; + sofa::pbrpc::RpcChannel rpc_channel(&rpc_client, "127.0.0.1:12300", channel_options); + + CookiePtr cookie(new sofa::pbrpc::Cookie(&cookie_manager)); + cookie->Set("type", "async"); + cookie->Set("logid", "123456"); + + // Prepare parameters. + sofa::pbrpc::RpcController* cntl = new sofa::pbrpc::RpcController(); + cntl->SetTimeout(3000); + cntl->SetRequestAttachment(cookie.get()); + sofa::pbrpc::test::EchoRequest* request = new sofa::pbrpc::test::EchoRequest(); + request->set_message("Hello from qinzuoyan01"); + sofa::pbrpc::test::EchoResponse* response = new sofa::pbrpc::test::EchoResponse(); + bool callbacked = false; + google::protobuf::Closure* done = sofa::pbrpc::NewClosure( + &EchoCallback, cntl, request, response, &callbacked); + + // Async call. + sofa::pbrpc::test::EchoServer_Stub stub(&rpc_channel); + stub.Echo(cntl, request, response, done); + + // Wait call done. + while (!callbacked) { + usleep(100000); + } + + return EXIT_SUCCESS; +} + +/* vim: set ts=4 sw=4 sts=4 tw=100 */ diff --git a/sample/echo_cookie/client_http.sh b/sample/echo_cookie/client_http.sh new file mode 100755 index 0000000..ab72311 --- /dev/null +++ b/sample/echo_cookie/client_http.sh @@ -0,0 +1,10 @@ +#!/bin/sh + +# POST +curl -d '{"message":"Hello, world!"}' http://localhost:12321/sofa.pbrpc.test.EchoServer.Echo +echo + +# GET +curl http://localhost:12321/sofa.pbrpc.test.EchoServer.Echo?request=%7B%22message%22%3A%22Hello%2C%20world%21%22%7D +echo + diff --git a/sample/echo_cookie/client_sync.cc b/sample/echo_cookie/client_sync.cc new file mode 100644 index 0000000..e029a94 --- /dev/null +++ b/sample/echo_cookie/client_sync.cc @@ -0,0 +1,79 @@ +// Copyright (c) 2014 Baidu.com, Inc. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. +// +// Author: qinzuoyan01@baidu.com (Qin Zuoyan) + +#include +#include "cookie_plugin/rpc_cookie.h" +#include "echo_service.pb.h" +#include + +typedef sofa::pbrpc::shared_ptr CookiePtr; +sofa::pbrpc::RpcCookieManager cookie_manager; + +// Using global RpcClient object can help share resources such as threads and buffers. +sofa::pbrpc::RpcClient g_rpc_client; + +int main() +{ + SOFA_PBRPC_SET_LOG_LEVEL(NOTICE); + + // Define an rpc channel. + sofa::pbrpc::RpcChannelOptions channel_options; + sofa::pbrpc::RpcChannel rpc_channel(&g_rpc_client, "127.0.0.1:12300", channel_options); + + // Prepare parameters. + sofa::pbrpc::RpcController* cntl = new sofa::pbrpc::RpcController(); + cntl->SetTimeout(30000); + CookiePtr cookie(new sofa::pbrpc::Cookie(&cookie_manager)); + cookie->Set("type", "sync"); + cookie->Set("logid", "123456"); + cntl->SetRequestAttachment(cookie.get()); + sofa::pbrpc::test::EchoRequest* request = + new sofa::pbrpc::test::EchoRequest(); + request->set_message("Hello from qinzuoyan01"); + sofa::pbrpc::test::EchoResponse* response = + new sofa::pbrpc::test::EchoResponse(); + + // Sync call. + sofa::pbrpc::test::EchoServer_Stub* stub = + new sofa::pbrpc::test::EchoServer_Stub(&rpc_channel); + stub->Echo(cntl, request, response, NULL); + + // Check if the request has been sent. + // If has been sent, then can get the sent bytes. + SLOG(NOTICE, "RemoteAddress=%s", cntl->RemoteAddress().c_str()); + SLOG(NOTICE, "IsRequestSent=%s", cntl->IsRequestSent() ? "true" : "false"); + if (cntl->IsRequestSent()) + { + SLOG(NOTICE, "LocalAddress=%s", cntl->LocalAddress().c_str()); + SLOG(NOTICE, "SentBytes=%ld", cntl->SentBytes()); + } + + // Check if failed. + if (cntl->Failed()) + { + SLOG(ERROR, "request failed: %s", cntl->ErrorText().c_str()); + } + else + { + SLOG(NOTICE, "request succeed: %s", response->message().c_str()); + cookie.reset(new sofa::pbrpc::Cookie(&cookie_manager)); + cntl->GetResponseAttachment(cookie.get()); + std::string version; + cookie->Get("version", version); + SLOG(NOTICE, "cookie version : %s", version.c_str()); + cookie->Store(); + } + + // Destroy objects. + delete cntl; + delete request; + delete response; + delete stub; + + return EXIT_SUCCESS; +} + +/* vim: set ts=4 sw=4 sts=4 tw=100 */ diff --git a/sample/echo_cookie/cookie_plugin/rpc_cookie.cc b/sample/echo_cookie/cookie_plugin/rpc_cookie.cc new file mode 100644 index 0000000..5234cbd --- /dev/null +++ b/sample/echo_cookie/cookie_plugin/rpc_cookie.cc @@ -0,0 +1,180 @@ +/*************************************************************************** + * + * Copyright (c) 2016 Baidu.com, Inc. All Rights Reserved + * $Id$ + * + **************************************************************************/ + + /** + * @file rpc_cookie.cc + * @author zhangdi05(zhangdi05@baidu.com) + * @date 2016/03/18 13:43:53 + * @version $Revision$ + * @brief + * + **/ + +#include "serialize.h" +#include "rpc_cookie.h" + +namespace sofa { +namespace pbrpc { + +bool RpcCookieManager::Has(const std::string& key) +{ + ScopedLocker _(_lock); + return _kv_map.find(key) != _kv_map.end(); +} + +void RpcCookieManager::Load(MapType& kv_map) +{ + ScopedLocker _(_lock); + kv_map = _kv_map; +} + +void RpcCookieManager::Store(const MapType& kv_map) +{ + ScopedLocker _(_lock); + _kv_map.insert(kv_map.begin(), kv_map.end()); +} + +Cookie::Cookie() : _keep_type(ALWAYS_NEW) +{ } + +Cookie::Cookie(RpcCookieManager* manager, KeepType keep_type) : + _manager(manager), _keep_type(keep_type) +{ + if (_keep_type == ALWAYS_NEW) + { + return; + } + else if (_manager != NULL) + { + _manager->Load(_kv_map); + } +} + +bool Cookie::Get(const std::string& key, std::string& value) +{ + MapType::const_iterator it = _kv_map.find(key); + if (it == _kv_map.end()) + { + return false; + } + value = it->second; + return true; +} + +void Cookie::Set(const std::string& key, const std::string& value) +{ + if (_keep_type == MANAGER_FIRST) + { + if (!_manager->Has(key)) + { + _kv_map[key] = value; + } + return; + } + _kv_map[key] = value; +} + +void Cookie::Store() +{ + if (_manager != NULL) + { + _manager->Store(_kv_map); + } +} + +bool Cookie::Has(const std::string& key) +{ + return _kv_map.find(key) != _kv_map.end(); +} + +bool Cookie::Erase(const std::string& key) +{ + MapType::iterator it = _kv_map.find(key); + if (it == _kv_map.end()) + { + return false; + } + _kv_map.erase(key); + return true; +} + +void Cookie::Clear() +{ + _kv_map.clear(); +} + +bool Cookie::Serialize(sofa::pbrpc::ReadBufferPtr& attach_buffer) +{ + int size = _kv_map.size(); + SCHECK_LE(static_cast(size), SOFA_CONTAINER_MAX_SERIALIZE_SIZE); + WriteBufferPtr write_buffer(new WriteBuffer()); + Serializer serializer(write_buffer); + if (!serializer.serialize_varint(size)) + { + return false; + } + MapType::const_iterator it = _kv_map.begin(); + for (; it != _kv_map.end(); ++it) + { + if (!serializer.serialize_string(it->first)) + { + return false; + } + if (!serializer.serialize_string(it->second)) + { + return false; + } + } + write_buffer->SwapOut(attach_buffer.get()); + return true; +} + +int Cookie::SerializeLen() const +{ + int len = Serializer::varint_len(_kv_map.size()); + MapType::const_iterator it = _kv_map.begin(); + for (; it != _kv_map.end(); ++it) + { + len += Serializer::string_len(it->first); + len += Serializer::string_len(it->second); + } + return len; +} + +bool Cookie::Deserialize(sofa::pbrpc::ReadBufferPtr& attach_buffer) +{ + int size = 0; + Deserializer deserializer(attach_buffer); + if (!deserializer.deserialize_varint(size)) + { + return false; + } + if (static_cast(size) > SOFA_CONTAINER_MAX_SERIALIZE_SIZE) + { + return false; + } + _kv_map.clear(); + for (int i = 0; i < size; ++i) + { + std::string key; + if (!deserializer.deserialize_string(key)) + { + return false; + } + std::string value; + if (!deserializer.deserialize_string(value)) + { + return false; + } + _kv_map[key] = value; + } + return true; +} +} // namespace pbrpc +} // namespace sofa + +/* vim: set ts=4 sw=4 sts=4 tw=100 */ diff --git a/sample/echo_cookie/cookie_plugin/rpc_cookie.h b/sample/echo_cookie/cookie_plugin/rpc_cookie.h new file mode 100644 index 0000000..72e890c --- /dev/null +++ b/sample/echo_cookie/cookie_plugin/rpc_cookie.h @@ -0,0 +1,106 @@ +/*************************************************************************** + * + * Copyright (c) 2016 Baidu.com, Inc. All Rights Reserved + * $Id$ + * + **************************************************************************/ + + /** + * @file rpc_cookie.h + * @author zhangdi05(zhangdi05@baidu.com) + * @date 2016/03/18 13:32:12 + * @version $Revision$ + * @brief + * + **/ +#ifndef _SOFA_PBRPC_RPC_COOKIE_H_ +#define _SOFA_PBRPC_RPC_COOKIE_H_ + +#include +#include +#include +#include +#include +#include +#include + +namespace sofa { +namespace pbrpc { + +enum KeepType +{ + ALWAYS_NEW = 0, + USER_FIRST = 1, + MANAGER_FIRST = 2 +}; + +class RpcCookieManager; +typedef std::map MapType; + +class RpcCookieManager +{ +public: + RpcCookieManager() + { } + + ~RpcCookieManager() + { } + + bool Has(const std::string& key); + + void Load(MapType& kv_map); + + void Store(const MapType& kv_map); + +private: + MapType _kv_map; + + MutexLock _lock; +}; + +typedef std::map MapType; + +class Cookie : public sofa::pbrpc::RpcAttachment +//class Cookie +{ +public: + Cookie(); + + Cookie(RpcCookieManager* manager, KeepType keep_type = USER_FIRST); + + virtual ~Cookie() { } + + bool Get(const std::string& key, std::string& value); + + void Set(const std::string& key, const std::string& value); + + bool Has(const std::string& key); + + bool Erase(const std::string& key); + + void Clear(); + + void Store(); + + virtual bool Serialize(sofa::pbrpc::ReadBufferPtr& attach_buffer); + + virtual bool Deserialize(sofa::pbrpc::ReadBufferPtr& attach_buffer); + + int SerializeLen() const; + +private: + MapType _kv_map; + + RpcCookieManager* _manager; + + KeepType _keep_type; + + SOFA_PBRPC_DISALLOW_EVIL_CONSTRUCTORS(Cookie); +}; //class RpcCookie + +} // namespace pbrpc +} // namespace sofa + +#endif // _SOFA_PBRPC_RPC_COOKIE_H_ + +/* vim: set ts=4 sw=4 sts=4 tw=100 */ diff --git a/sample/echo_cookie/cookie_plugin/serialize.cc b/sample/echo_cookie/cookie_plugin/serialize.cc new file mode 100644 index 0000000..a1a83ec --- /dev/null +++ b/sample/echo_cookie/cookie_plugin/serialize.cc @@ -0,0 +1,361 @@ +#include +#include +//#include + +namespace sofa { +namespace pbrpc { + +// NOTICE: When compiling by gcc with '-O2' option, the data may be out of order. +// It seems a bug of gcc, fuck it. So we use 'sofa_io_barrier()' to prevent it. +#define sofa_io_barrier() __asm__ __volatile__("": : :"memory") + +// fast copy small data block +static inline void fast_memcpy(void* dest, const void* src, size_t n) +{ + switch (n) + { + case 0: + return; + case 1: + *(uint8_t*)dest = *(const uint8_t*)src; + return; + case 2: + *(uint16_t*)dest = *(const uint16_t*)src; + return; + case 3: + *(byte_helper_t<3>*)dest = *(const byte_helper_t<3>*)src; + return; + case 4: + *(uint32_t*)dest = *(const uint32_t*)src; + return; + case 5: + *(byte_helper_t<5>*)dest = *(const byte_helper_t<5>*)src; + return; + case 6: + *(byte_helper_t<6>*)dest = *(const byte_helper_t<6>*)src; + return; + case 7: + *(byte_helper_t<7>*)dest = *(const byte_helper_t<7>*)src; + return; + case 8: + *(uint64_t*)dest = *(const uint64_t*)src; + return; + case 9: + *(byte_helper_t<9>*)dest = *(const byte_helper_t<9>*)src; + return; + case 10: + *(byte_helper_t<10>*)dest = *(const byte_helper_t<10>*)src; + return; + case 11: + *(byte_helper_t<11>*)dest = *(const byte_helper_t<11>*)src; + return; + case 12: + *(byte_helper_t<12>*)dest = *(const byte_helper_t<12>*)src; + return; + case 13: + *(byte_helper_t<13>*)dest = *(const byte_helper_t<13>*)src; + return; + case 14: + *(byte_helper_t<14>*)dest = *(const byte_helper_t<14>*)src; + return; + case 15: + *(byte_helper_t<15>*)dest = *(const byte_helper_t<15>*)src; + return; + case 16: + *(byte_helper_t<16>*)dest = *(const byte_helper_t<16>*)src; + return; + default: + memcpy(dest, src, n); + } +} + +bool Serializer::close() +{ + bool ret = true; + if (_buf_size > 0) + { + _stream->BackUp(_buf_size); + } + //_stream = NULL; + _buf = NULL; + _buf_size = 0; + return ret; +} + +bool Serializer::serialize_buffer(const void* data, int data_len) +{ + while (_buf_size < data_len) + { + fast_memcpy(_buf, data, _buf_size); + data = ((const char*)data) + _buf_size; + data_len -= _buf_size; + if (!_stream->Next(&_buf, &_buf_size)) + { + _buf_size = 0; + return false; + } + } + fast_memcpy(_buf, data, data_len); + _buf = ((char*)_buf) + data_len; + _buf_size -= data_len; + return true; +} + +bool Serializer::serialize_string(const std::string& d) +{ + if (d.size() > SOFA_STRING_MAX_SERIALIZE_SIZE) + { + SLOG(ERROR, "too big string size: %u", d.size()); + return false; + } + if (!serialize_varint(d.size())) + { + return false; + } + return serialize_buffer(d.c_str(), d.size()); +} + +// NOTICE: When compiling by gcc with '-O2' option, the data may be out of order. +// It occurs between modify 'd' and 'serialize_X_byte', so we use 'sofa_io_barrier()' to prevent it. +// Fuck gcc again! +bool Serializer::serialize_varint(uint64_t d) +{ + if (d < (1UL << 7)) + { + d <<= 1; + sofa_io_barrier(); + return serialize_1_byte(&d); + } + if (d < (1UL << 14)) + { + d <<= 2; + d |= 0x1; + sofa_io_barrier(); + return serialize_2_byte(&d); + } + if (d < (1UL << 21)) + { + d <<= 3; + d |= 0x3; + sofa_io_barrier(); + return serialize_3_byte(&d); + } + if (d < (1UL << 28)) + { + d <<= 4; + d |= 0x7; + sofa_io_barrier(); + return serialize_4_byte(&d); + } + if (d < (1UL << 35)) + { + d <<= 5; + d |= 0xF; + sofa_io_barrier(); + return serialize_5_byte(&d); + } + if (d < (1UL << 42)) + { + d <<= 6; + d |= 0x1F; + sofa_io_barrier(); + return serialize_6_byte(&d); + } + if (d < (1UL << 49)) + { + d <<= 7; + d |= 0x3F; + sofa_io_barrier(); + return serialize_7_byte(&d); + } + if (d < (1UL << 56)) + { + d <<= 8; + d |= 0x7F; + sofa_io_barrier(); + return serialize_8_byte(&d); + } + return serialize_raw_data((uint8_t)0xFF) && serialize_raw_data(d); +} + +size_t Serializer::varint_len(uint64_t d) +{ + if (d < (1UL << 7)) + { + return 1; + } + if (d < (1UL << 14)) + { + return 2; + } + if (d < (1UL << 21)) + { + return 3; + } + if (d < (1UL << 28)) + { + return 4; + } + if (d < (1UL << 35)) + { + return 5; + } + if (d < (1UL << 42)) + { + return 6; + } + if (d < (1UL << 49)) + { + return 7; + } + if (d < (1UL << 56)) + { + return 8; + } + return 9; +} + +void Deserializer::close() +{ + if (_buf_size > 0) + { + _stream->BackUp(_buf_size); + } + //_stream = NULL; + _buf = NULL; + _buf_size = 0; +} + +bool Deserializer::deserialize_buffer(void* data, size_t data_len) +{ + while (static_cast(_buf_size) < data_len) + { + fast_memcpy(data, _buf, _buf_size); + data = ((char*)data) + _buf_size; + data_len -= _buf_size; + if (!_stream->Next(&_buf, &_buf_size)) + { + _buf_size = 0; + return false; + } + } + fast_memcpy(data, _buf, data_len); + _buf = ((const char*)_buf) + data_len; + _buf_size -= data_len; + return true; +} + +bool Deserializer::deserialize_string(std::string& d) +{ + size_t str_len = 0; + if (!deserialize_varint(str_len)) + { + return false; + } + if (str_len == 0) + { + d.clear(); + return true; + } + if (str_len > SOFA_STRING_MAX_SERIALIZE_SIZE) + { + SLOG(ERROR, "too big string size: %u", str_len); + return false; + } + StringUtils::resize_uninitialized(&d, str_len); + return deserialize_buffer(StringUtils::string_as_array(&d), str_len); +} + +bool Deserializer::deserialize_varint(uint64_t& d) +{ + d = 0; + if (!deserialize_raw_data(*(uint8_t*)&d)) + { + return false; + } + if ((d & 0x1) == 0) + { + d >>= 1; + return true; + } + if ((d & 0x2) == 0) + { + if (!deserialize_1_byte((uint8_t*)&d + 1)) + { + return false; + } + sofa_io_barrier(); + d >>= 2; + return true; + } + if ((d & 0x4) == 0) + { + if (!deserialize_2_byte((uint8_t*)&d + 1)) + { + return false; + } + sofa_io_barrier(); + d >>= 3; + return true; + } + if ((d & 0x8) == 0) + { + if (!deserialize_3_byte((uint8_t*)&d + 1)) + { + return false; + } + sofa_io_barrier(); + d >>= 4; + return true; + } + if ((d & 0x10) == 0) + { + if (!deserialize_4_byte((uint8_t*)&d + 1)) + { + return false; + } + sofa_io_barrier(); + d >>= 5; + return true; + } + if ((d & 0x20) == 0) + { + if (!deserialize_5_byte((uint8_t*)&d + 1)) + { + return false; + } + sofa_io_barrier(); + d >>= 6; + return true; + } + if ((d & 0x40) == 0) + { + if (!deserialize_6_byte((uint8_t*)&d + 1)) + { + return false; + } + sofa_io_barrier(); + d >>= 7; + return true; + } + if ((d & 0x80) == 0) + { + if (!deserialize_7_byte((uint8_t*)&d + 1)) + { + return false; + } + sofa_io_barrier(); + d >>= 8; + return true; + } + return deserialize_raw_data(d); +} + +} // namespace io +} // namespace sofa + +void sofa_touch_sofa_io_serializer() +{ +} + +/* vim: set ts=4 sw=4 sts=4 tw=100 */ diff --git a/sample/echo_cookie/cookie_plugin/serialize.h b/sample/echo_cookie/cookie_plugin/serialize.h new file mode 100644 index 0000000..8f14b7e --- /dev/null +++ b/sample/echo_cookie/cookie_plugin/serialize.h @@ -0,0 +1,346 @@ +#ifndef SOFA_IO_SERIALIZER_H +#define SOFA_IO_SERIALIZER_H + +#include +#include +//#include +#include + +// String max serialize size: 64MB. +#define SOFA_STRING_MAX_SERIALIZE_SIZE 67108864u + +// Container max serialize size: 64MB. +#define SOFA_CONTAINER_MAX_SERIALIZE_SIZE 67108864u + +namespace sofa { +namespace pbrpc { + +// helper for fast copy small data block +template +struct byte_helper_t +{ + uint8_t data[N]; +}; + +// Serializer on output stream +class Serializer +{ +public: + explicit Serializer(WriteBufferPtr& stream) : _stream(stream) + { + _buf = NULL; + _buf_size = 0; + } + void set_stream(WriteBufferPtr& stream) + { + close(); + _stream = stream; + } + ~Serializer() + { + //close(); + } + // close serializer, backup un-used buffer + bool close(); + + template + bool serialize_raw_data(const T& d) + { + if (static_cast(_buf_size) >= sizeof(T)) + { + *(T*)_buf = d; + _buf = ((char*)_buf) + sizeof(T); + _buf_size -= sizeof(T); + return true; + } + return serialize_buffer(&d, sizeof(T)); + } + template + static size_t raw_data_len(const T&) + { + return sizeof(T); + } + template + static size_t max_raw_data_len(const T&) + { + return sizeof(T); + } + bool serialize_1_byte(const void __attribute__((may_alias)) * d) + { + return serialize_raw_data(*(const uint8_t __attribute__((may_alias))*)d); + } + bool serialize_2_byte(const void* d) + { + return serialize_raw_data(*(const uint16_t*)d); + } + bool serialize_3_byte(const void* d) + { + return serialize_raw_data(*(const byte_helper_t<3>*)d); + } + bool serialize_4_byte(const void* d) + { + return serialize_raw_data(*(const uint32_t*)d); + } + bool serialize_5_byte(const void* d) + { + return serialize_raw_data(*(const byte_helper_t<5>*)d); + } + bool serialize_6_byte(const void* d) + { + return serialize_raw_data(*(const byte_helper_t<6>*)d); + } + bool serialize_7_byte(const void* d) + { + return serialize_raw_data(*(const byte_helper_t<7>*)d); + } + bool serialize_8_byte(const void* d) + { + return serialize_raw_data(*(const uint64_t*)d); + } + bool serialize_buffer(const void* data, int data_len); + bool serialize_string(const std::string& d); + static size_t string_len(const std::string& d) + { + return varint_len(d.size()) + d.size(); + } + bool serialize_varint(uint8_t d) + { + return serialize_varint((uint64_t)d); + } + static size_t varint_len(uint8_t d) + { + return varint_len((uint64_t)d); + } + static size_t max_varint_len(uint8_t) + { + return max_varint_len((uint64_t)0); + } + bool serialize_varint(uint16_t d) + { + return serialize_varint((uint64_t)d); + } + static size_t varint_len(uint16_t d) + { + return varint_len((uint64_t)d); + } + static size_t max_varint_len(uint16_t) + { + return max_varint_len((uint64_t)0); + } + bool serialize_varint(uint32_t d) + { + return serialize_varint((uint64_t)d); + } + static size_t varint_len(uint32_t d) + { + return varint_len((uint64_t)d); + } + static size_t max_varint_len(uint32_t) + { + return max_varint_len((uint64_t)0); + } + bool serialize_varint(int8_t d) + { + return serialize_varint((int64_t)d); + } + static size_t varint_len(int8_t d) + { + return varint_len((int64_t)d); + } + static size_t max_varint_len(int8_t) + { + return max_varint_len((int64_t)0); + } + bool serialize_varint(int16_t d) + { + return serialize_varint((int64_t)d); + } + static size_t varint_len(int16_t d) + { + return varint_len((int64_t)d); + } + static size_t max_varint_len(int16_t) + { + return max_varint_len((int64_t)0); + } + bool serialize_varint(int32_t d) + { + return serialize_varint((int64_t)d); + } + static size_t varint_len(int32_t d) + { + return varint_len((int64_t)d); + } + static size_t max_varint_len(int32_t) + { + return max_varint_len((int64_t)0); + } + bool serialize_varint(int64_t d) + { + return serialize_varint((uint64_t)((d << 1) ^ (d >> 63))); + } + static size_t varint_len(int64_t d) + { + return varint_len((uint64_t)((d << 1) ^ (d >> 63))); + } + static size_t max_varint_len(int64_t) + { + return max_varint_len((uint64_t)0); + } + bool serialize_varint(uint64_t d); + static size_t varint_len(uint64_t d); + static size_t max_varint_len(uint64_t) + { + return 9; + } +private: + void* _buf; // current buffer to write + int _buf_size; // size of current buffer to write + WriteBufferPtr _stream; // output stream +}; + +// Deserializer on input stream +class Deserializer +{ +public: + explicit Deserializer(ReadBufferPtr& stream) : _stream(stream) + { + _buf = NULL; + _buf_size = 0; + } + ~Deserializer() + { + //close(); + } + // close deserializer, backup un-used buffer + void close(); + + template + bool deserialize_raw_data(T& d) + { + if (static_cast(_buf_size) >= sizeof(T)) + { + d = *(const T*)_buf; + _buf = ((const char*)_buf) + sizeof(T); + _buf_size -= sizeof(T); + return true; + } + return deserialize_buffer(&d, sizeof(T)); + } + bool deserialize_1_byte(void* d) + { + return deserialize_raw_data(*(uint8_t*)d); + } + bool deserialize_2_byte(void* d) + { + return deserialize_raw_data(*(uint16_t*)d); + } + bool deserialize_3_byte(void* d) + { + return deserialize_raw_data(*(byte_helper_t<3>*)d); + } + bool deserialize_4_byte(void* d) + { + return deserialize_raw_data(*(uint32_t*)d); + } + bool deserialize_5_byte(void* d) + { + return deserialize_raw_data(*(byte_helper_t<5>*)d); + } + bool deserialize_6_byte(void* d) + { + return deserialize_raw_data(*(byte_helper_t<6>*)d); + } + bool deserialize_7_byte(void* d) + { + return deserialize_raw_data(*(byte_helper_t<7>*)d); + } + bool deserialize_8_byte(void* d) + { + return deserialize_raw_data(*(uint64_t*)d); + } + bool deserialize_buffer(void* data, size_t data_len); + bool deserialize_string(std::string& d); + bool deserialize_varint(uint8_t& d) + { + uint64_t tmp; + if (!deserialize_varint(tmp)) + { + return false; + } + d = tmp; + return true; + } + bool deserialize_varint(uint16_t& d) + { + uint64_t tmp; + if (!deserialize_varint(tmp)) + { + return false; + } + d = tmp; + return true; + } + bool deserialize_varint(uint32_t& d) + { + uint64_t tmp; + if (!deserialize_varint(tmp)) + { + return false; + } + d = tmp; + return true; + } + bool deserialize_varint(int8_t& d) + { + int64_t tmp; + if (!deserialize_varint(tmp)) + { + return false; + } + d = tmp; + return true; + } + bool deserialize_varint(int16_t& d) + { + int64_t tmp; + if (!deserialize_varint(tmp)) + { + return false; + } + d = tmp; + return true; + } + bool deserialize_varint(int32_t& d) + { + int64_t tmp; + if (!deserialize_varint(tmp)) + { + return false; + } + d = tmp; + return true; + } + bool deserialize_varint(int64_t& d) + { + if (!deserialize_varint((uint64_t&)d)) + { + return false; + } + d = (((d & 1) << 63 >> 63) ^ (((uint64_t)d) >> 1)); + return true; + } + bool deserialize_varint(uint64_t& d); +private: + const void* _buf; // current buffer to read + int _buf_size; // size of current buffer to read + //InputStream* _stream; // input stream + ReadBufferPtr _stream; +}; + +} // namespace io +} // namespace sofa + +#endif // SOFA_IO_SERIALIZER_H + +/* vim: set ts=4 sw=4 sts=4 tw=100 */ diff --git a/sample/echo_cookie/echo_service.proto b/sample/echo_cookie/echo_service.proto new file mode 100644 index 0000000..7886d13 --- /dev/null +++ b/sample/echo_cookie/echo_service.proto @@ -0,0 +1,20 @@ +import "sofa/pbrpc/rpc_option.proto"; + +package sofa.pbrpc.test; + +option cc_generic_services = true; + +message EchoRequest { + required string message = 1; +} + +message EchoResponse { + required string message = 1; +} + +service EchoServer { + rpc Echo(EchoRequest) returns(EchoResponse) { + option (sofa.pbrpc.request_compress_type) = CompressTypeSnappy; + option (sofa.pbrpc.response_compress_type) = CompressTypeSnappy; + } +} diff --git a/sample/echo_cookie/server.cc b/sample/echo_cookie/server.cc new file mode 100644 index 0000000..07b150c --- /dev/null +++ b/sample/echo_cookie/server.cc @@ -0,0 +1,126 @@ +// Copyright (c) 2014 Baidu.com, Inc. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. +// +// Author: qinzuoyan01@baidu.com (Qin Zuoyan) + +#include +#include +#include +#include "cookie_plugin/rpc_cookie.h" +#include "echo_service.pb.h" + +typedef sofa::pbrpc::shared_ptr CookiePtr; +//sofa::pbrpc::RpcCookieManager cookie_manager; + +bool WebServlet(const sofa::pbrpc::HTTPRequest& request, sofa::pbrpc::HTTPResponse& response) +{ + SLOG(INFO, "WebServlet(): request message from %s:%u", + request.client_ip.c_str(), request.client_port); + SLOG(INFO, "HTTP-PATH=\"%s\"", request.path.c_str()); + std::map::const_iterator it; + const std::map& query_params = *request.query_params; + for (it = query_params.begin(); it != query_params.end(); ++it) { + SLOG(INFO, "QueryParam[\"%s\"]=\"%s\"", it->first.c_str(), it->second.c_str()); + } + const std::map& headers = *request.headers; + for (it = headers.begin(); it != headers.end(); ++it) { + SLOG(INFO, "Header[\"%s\"]=\"%s\"", it->first.c_str(), it->second.c_str()); + } + return response.content->Append("

Hello from sofa-pbrpc web server

"); +} + +class EchoServerImpl : public sofa::pbrpc::test::EchoServer +{ +public: + EchoServerImpl() {} + virtual ~EchoServerImpl() {} + +private: + virtual void Echo(google::protobuf::RpcController* controller, + const sofa::pbrpc::test::EchoRequest* request, + sofa::pbrpc::test::EchoResponse* response, + google::protobuf::Closure* done) + { + sofa::pbrpc::RpcController* cntl = static_cast(controller); + SLOG(INFO, "Echo(): request message from %s: %s", + cntl->RemoteAddress().c_str(), request->message().c_str()); + if (cntl->IsHttp()) { + SLOG(INFO, "HTTP-PATH=\"%s\"", cntl->HttpPath().c_str()); + std::map::const_iterator it; + const std::map& query_params = cntl->HttpQueryParams(); + for (it = query_params.begin(); it != query_params.end(); ++it) { + SLOG(INFO, "QueryParam[\"%s\"]=\"%s\"", it->first.c_str(), it->second.c_str()); + } + const std::map& headers = cntl->HttpHeaders(); + for (it = headers.begin(); it != headers.end(); ++it) { + SLOG(INFO, "Header[\"%s\"]=\"%s\"", it->first.c_str(), it->second.c_str()); + } + } + CookiePtr cookie(new sofa::pbrpc::Cookie()); + cntl->GetRequestAttachment(cookie.get()); + std::string type; + std::string logid; + cookie->Get("type", type); + cookie->Get("logid", logid); + SLOG(INFO, "cookie info : type=%s, logid=%s", type.c_str(), logid.c_str()); + response->set_message("echo message: " + request->message()); + cookie->Set("version", "1.00"); + cntl->SetResponseAttachment(cookie.get()); + done->Run(); + } +}; + +bool thread_init_func() +{ + sleep(1); + SLOG(INFO, "Init work thread succeed"); + return true; +} + +void thread_dest_func() +{ + SLOG(INFO, "Destroy work thread succeed"); +} + +int main() +{ + SOFA_PBRPC_SET_LOG_LEVEL(NOTICE); + + // Define an rpc server. + sofa::pbrpc::RpcServerOptions options; + options.work_thread_init_func = sofa::pbrpc::NewPermanentExtClosure(&thread_init_func); + options.work_thread_dest_func = sofa::pbrpc::NewPermanentExtClosure(&thread_dest_func); + sofa::pbrpc::RpcServer rpc_server(options); + + sofa::pbrpc::Servlet servlet = sofa::pbrpc::NewPermanentExtClosure(&WebServlet); + rpc_server.RegisterWebServlet("/hello", servlet); + + // Start rpc server. + if (!rpc_server.Start("0.0.0.0:12300")) { + SLOG(ERROR, "start server failed"); + return EXIT_FAILURE; + } + + // Register service. + sofa::pbrpc::test::EchoServer* echo_service = new EchoServerImpl(); + if (!rpc_server.RegisterService(echo_service)) { + SLOG(ERROR, "export service failed"); + return EXIT_FAILURE; + } + + // Wait signal. + rpc_server.Run(); + + // Stop rpc server. + rpc_server.Stop(); + + // Delete closures. + // Attention: should delete the closures after server stopped, or may be crash. + delete options.work_thread_init_func; + delete options.work_thread_dest_func; + + return EXIT_SUCCESS; +} + +/* vim: set ts=4 sw=4 sts=4 tw=100 */ diff --git a/src/sofa/pbrpc/binary_rpc_request.cc b/src/sofa/pbrpc/binary_rpc_request.cc index aa07962..be358be 100644 --- a/src/sofa/pbrpc/binary_rpc_request.cc +++ b/src/sofa/pbrpc/binary_rpc_request.cc @@ -119,6 +119,12 @@ void BinaryRpcRequest::ProcessRequest( cntl->SetResponseCompressType(_req_meta.has_expected_response_compress_type() ? _req_meta.expected_response_compress_type() : CompressTypeNone); + WriteBuffer write_buffer; + write_buffer.Append(_req_header.attach_buffer, _req_header.attach_size); + ReadBufferPtr read_buffer(new ReadBuffer()); + write_buffer.SwapOut(read_buffer.get()); + cntl->SetRequestAttachBuffer(read_buffer); + CallMethod(method_board, controller, request, response); } @@ -167,6 +173,13 @@ ReadBufferPtr BinaryRpcRequest::AssembleSucceedResponse( } header.data_size = write_buffer.ByteCount() - header_pos - header_size - header.meta_size; header.message_size = header.meta_size + header.data_size; + + std::string response_attach_str = cntl->GetResponseAttachBuffer()->ToString(); + size_t attach_size = response_attach_str.size(); + header.attach_size = attach_size; + memcpy(header.attach_buffer, response_attach_str.c_str(), attach_size); + + write_buffer.SetData(header_pos, reinterpret_cast(&header), header_size); ReadBufferPtr read_buffer(new ReadBuffer()); diff --git a/src/sofa/pbrpc/rpc_attachment.h b/src/sofa/pbrpc/rpc_attachment.h new file mode 100644 index 0000000..8ba6f0d --- /dev/null +++ b/src/sofa/pbrpc/rpc_attachment.h @@ -0,0 +1,43 @@ +/*************************************************************************** + * + * Copyright (c) 2016 Baidu.com, Inc. All Rights Reserved + * $Id$ + * + **************************************************************************/ + + /** + * @file rpc_attachment.h + * @author zhangdi05(zhangdi05@baidu.com) + * @date 2016/03/16 11:14:32 + * @version $Revision$ + * @brief + * + **/ +#ifndef RPC_ATTACHMENT_H +#define RPC_ATTACHMENT_H + +#include +#include + +namespace sofa { +namespace pbrpc { + +class RpcAttachment; +typedef sofa::pbrpc::shared_ptr RpcAttachmentPtr; + +class RpcAttachment +{ +public: + virtual ~RpcAttachment() {} + + virtual bool Serialize(ReadBufferPtr& attachment_buffer) = 0; + + virtual bool Deserialize(ReadBufferPtr& attachment_buffer) = 0; +}; + +} // namespace pbrpc +} // namespace sofa + +#endif // RPC_ATTACHMENT_H + +/* vim: set ts=4 sw=4 sts=4 tw=100 */ diff --git a/src/sofa/pbrpc/rpc_client_impl.cc b/src/sofa/pbrpc/rpc_client_impl.cc index e6ba58f..b5da1e8 100644 --- a/src/sofa/pbrpc/rpc_client_impl.cc +++ b/src/sofa/pbrpc/rpc_client_impl.cc @@ -318,6 +318,26 @@ void RpcClientImpl::CallMethod(const google::protobuf::Message* request, } header.data_size = write_buffer.ByteCount() - header_pos - header_size - header.meta_size; header.message_size = header.meta_size + header.data_size; + std::string request_attach_str = cntl->GetRequestAttachBuffer()->ToString(); + size_t attach_size = request_attach_str.size(); + if (attach_size > 0) + { + if (attach_size > ATTACH_BUFFER_SIZE) + { +#if defined( LOG ) + LOG(ERROR) << "CallMethod(): " << RpcEndpointToString(cntl->RemoteEndpoint()) + << ":attach data is too big"; +#else + SLOG(ERROR, "CallMethod(): %s: attach data is too big", + RpcEndpointToString(cntl->RemoteEndpoint()).c_str()); +#endif + cntl->Done(RPC_ERROR_SERIALIZE_REQUEST, "attach data is too big"); + return; + } + memcpy(header.attach_buffer, request_attach_str.c_str(), attach_size); + header.attach_size = attach_size; + } + write_buffer.SetData(header_pos, reinterpret_cast(&header), header_size); ReadBufferPtr read_buffer(new ReadBuffer()); diff --git a/src/sofa/pbrpc/rpc_client_stream.h b/src/sofa/pbrpc/rpc_client_stream.h index 7327983..fecd9cb 100644 --- a/src/sofa/pbrpc/rpc_client_stream.h +++ b/src/sofa/pbrpc/rpc_client_stream.h @@ -149,7 +149,8 @@ class RpcClientStream : public RpcMessageStream virtual void on_received( const ReadBufferPtr& message, int meta_size, - int64 data_size) + int64 data_size, + const std::string& attach_buffer) { SOFA_PBRPC_FUNCTION_TRACE; @@ -258,6 +259,11 @@ class RpcClientStream : public RpcMessageStream else // !meta.failed() { SCHECK_EQ(data_size, message->TotalCount() - message->ByteCount()); + WriteBuffer write_buffer; + write_buffer.Append(attach_buffer); + ReadBufferPtr read_buffer(new ReadBuffer()); + write_buffer.SwapOut(read_buffer.get()); + cntl->SetResponseAttachBuffer(read_buffer); cntl->SetResponseBuffer(message); cntl->SetResponseCompressType(meta.has_compress_type() ? meta.compress_type() : CompressTypeNone); diff --git a/src/sofa/pbrpc/rpc_controller.cc b/src/sofa/pbrpc/rpc_controller.cc index 57cad69..a3d5389 100644 --- a/src/sofa/pbrpc/rpc_controller.cc +++ b/src/sofa/pbrpc/rpc_controller.cc @@ -119,6 +119,26 @@ void RpcController::NotifyOnCancel(google::protobuf::Closure* callback) _impl->NotifyOnCancel(callback); } +void RpcController::SetRequestAttachment(RpcAttachment* request_attachment) +{ + _impl->SetRequestAttachment(request_attachment); +} + +void RpcController::GetRequestAttachment(RpcAttachment* request_attachment) +{ + _impl->GetRequestAttachment(request_attachment); +} + +void RpcController::SetResponseAttachment(RpcAttachment* response_attachment) +{ + _impl->SetResponseAttachment(response_attachment); +} + +void RpcController::GetResponseAttachment(RpcAttachment* response_attachment) +{ + _impl->GetResponseAttachment(response_attachment); +} + } // namespace pbrpc } // namespace sofa diff --git a/src/sofa/pbrpc/rpc_controller.h b/src/sofa/pbrpc/rpc_controller.h index 2ec3c14..1ceedd1 100644 --- a/src/sofa/pbrpc/rpc_controller.h +++ b/src/sofa/pbrpc/rpc_controller.h @@ -11,6 +11,7 @@ #include #include +#include namespace sofa { namespace pbrpc { @@ -164,6 +165,14 @@ class RpcController : public google::protobuf::RpcController // NotifyOnCancel() must be called no more than once per request. virtual void NotifyOnCancel(google::protobuf::Closure* callback); + void SetRequestAttachment(RpcAttachment* request_attachment); + + void GetRequestAttachment(RpcAttachment* attachment); + + void SetResponseAttachment(RpcAttachment* response_attachment); + + void GetResponseAttachment(RpcAttachment* attachment); + public: const sofa::pbrpc::shared_ptr& impl() const { diff --git a/src/sofa/pbrpc/rpc_controller_impl.h b/src/sofa/pbrpc/rpc_controller_impl.h index 4ee1679..e310d31 100644 --- a/src/sofa/pbrpc/rpc_controller_impl.h +++ b/src/sofa/pbrpc/rpc_controller_impl.h @@ -9,6 +9,7 @@ #include #include +#include #include @@ -19,6 +20,7 @@ #include #include #include +#include namespace sofa { namespace pbrpc { @@ -44,6 +46,8 @@ class RpcControllerImpl : public sofa::pbrpc::enable_shared_from_thisSerialize(_request_attach_buffer); + } + + void GetRequestAttachment(RpcAttachment* request_attachment) + { + request_attachment->Deserialize(_request_attach_buffer); + } + + void SetResponseAttachment(RpcAttachment* response_attachment) + { + response_attachment->Serialize(_response_attach_buffer); + } + + void GetResponseAttachment(RpcAttachment* response_attachment) + { + response_attachment->Deserialize(_response_attach_buffer);; + } + + void SetRequestAttachBuffer(const ReadBufferPtr& request_attach_buffer) + { + _request_attach_buffer = request_attach_buffer; + } + + ReadBufferPtr GetRequestAttachBuffer() + { + return _request_attach_buffer; + } + + void SetResponseAttachBuffer(const ReadBufferPtr& response_attach_buffer) + { + _response_attach_buffer = response_attach_buffer; + } + + ReadBufferPtr GetResponseAttachBuffer() + { + return _response_attach_buffer; + } + void NotifyRequestSent(const RpcEndpoint& local_endpoint, int64 sent_bytes) { _is_request_sent = true; @@ -447,6 +491,8 @@ class RpcControllerImpl : public sofa::pbrpc::enable_shared_from_this* _http_query_params; const std::map* _http_headers; + ReadBufferPtr _request_attach_buffer; + ReadBufferPtr _response_attach_buffer; + SOFA_PBRPC_DISALLOW_EVIL_CONSTRUCTORS(RpcControllerImpl); }; // class RpcControllerImpl diff --git a/src/sofa/pbrpc/rpc_message_header.h b/src/sofa/pbrpc/rpc_message_header.h index 7cedcc8..036d0b4 100644 --- a/src/sofa/pbrpc/rpc_message_header.h +++ b/src/sofa/pbrpc/rpc_message_header.h @@ -14,6 +14,7 @@ namespace pbrpc { // Magic string "SOFA" in little endian. #define SOFA_RPC_MAGIC 1095126867u +#define ATTACH_BUFFER_SIZE 65536u // total 24 bytes struct RpcMessageHeader { @@ -22,12 +23,18 @@ struct RpcMessageHeader { uint32 magic_str_value; }; // 4 bytes int32 meta_size; // 4 bytes + int64 attach_size; // 4 bytes int64 data_size; // 8 bytes int64 message_size; // 8 bytes: message_size = meta_size + data_size, for check + char attach_buffer[ATTACH_BUFFER_SIZE]; RpcMessageHeader() : magic_str_value(SOFA_RPC_MAGIC) - , meta_size(0), data_size(0), message_size(0) {} + , meta_size(0), attach_size(0) + , data_size(0), message_size(0) + { + memset(attach_buffer, '0', ATTACH_BUFFER_SIZE); + } bool CheckMagicString() const { diff --git a/src/sofa/pbrpc/rpc_message_stream.h b/src/sofa/pbrpc/rpc_message_stream.h index 4522991..a6bf3e9 100644 --- a/src/sofa/pbrpc/rpc_message_stream.h +++ b/src/sofa/pbrpc/rpc_message_stream.h @@ -197,10 +197,12 @@ class RpcMessageStream : public RpcByteStream // @param message the rough received message, including meta and data. // @param meta_size the size of meta. // @param data_size the size of data. + // @param attch_buffer the attachment buffer. virtual void on_received( const ReadBufferPtr& message, int meta_size, - int64 data_size) = 0; + int64 data_size, + const std::string& attach_buffer) = 0; private: virtual bool on_connected() @@ -276,7 +278,7 @@ class RpcMessageStream : public RpcByteStream while (!is_closed() && !received_messages.empty()) { const ReceivedItem& item = received_messages.front(); - on_received(item.message, item.meta_size, item.data_size); + on_received(item.message, item.meta_size, item.data_size, item.attach_buffer); received_messages.pop_front(); } } @@ -604,8 +606,9 @@ class RpcMessageStream : public RpcByteStream { _receiving_message->Append(BufHandle(_tran_buf, consume_size, data - _tran_buf)); } + std::string response_attach_str(_receiving_header.attach_buffer, _receiving_header.attach_size); received_messages->push_back(ReceivedItem(_receiving_message, - _receiving_header.meta_size, _receiving_header.data_size)); + _receiving_header.meta_size, _receiving_header.data_size, response_attach_str)); reset_receiving_env(); data += consume_size; size -= consume_size; @@ -656,13 +659,15 @@ class RpcMessageStream : public RpcByteStream << ": check size in header failed" << ": meta_size=" << _receiving_header.meta_size << ", data_size=" << _receiving_header.data_size + << ", attach_size=" << _receiving_header.attach_size << ", message_size=" << _receiving_header.message_size; #else SLOG(ERROR, "identify_message_header(): %s: " - "check size in header failed: meta_size=%d, data_size=%lld, message_size=%lld", + "check size in header failed: meta_size=%d, data_size=%lld, attach_size=%lld, message_size=%lld", RpcEndpointToString(_remote_endpoint).c_str(), _receiving_header.meta_size, _receiving_header.data_size, + _receiving_header.attach_size, _receiving_header.message_size); #endif return -1; @@ -736,12 +741,15 @@ class RpcMessageStream : public RpcByteStream ReadBufferPtr message; int meta_size; int64 data_size; + std::string attach_buffer; ReceivedItem(const ReadBufferPtr& _message, int _meta_size, - int64 _data_size) + int64 _data_size, + const std::string& _attach_buffer) : message(_message) , meta_size(_meta_size) - , data_size(_data_size) {} + , data_size(_data_size) + , attach_buffer(_attach_buffer) {} }; // TODO improve sync queue performance From 0b5c1869cfab4faf5cf73371641a9ed50171d541 Mon Sep 17 00:00:00 2001 From: zd-double Date: Mon, 28 Mar 2016 14:28:49 +0800 Subject: [PATCH 02/13] add rpc cookie --- Makefile | 2 +- sample/echo_cookie/Makefile | 8 ++--- .../echo_cookie/cookie_plugin/rpc_cookie.cc | 24 ++++--------- sample/echo_cookie/cookie_plugin/rpc_cookie.h | 34 ++++++------------- src/sofa/pbrpc/binary_rpc_request.cc | 13 +++++-- src/sofa/pbrpc/rpc_client_impl.cc | 6 ++-- .../sofa/pbrpc}/serialize.cc | 11 +++--- .../sofa/pbrpc}/serialize.h | 34 +++++++++---------- 8 files changed, 60 insertions(+), 72 deletions(-) rename {sample/echo_cookie/cookie_plugin => src/sofa/pbrpc}/serialize.cc (97%) rename {sample/echo_cookie/cookie_plugin => src/sofa/pbrpc}/serialize.h (92%) diff --git a/Makefile b/Makefile index c080f1d..bf650c3 100644 --- a/Makefile +++ b/Makefile @@ -44,7 +44,7 @@ PUB_INC=src/sofa/pbrpc/pbrpc.h src/sofa/pbrpc/closure_helper.h src/sofa/pbrpc/cl src/sofa/pbrpc/locks.h src/sofa/pbrpc/mutex_lock.h src/sofa/pbrpc/spin_lock.h \ src/sofa/pbrpc/fast_lock.h src/sofa/pbrpc/rw_lock.h src/sofa/pbrpc/scoped_locker.h \ src/sofa/pbrpc/condition_variable.h src/sofa/pbrpc/wait_event.h src/sofa/pbrpc/http.h \ - src/sofa/pbrpc/buffer.h src/sofa/pbrpc/buf_handle.h src/sofa/pbrpc/rpc_attachment.h \ + src/sofa/pbrpc/buffer.h src/sofa/pbrpc/buf_handle.h src/sofa/pbrpc/rpc_attachment.h src/sofa/pbrpc/serialize.h \ $(PROTO) $(PROTO_HEADER) #----------------------------------------------- diff --git a/sample/echo_cookie/Makefile b/sample/echo_cookie/Makefile index 3b2daf3..6a8aca5 100644 --- a/sample/echo_cookie/Makefile +++ b/sample/echo_cookie/Makefile @@ -60,17 +60,17 @@ check_depends: @if [ ! -f "$(SOFA_PBRPC)/lib/libsofa-pbrpc.a" ]; then echo "ERROR: need sofa-pbrpc lib"; exit 1; fi clean: - @rm -f $(BIN) *.o *.pb.* + @rm -f $(BIN) *.o *.pb.* cookie_plugin/*.o rebuild: clean all -server: $(PROTO_OBJ) server.o cookie_plugin/rpc_cookie.o cookie_plugin/serialize.o +server: $(PROTO_OBJ) server.o cookie_plugin/rpc_cookie.o $(CXX) $^ -o $@ $(LIBRARY) $(LDFLAGS) -client_sync: $(PROTO_OBJ) client_sync.o cookie_plugin/rpc_cookie.o cookie_plugin/serialize.o +client_sync: $(PROTO_OBJ) client_sync.o cookie_plugin/rpc_cookie.o $(CXX) $^ -o $@ $(LIBRARY) $(LDFLAGS) -client_async: $(PROTO_OBJ) client_async.o cookie_plugin/rpc_cookie.o cookie_plugin/serialize.o +client_async: $(PROTO_OBJ) client_async.o cookie_plugin/rpc_cookie.o $(CXX) $^ -o $@ $(LIBRARY) $(LDFLAGS) %.pb.o: %.pb.cc diff --git a/sample/echo_cookie/cookie_plugin/rpc_cookie.cc b/sample/echo_cookie/cookie_plugin/rpc_cookie.cc index 5234cbd..e6f3a79 100644 --- a/sample/echo_cookie/cookie_plugin/rpc_cookie.cc +++ b/sample/echo_cookie/cookie_plugin/rpc_cookie.cc @@ -1,20 +1,10 @@ -/*************************************************************************** - * - * Copyright (c) 2016 Baidu.com, Inc. All Rights Reserved - * $Id$ - * - **************************************************************************/ - - /** - * @file rpc_cookie.cc - * @author zhangdi05(zhangdi05@baidu.com) - * @date 2016/03/18 13:43:53 - * @version $Revision$ - * @brief - * - **/ - -#include "serialize.h" +// Copyright (c) 2014 Baidu.com, Inc. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. +// +// Author: zhangdi05@baidu.com (Zhang Di) + +#include #include "rpc_cookie.h" namespace sofa { diff --git a/sample/echo_cookie/cookie_plugin/rpc_cookie.h b/sample/echo_cookie/cookie_plugin/rpc_cookie.h index 72e890c..4608a10 100644 --- a/sample/echo_cookie/cookie_plugin/rpc_cookie.h +++ b/sample/echo_cookie/cookie_plugin/rpc_cookie.h @@ -1,22 +1,12 @@ -/*************************************************************************** - * - * Copyright (c) 2016 Baidu.com, Inc. All Rights Reserved - * $Id$ - * - **************************************************************************/ - - /** - * @file rpc_cookie.h - * @author zhangdi05(zhangdi05@baidu.com) - * @date 2016/03/18 13:32:12 - * @version $Revision$ - * @brief - * - **/ -#ifndef _SOFA_PBRPC_RPC_COOKIE_H_ -#define _SOFA_PBRPC_RPC_COOKIE_H_ - -#include +// Copyright (c) 2014 Baidu.com, Inc. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. +// +// Author: zhangdi05@baidu.com (Zhang Di) + +#ifndef _RPC_COOKIE_H_ +#define _RPC_COOKIE_H_ + #include #include #include @@ -34,9 +24,9 @@ enum KeepType MANAGER_FIRST = 2 }; -class RpcCookieManager; typedef std::map MapType; +//class RpcCookieManager class RpcCookieManager { public: @@ -58,10 +48,8 @@ class RpcCookieManager MutexLock _lock; }; -typedef std::map MapType; - -class Cookie : public sofa::pbrpc::RpcAttachment //class Cookie +class Cookie : public sofa::pbrpc::RpcAttachment { public: Cookie(); diff --git a/src/sofa/pbrpc/binary_rpc_request.cc b/src/sofa/pbrpc/binary_rpc_request.cc index be358be..1a67bfa 100644 --- a/src/sofa/pbrpc/binary_rpc_request.cc +++ b/src/sofa/pbrpc/binary_rpc_request.cc @@ -176,9 +176,16 @@ ReadBufferPtr BinaryRpcRequest::AssembleSucceedResponse( std::string response_attach_str = cntl->GetResponseAttachBuffer()->ToString(); size_t attach_size = response_attach_str.size(); - header.attach_size = attach_size; - memcpy(header.attach_buffer, response_attach_str.c_str(), attach_size); - + if (attach_size > 0) + { + if (attach_size > ATTACH_BUFFER_SIZE) + { + err = "serialize response message failed for attach data exceed the limit"; + return ReadBufferPtr(); + } + memcpy(header.attach_buffer, response_attach_str.c_str(), attach_size); + header.attach_size = attach_size; + } write_buffer.SetData(header_pos, reinterpret_cast(&header), header_size); diff --git a/src/sofa/pbrpc/rpc_client_impl.cc b/src/sofa/pbrpc/rpc_client_impl.cc index b5da1e8..7608c86 100644 --- a/src/sofa/pbrpc/rpc_client_impl.cc +++ b/src/sofa/pbrpc/rpc_client_impl.cc @@ -326,12 +326,12 @@ void RpcClientImpl::CallMethod(const google::protobuf::Message* request, { #if defined( LOG ) LOG(ERROR) << "CallMethod(): " << RpcEndpointToString(cntl->RemoteEndpoint()) - << ":attach data is too big"; + << ":attach data exceed the limit"; #else - SLOG(ERROR, "CallMethod(): %s: attach data is too big", + SLOG(ERROR, "CallMethod(): %s: attach data exceed the limit", RpcEndpointToString(cntl->RemoteEndpoint()).c_str()); #endif - cntl->Done(RPC_ERROR_SERIALIZE_REQUEST, "attach data is too big"); + cntl->Done(RPC_ERROR_SERIALIZE_REQUEST, "attach data exceed the limit"); return; } memcpy(header.attach_buffer, request_attach_str.c_str(), attach_size); diff --git a/sample/echo_cookie/cookie_plugin/serialize.cc b/src/sofa/pbrpc/serialize.cc similarity index 97% rename from sample/echo_cookie/cookie_plugin/serialize.cc rename to src/sofa/pbrpc/serialize.cc index a1a83ec..5daa167 100644 --- a/sample/echo_cookie/cookie_plugin/serialize.cc +++ b/src/sofa/pbrpc/serialize.cc @@ -1,6 +1,11 @@ -#include +// Copyright (c) 2014 Baidu.com, Inc. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. +// +// Author: qinzuoyan01@baidu.com (Qin Zuoyan) + #include -//#include +#include "serialize.h" namespace sofa { namespace pbrpc { @@ -76,7 +81,6 @@ bool Serializer::close() { _stream->BackUp(_buf_size); } - //_stream = NULL; _buf = NULL; _buf_size = 0; return ret; @@ -221,7 +225,6 @@ void Deserializer::close() { _stream->BackUp(_buf_size); } - //_stream = NULL; _buf = NULL; _buf_size = 0; } diff --git a/sample/echo_cookie/cookie_plugin/serialize.h b/src/sofa/pbrpc/serialize.h similarity index 92% rename from sample/echo_cookie/cookie_plugin/serialize.h rename to src/sofa/pbrpc/serialize.h index 8f14b7e..68d16ca 100644 --- a/sample/echo_cookie/cookie_plugin/serialize.h +++ b/src/sofa/pbrpc/serialize.h @@ -1,9 +1,14 @@ -#ifndef SOFA_IO_SERIALIZER_H -#define SOFA_IO_SERIALIZER_H +// Copyright (c) 2014 Baidu.com, Inc. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. +// +// Author: qinzuoyan01@baidu.com (Qin Zuoyan) + +#ifndef _SOFA_PBRPC_RPC_COOKIE_H_ +#define _SOFA_PBRPC_RPC_COOKIE_H_ #include #include -//#include #include // String max serialize size: 64MB. @@ -37,9 +42,7 @@ class Serializer _stream = stream; } ~Serializer() - { - //close(); - } + { } // close serializer, backup un-used buffer bool close(); @@ -194,8 +197,8 @@ class Serializer return 9; } private: - void* _buf; // current buffer to write - int _buf_size; // size of current buffer to write + void* _buf; // current buffer to write + int _buf_size; // size of current buffer to write WriteBufferPtr _stream; // output stream }; @@ -209,9 +212,7 @@ class Deserializer _buf_size = 0; } ~Deserializer() - { - //close(); - } + { } // close deserializer, backup un-used buffer void close(); @@ -332,15 +333,14 @@ class Deserializer } bool deserialize_varint(uint64_t& d); private: - const void* _buf; // current buffer to read - int _buf_size; // size of current buffer to read - //InputStream* _stream; // input stream - ReadBufferPtr _stream; + const void* _buf; // current buffer to read + int _buf_size; // size of current buffer to read + ReadBufferPtr _stream; // input stream }; -} // namespace io +} // namespace pbrpc } // namespace sofa -#endif // SOFA_IO_SERIALIZER_H +#endif // _SOFA_PBRPC_RPC_COOKIE_H_ /* vim: set ts=4 sw=4 sts=4 tw=100 */ From 1afbaadbd66b154ba3149d9bf514d75596a3748e Mon Sep 17 00:00:00 2001 From: zd-double Date: Mon, 28 Mar 2016 14:32:48 +0800 Subject: [PATCH 03/13] add rpc cookie --- sample/echo_cookie/server.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/sample/echo_cookie/server.cc b/sample/echo_cookie/server.cc index 07b150c..2feb775 100644 --- a/sample/echo_cookie/server.cc +++ b/sample/echo_cookie/server.cc @@ -11,7 +11,6 @@ #include "echo_service.pb.h" typedef sofa::pbrpc::shared_ptr CookiePtr; -//sofa::pbrpc::RpcCookieManager cookie_manager; bool WebServlet(const sofa::pbrpc::HTTPRequest& request, sofa::pbrpc::HTTPResponse& response) { From eacf6ef69318e664b24177caf4518475f3468882 Mon Sep 17 00:00:00 2001 From: zd-double Date: Mon, 28 Mar 2016 14:37:14 +0800 Subject: [PATCH 04/13] remove client_http.sh --- sample/echo_cookie/client_http.sh | 10 ---------- 1 file changed, 10 deletions(-) delete mode 100755 sample/echo_cookie/client_http.sh diff --git a/sample/echo_cookie/client_http.sh b/sample/echo_cookie/client_http.sh deleted file mode 100755 index ab72311..0000000 --- a/sample/echo_cookie/client_http.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/sh - -# POST -curl -d '{"message":"Hello, world!"}' http://localhost:12321/sofa.pbrpc.test.EchoServer.Echo -echo - -# GET -curl http://localhost:12321/sofa.pbrpc.test.EchoServer.Echo?request=%7B%22message%22%3A%22Hello%2C%20world%21%22%7D -echo - From 17bb28fc252088cb577f1bf2fdb9547a90eb1718 Mon Sep 17 00:00:00 2001 From: zd-double Date: Mon, 28 Mar 2016 14:43:07 +0800 Subject: [PATCH 05/13] delete test code --- src/sofa/pbrpc/rpc_controller_impl.h | 1 - 1 file changed, 1 deletion(-) diff --git a/src/sofa/pbrpc/rpc_controller_impl.h b/src/sofa/pbrpc/rpc_controller_impl.h index e310d31..93569f8 100644 --- a/src/sofa/pbrpc/rpc_controller_impl.h +++ b/src/sofa/pbrpc/rpc_controller_impl.h @@ -9,7 +9,6 @@ #include #include -#include #include From 8acb2db78730c3cca1ae50a3024fddaea6e80914 Mon Sep 17 00:00:00 2001 From: zd-double Date: Mon, 28 Mar 2016 14:45:26 +0800 Subject: [PATCH 06/13] remove test code --- src/sofa/pbrpc/rpc_controller_impl.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/sofa/pbrpc/rpc_controller_impl.h b/src/sofa/pbrpc/rpc_controller_impl.h index 93569f8..7ab17a5 100644 --- a/src/sofa/pbrpc/rpc_controller_impl.h +++ b/src/sofa/pbrpc/rpc_controller_impl.h @@ -490,8 +490,6 @@ class RpcControllerImpl : public sofa::pbrpc::enable_shared_from_this Date: Mon, 28 Mar 2016 14:57:13 +0800 Subject: [PATCH 07/13] update serializer --- src/sofa/pbrpc/serialize.cc | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/sofa/pbrpc/serialize.cc b/src/sofa/pbrpc/serialize.cc index 5daa167..28a5c56 100644 --- a/src/sofa/pbrpc/serialize.cc +++ b/src/sofa/pbrpc/serialize.cc @@ -11,7 +11,6 @@ namespace sofa { namespace pbrpc { // NOTICE: When compiling by gcc with '-O2' option, the data may be out of order. -// It seems a bug of gcc, fuck it. So we use 'sofa_io_barrier()' to prevent it. #define sofa_io_barrier() __asm__ __volatile__("": : :"memory") // fast copy small data block @@ -121,7 +120,6 @@ bool Serializer::serialize_string(const std::string& d) // NOTICE: When compiling by gcc with '-O2' option, the data may be out of order. // It occurs between modify 'd' and 'serialize_X_byte', so we use 'sofa_io_barrier()' to prevent it. -// Fuck gcc again! bool Serializer::serialize_varint(uint64_t d) { if (d < (1UL << 7)) From 0b75c92ee8093b58d180d85205ee0cf2ef09d4ae Mon Sep 17 00:00:00 2001 From: zd-double Date: Sat, 2 Apr 2016 10:41:49 +0800 Subject: [PATCH 08/13] change implementation of rpc cookie --- src/sofa/pbrpc/plugin/cookie/rpc_cookie.cc | 197 +++++++++++++++++++++ src/sofa/pbrpc/plugin/cookie/rpc_cookie.h | 89 ++++++++++ 2 files changed, 286 insertions(+) create mode 100644 src/sofa/pbrpc/plugin/cookie/rpc_cookie.cc create mode 100644 src/sofa/pbrpc/plugin/cookie/rpc_cookie.h diff --git a/src/sofa/pbrpc/plugin/cookie/rpc_cookie.cc b/src/sofa/pbrpc/plugin/cookie/rpc_cookie.cc new file mode 100644 index 0000000..5723088 --- /dev/null +++ b/src/sofa/pbrpc/plugin/cookie/rpc_cookie.cc @@ -0,0 +1,197 @@ +// Copyright (c) 2014 Baidu.com, Inc. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. +// +// Author: zhangdi05@baidu.com (Zhang Di) + +#include +#include +#include + +namespace sofa { +namespace pbrpc { + +bool RpcCookieManager::Has(const std::string& key) +{ + ScopedLocker _(_lock); + return _kv_map.find(key) != _kv_map.end(); +} + +void RpcCookieManager::Load(MapType& kv_map) +{ + ScopedLocker _(_lock); + kv_map = _kv_map; +} + +void RpcCookieManager::Store(const MapType& kv_map) +{ + ScopedLocker _(_lock); + _kv_map.insert(kv_map.begin(), kv_map.end()); + + std::vector remove_list; + MapType::const_iterator old_it = _kv_map.begin(); + MapType::const_iterator old_end = _kv_map.end(); + MapType::const_iterator new_it = kv_map.begin(); + MapType::const_iterator new_end = kv_map.end(); + while (old_it != old_end && new_it != new_end) + { + if(old_it->first < new_it->first) + { + remove_list.push_back(old_it->first); + ++old_it; + } + else + { + ++new_it; + ++old_it; + } + } + while (old_it != old_end) + { + remove_list.push_back(old_it->first); + ++old_it; + } + if (!remove_list.empty()) + { + for (std::vector::const_iterator it = remove_list.begin(); + it != remove_list.end(); ++it) + { + _kv_map.erase(*it); + } + } +} + +Cookie::Cookie() : _manager(NULL) +{ } + +Cookie::Cookie(RpcCookieManager* manager) : _manager(manager) +{ } + +bool Cookie::Get(const std::string& key, std::string& value) +{ + MapType::const_iterator it = _kv_map.find(key); + if (it == _kv_map.end()) + { + return false; + } + value = it->second; + return true; +} + +void Cookie::Set(const std::string& key, const std::string& value) +{ + _kv_map[key] = value; +} + +void Cookie::Load() +{ + if (_manager != NULL) + { + _manager->Load(_kv_map); + } +} + +void Cookie::Store() +{ + if (_manager != NULL) + { + _manager->Store(_kv_map); + } +} + +bool Cookie::Has(const std::string& key) +{ + return _kv_map.find(key) != _kv_map.end(); +} + +bool Cookie::Empty() +{ + return _kv_map.empty(); +} + +bool Cookie::Erase(const std::string& key) +{ + if (_kv_map.find(key) == _kv_map.end()) + { + return false; + } + _kv_map.erase(key); + return true; +} + +void Cookie::Clear() +{ + _kv_map.clear(); +} + +bool Cookie::Serialize(sofa::pbrpc::ReadBufferPtr& attach_buffer) +{ + int size = _kv_map.size(); + SCHECK_LE(static_cast(size), SOFA_CONTAINER_MAX_SERIALIZE_SIZE); + WriteBufferPtr write_buffer(new WriteBuffer()); + Serializer serializer(write_buffer); + if (!serializer.serialize_varint(size)) + { + return false; + } + MapType::const_iterator it = _kv_map.begin(); + for (; it != _kv_map.end(); ++it) + { + if (!serializer.serialize_string(it->first)) + { + return false; + } + if (!serializer.serialize_string(it->second)) + { + return false; + } + } + write_buffer->SwapOut(attach_buffer.get()); + return true; +} + +int Cookie::SerializeLen() const +{ + int len = Serializer::varint_len(_kv_map.size()); + MapType::const_iterator it = _kv_map.begin(); + for (; it != _kv_map.end(); ++it) + { + len += Serializer::string_len(it->first); + len += Serializer::string_len(it->second); + } + return len; +} + +bool Cookie::Deserialize(sofa::pbrpc::ReadBufferPtr& attach_buffer) +{ + int size = 0; + Deserializer deserializer(attach_buffer); + if (!deserializer.deserialize_varint(size)) + { + return false; + } + if (static_cast(size) > SOFA_CONTAINER_MAX_SERIALIZE_SIZE) + { + return false; + } + _kv_map.clear(); + for (int i = 0; i < size; ++i) + { + std::string key; + if (!deserializer.deserialize_string(key)) + { + return false; + } + std::string value; + if (!deserializer.deserialize_string(value)) + { + return false; + } + _kv_map[key] = value; + } + return true; +} +} // namespace pbrpc +} // namespace sofa + +/* vim: set ts=4 sw=4 sts=4 tw=100 */ diff --git a/src/sofa/pbrpc/plugin/cookie/rpc_cookie.h b/src/sofa/pbrpc/plugin/cookie/rpc_cookie.h new file mode 100644 index 0000000..43fa1a9 --- /dev/null +++ b/src/sofa/pbrpc/plugin/cookie/rpc_cookie.h @@ -0,0 +1,89 @@ +// Copyright (c) 2014 Baidu.com, Inc. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. +// +// Author: zhangdi05@baidu.com (Zhang Di) + +#ifndef _SOFA_PBRPC_PLUGIN_COOKIE_RPC_COOKIE_H_ +#define _SOFA_PBRPC_PLUGIN_COOKIE_RPC_COOKIE_H_ + +#include +#include +#include +#include +#include +#include + +namespace sofa { +namespace pbrpc { + +typedef std::map MapType; + +//class RpcCookieManager +class RpcCookieManager +{ +public: + RpcCookieManager() + { } + + ~RpcCookieManager() + { } + + bool Has(const std::string& key); + + void Load(MapType& kv_map); + + void Store(const MapType& kv_map); + +private: + MapType _kv_map; + + MutexLock _lock; +}; + +//class Cookie +class Cookie : public sofa::pbrpc::RpcAttachment +{ +public: + Cookie(); + + Cookie(RpcCookieManager* manager); + + virtual ~Cookie() { } + + bool Get(const std::string& key, std::string& value); + + void Set(const std::string& key, const std::string& value); + + bool Has(const std::string& key); + + bool Empty(); + + bool Erase(const std::string& key); + + void Clear(); + + void Load(); + + void Store(); + + virtual bool Serialize(sofa::pbrpc::ReadBufferPtr& attach_buffer); + + virtual bool Deserialize(sofa::pbrpc::ReadBufferPtr& attach_buffer); + + int SerializeLen() const; + +private: + MapType _kv_map; + + RpcCookieManager* _manager; + + SOFA_PBRPC_DISALLOW_EVIL_CONSTRUCTORS(Cookie); +}; //class RpcCookie + +} // namespace pbrpc +} // namespace sofa + +#endif // _SOFA_PBRPC_PLUGIN_COOKIE_RPC_COOKIE_H_ + +/* vim: set ts=4 sw=4 sts=4 tw=100 */ From a4e17b0f04ad37e7d42f37bf47674f0809b29b82 Mon Sep 17 00:00:00 2001 From: zd-double Date: Sat, 2 Apr 2016 10:52:09 +0800 Subject: [PATCH 09/13] change implementation of rpc cookie --- Makefile | 17 +- sample/echo/Makefile | 6 +- sample/echo/client_async.cc | 16 ++ sample/echo/client_sync.cc | 15 ++ sample/echo/server.cc | 12 ++ sample/echo_cookie/Makefile | 84 --------- sample/echo_cookie/client_async.cc | 87 --------- sample/echo_cookie/client_sync.cc | 79 -------- .../echo_cookie/cookie_plugin/rpc_cookie.cc | 170 ------------------ sample/echo_cookie/cookie_plugin/rpc_cookie.h | 94 ---------- sample/echo_cookie/echo_service.proto | 20 --- sample/echo_cookie/server.cc | 125 ------------- src/sofa/pbrpc/binary_rpc_request.cc | 64 ++++--- src/sofa/pbrpc/buffer.cc | 10 ++ src/sofa/pbrpc/buffer.h | 2 + src/sofa/pbrpc/rpc_client_impl.cc | 63 ++++--- src/sofa/pbrpc/rpc_client_stream.h | 11 +- src/sofa/pbrpc/rpc_controller_impl.h | 23 +++ src/sofa/pbrpc/rpc_message_header.h | 9 +- src/sofa/pbrpc/rpc_message_stream.h | 20 +-- src/sofa/pbrpc/serialize.h | 6 +- 21 files changed, 195 insertions(+), 738 deletions(-) delete mode 100644 sample/echo_cookie/Makefile delete mode 100644 sample/echo_cookie/client_async.cc delete mode 100644 sample/echo_cookie/client_sync.cc delete mode 100644 sample/echo_cookie/cookie_plugin/rpc_cookie.cc delete mode 100644 sample/echo_cookie/cookie_plugin/rpc_cookie.h delete mode 100644 sample/echo_cookie/echo_service.proto delete mode 100644 sample/echo_cookie/server.cc diff --git a/Makefile b/Makefile index bf650c3..2c98ff6 100644 --- a/Makefile +++ b/Makefile @@ -24,8 +24,11 @@ OPT ?= -O2 # (A) Production use (optimized mode) include depends.mk LIB=libsofa-pbrpc.a +LIB_FULL=libsofa-pbrpc-full.a LIB_SRC=$(wildcard src/sofa/pbrpc/*.cc) +PLUGIN_SRC=$(wildcard src/sofa/pbrpc/plugin/*/*.cc) LIB_OBJ=$(patsubst %.cc,%.o,$(LIB_SRC)) +PLUGIN_OBJ=$(patsubst %.cc,%.o,$(PLUGIN_SRC)) PROTO=$(wildcard src/sofa/pbrpc/*.proto) PROTO_SRC=$(patsubst %.proto,%.pb.cc,$(PROTO)) PROTO_HEADER=$(patsubst %.proto,%.pb.h,$(PROTO)) @@ -44,7 +47,7 @@ PUB_INC=src/sofa/pbrpc/pbrpc.h src/sofa/pbrpc/closure_helper.h src/sofa/pbrpc/cl src/sofa/pbrpc/locks.h src/sofa/pbrpc/mutex_lock.h src/sofa/pbrpc/spin_lock.h \ src/sofa/pbrpc/fast_lock.h src/sofa/pbrpc/rw_lock.h src/sofa/pbrpc/scoped_locker.h \ src/sofa/pbrpc/condition_variable.h src/sofa/pbrpc/wait_event.h src/sofa/pbrpc/http.h \ - src/sofa/pbrpc/buffer.h src/sofa/pbrpc/buf_handle.h src/sofa/pbrpc/rpc_attachment.h src/sofa/pbrpc/serialize.h \ + src/sofa/pbrpc/buffer.h src/sofa/pbrpc/buf_handle.h src/sofa/pbrpc/rpc_attachment.h \ $(PROTO) $(PROTO_HEADER) #----------------------------------------------- @@ -84,7 +87,7 @@ check_depends: @if [ ! -f "$(SNAPPY_DIR)/lib/libsnappy.a" ]; then echo "ERROR: need snappy lib"; exit 1; fi clean: - rm -f $(LIB) $(BIN) $(LIB_OBJ) $(PROTO_OBJ) $(BIN_OBJ) $(PROTO_HEADER) $(PROTO_SRC) + rm -f $(LIB) $(LIB_FULL) $(BIN) $(LIB_OBJ) $(PLUGIN_OBJ) $(PROTO_OBJ) $(BIN_OBJ) $(PROTO_HEADER) $(PROTO_SRC) rebuild: clean all @@ -95,6 +98,9 @@ $(LIB_OBJ): $(PROTO_HEADER) $(LIB): $(LIB_OBJ) $(PROTO_OBJ) ar crs $@ $(LIB_OBJ) $(PROTO_OBJ) +$(LIB_FULL): $(PLUGIN_OBJ) $(LIB_OBJ) $(PROTO_OBJ) + ar crs $@ $(PLUGIN_OBJ) $(LIB_OBJ) $(PROTO_OBJ) + $(BIN): $(LIB) $(BIN_OBJ) $(CXX) $(BIN_OBJ) -o $@ $(LIB) $(LDFLAGS) @@ -104,19 +110,22 @@ $(BIN): $(LIB) $(BIN_OBJ) %.o: %.cc $(CXX) $(CXXFLAGS) -c $< -o $@ -build: $(LIB) $(BIN) +build: $(LIB) $(LIB_FULL) $(BIN) @echo @echo 'Build succeed, run "make install" to install sofa-pbrpc to "'$(PREFIX)'".' -install: $(LIB) $(BIN) +install: $(LIB) $(LIB_FULL) $(BIN) mkdir -p $(PREFIX)/include/sofa/pbrpc cp -r $(PUB_INC) $(TARGET_DIRECTORY) $(PREFIX)/include/sofa/pbrpc/ mkdir -p $(PREFIX)/include/sofa/pbrpc/smart_ptr cp src/sofa/pbrpc/smart_ptr/*.hpp $(PREFIX)/include/sofa/pbrpc/smart_ptr mkdir -p $(PREFIX)/include/sofa/pbrpc/smart_ptr/detail cp src/sofa/pbrpc/smart_ptr/detail/*.hpp $(PREFIX)/include/sofa/pbrpc/smart_ptr/detail + mkdir -p $(PREFIX)/include/sofa/pbrpc/plugin/cookie + cp src/sofa/pbrpc/plugin/cookie/*.h $(PREFIX)/include/sofa/pbrpc/plugin/cookie mkdir -p $(PREFIX)/lib cp $(LIB) $(PREFIX)/lib/ + cp $(LIB_FULL) $(PREFIX)/lib/ mkdir -p $(PREFIX)/bin cp $(BIN) $(PREFIX)/bin/ @echo diff --git a/sample/echo/Makefile b/sample/echo/Makefile index 9b12585..62e2d68 100644 --- a/sample/echo/Makefile +++ b/sample/echo/Makefile @@ -32,7 +32,7 @@ INCPATH=-I. -I$(SOFA_PBRPC)/include -I$(PROTOBUF_DIR)/include \ -I$(SNAPPY_DIR)/include -I$(ZLIB_DIR)/include CXXFLAGS += $(OPT) -pipe -W -Wall -fPIC -D_GNU_SOURCE -D__STDC_LIMIT_MACROS $(INCPATH) -LIBRARY=$(SOFA_PBRPC)/lib/libsofa-pbrpc.a $(PROTOBUF_DIR)/lib/libprotobuf.a $(SNAPPY_DIR)/lib/libsnappy.a +LIBRARY=$(SOFA_PBRPC)/lib/libsofa-pbrpc-full.a $(PROTOBUF_DIR)/lib/libprotobuf.a $(SNAPPY_DIR)/lib/libsnappy.a LDFLAGS += -L$(ZLIB_DIR)/lib -lpthread -lz UNAME_S := $(shell uname -s) @@ -57,10 +57,10 @@ check_depends: @if [ ! -f "$(SNAPPY_DIR)/include/snappy.h" ]; then echo "ERROR: need snappy header"; exit 1; fi @if [ ! -f "$(SNAPPY_DIR)/lib/libsnappy.a" ]; then echo "ERROR: need snappy lib"; exit 1; fi @if [ ! -f "$(SOFA_PBRPC)/include/sofa/pbrpc/pbrpc.h" ]; then echo "ERROR: need sofa-pbrpc header"; exit 1; fi - @if [ ! -f "$(SOFA_PBRPC)/lib/libsofa-pbrpc.a" ]; then echo "ERROR: need sofa-pbrpc lib"; exit 1; fi + @if [ ! -f "$(SOFA_PBRPC)/lib/libsofa-pbrpc-full.a" ]; then echo "ERROR: need sofa-pbrpc-full lib"; exit 1; fi clean: - @rm -f $(BIN) *.o *.pb.* + @rm -f $(BIN) *.o *.pb.* rebuild: clean all diff --git a/sample/echo/client_async.cc b/sample/echo/client_async.cc index 3055c09..a29a63a 100644 --- a/sample/echo/client_async.cc +++ b/sample/echo/client_async.cc @@ -6,8 +6,12 @@ #include #include +#include #include "echo_service.pb.h" +typedef sofa::pbrpc::shared_ptr CookiePtr; +sofa::pbrpc::RpcCookieManager cookie_manager; + void EchoCallback(sofa::pbrpc::RpcController* cntl, sofa::pbrpc::test::EchoRequest* request, sofa::pbrpc::test::EchoResponse* response, @@ -26,6 +30,12 @@ void EchoCallback(sofa::pbrpc::RpcController* cntl, } else { SLOG(NOTICE, "request succeed: %s", response->message().c_str()); + CookiePtr cookie(new sofa::pbrpc::Cookie(&cookie_manager)); + cntl->GetResponseAttachment(cookie.get()); + std::string version; + cookie->Get("version", version); + SLOG(NOTICE, "cookie version : %s", version.c_str()); + cookie->Store(); } delete cntl; @@ -46,9 +56,15 @@ int main() sofa::pbrpc::RpcChannelOptions channel_options; sofa::pbrpc::RpcChannel rpc_channel(&rpc_client, "127.0.0.1:12321", channel_options); + CookiePtr cookie(new sofa::pbrpc::Cookie(&cookie_manager)); + cookie->Load(); + cookie->Set("type", "async"); + cookie->Set("logid", "123456"); + // Prepare parameters. sofa::pbrpc::RpcController* cntl = new sofa::pbrpc::RpcController(); cntl->SetTimeout(3000); + cntl->SetRequestAttachment(cookie.get()); sofa::pbrpc::test::EchoRequest* request = new sofa::pbrpc::test::EchoRequest(); request->set_message("Hello from qinzuoyan01"); sofa::pbrpc::test::EchoResponse* response = new sofa::pbrpc::test::EchoResponse(); diff --git a/sample/echo/client_sync.cc b/sample/echo/client_sync.cc index 7b26ff5..c59641a 100644 --- a/sample/echo/client_sync.cc +++ b/sample/echo/client_sync.cc @@ -5,8 +5,12 @@ // Author: qinzuoyan01@baidu.com (Qin Zuoyan) #include +#include #include "echo_service.pb.h" +typedef sofa::pbrpc::shared_ptr CookiePtr; +sofa::pbrpc::RpcCookieManager cookie_manager; + // Using global RpcClient object can help share resources such as threads and buffers. sofa::pbrpc::RpcClient g_rpc_client; @@ -21,6 +25,11 @@ int main() // Prepare parameters. sofa::pbrpc::RpcController* cntl = new sofa::pbrpc::RpcController(); cntl->SetTimeout(3000); + CookiePtr cookie(new sofa::pbrpc::Cookie(&cookie_manager)); + cookie->Load(); + cookie->Set("type", "sync"); + cookie->Set("logid", "123456"); + cntl->SetRequestAttachment(cookie.get()); sofa::pbrpc::test::EchoRequest* request = new sofa::pbrpc::test::EchoRequest(); request->set_message("Hello from qinzuoyan01"); @@ -50,6 +59,12 @@ int main() else { SLOG(NOTICE, "request succeed: %s", response->message().c_str()); + cookie.reset(new sofa::pbrpc::Cookie(&cookie_manager)); + cntl->GetResponseAttachment(cookie.get()); + std::string version; + cookie->Get("version", version); + SLOG(NOTICE, "cookie version : %s", version.c_str()); + cookie->Store(); } // Destroy objects. diff --git a/sample/echo/server.cc b/sample/echo/server.cc index c38015e..59177b4 100644 --- a/sample/echo/server.cc +++ b/sample/echo/server.cc @@ -7,8 +7,11 @@ #include #include #include +#include #include "echo_service.pb.h" +typedef sofa::pbrpc::shared_ptr CookiePtr; + bool WebServlet(const sofa::pbrpc::HTTPRequest& request, sofa::pbrpc::HTTPResponse& response) { SLOG(INFO, "WebServlet(): request message from %s:%u", @@ -53,7 +56,16 @@ class EchoServerImpl : public sofa::pbrpc::test::EchoServer SLOG(INFO, "Header[\"%s\"]=\"%s\"", it->first.c_str(), it->second.c_str()); } } + CookiePtr cookie(new sofa::pbrpc::Cookie()); + cntl->GetRequestAttachment(cookie.get()); + std::string type; + std::string logid; + cookie->Get("type", type); + cookie->Get("logid", logid); + SLOG(INFO, "cookie info : type=%s, logid=%s", type.c_str(), logid.c_str()); response->set_message("echo message: " + request->message()); + cookie->Set("version", "1.00"); + cntl->SetResponseAttachment(cookie.get()); done->Run(); } }; diff --git a/sample/echo_cookie/Makefile b/sample/echo_cookie/Makefile deleted file mode 100644 index 6a8aca5..0000000 --- a/sample/echo_cookie/Makefile +++ /dev/null @@ -1,84 +0,0 @@ -# Copyright (c) 2014 Baidu.com, Inc. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. See the AUTHORS file for names of contributors. - -#----------------------------------------------- -## Sofa-pbrpc path containing `include'and `lib'. -## -## Check file exist: -## $(SOFA_PBRPC)/include/sofa/pbrpc/pbrpc.h -## $(SOFA_PBRPC)/lib/libsofa-pbrpc.a -## -SOFA_PBRPC=../../output -#----------------------------------------------- - -#----------------------------------------------- -# Uncomment exactly one of the lines labelled (A), (B), and (C) below -# to switch between compilation modes. -# -OPT ?= -O2 # (A) Production use (optimized mode) -# OPT ?= -g2 # (B) Debug mode, w/ full line-level debugging symbols -# OPT ?= -O2 -g2 # (C) Profiling mode: opt, but w/debugging symbols -#----------------------------------------------- - -#----------------------------------------------- -# !!! Do not change the following lines !!! -#----------------------------------------------- - -include ../../depends.mk - -CXX=g++ -INCPATH=-I. -I$(SOFA_PBRPC)/include -I$(PROTOBUF_DIR)/include \ - -I$(SNAPPY_DIR)/include -I$(ZLIB_DIR)/include -CXXFLAGS += $(OPT) -pipe -W -Wall -fPIC -D_GNU_SOURCE -D__STDC_LIMIT_MACROS $(INCPATH) - -LIBRARY=$(SOFA_PBRPC)/lib/libsofa-pbrpc.a $(PROTOBUF_DIR)/lib/libprotobuf.a $(SNAPPY_DIR)/lib/libsnappy.a -LDFLAGS += -L$(ZLIB_DIR)/lib -lpthread -lz - -UNAME_S := $(shell uname -s) -ifeq ($(UNAME_S),Linux) - LDFLAGS += -lrt -endif - -PROTO_SRC=echo_service.proto -PROTO_OBJ=$(patsubst %.proto,%.pb.o,$(PROTO_SRC)) -PROTO_OPTIONS=--proto_path=. --proto_path=$(SOFA_PBRPC)/include --proto_path=$(PROTOBUF_DIR)/include - -BIN=server client_sync client_async - -all: check_depends $(BIN) - -.PHONY: check_depends clean - -check_depends: - @if [ ! -f "$(PROTOBUF_DIR)/include/google/protobuf/message.h" ]; then echo "ERROR: need protobuf header"; exit 1; fi - @if [ ! -f "$(PROTOBUF_DIR)/lib/libprotobuf.a" ]; then echo "ERROR: need protobuf lib"; exit 1; fi - @if [ ! -f "$(PROTOBUF_DIR)/bin/protoc" ]; then echo "ERROR: need protoc binary"; exit 1; fi - @if [ ! -f "$(SNAPPY_DIR)/include/snappy.h" ]; then echo "ERROR: need snappy header"; exit 1; fi - @if [ ! -f "$(SNAPPY_DIR)/lib/libsnappy.a" ]; then echo "ERROR: need snappy lib"; exit 1; fi - @if [ ! -f "$(SOFA_PBRPC)/include/sofa/pbrpc/pbrpc.h" ]; then echo "ERROR: need sofa-pbrpc header"; exit 1; fi - @if [ ! -f "$(SOFA_PBRPC)/lib/libsofa-pbrpc.a" ]; then echo "ERROR: need sofa-pbrpc lib"; exit 1; fi - -clean: - @rm -f $(BIN) *.o *.pb.* cookie_plugin/*.o - -rebuild: clean all - -server: $(PROTO_OBJ) server.o cookie_plugin/rpc_cookie.o - $(CXX) $^ -o $@ $(LIBRARY) $(LDFLAGS) - -client_sync: $(PROTO_OBJ) client_sync.o cookie_plugin/rpc_cookie.o - $(CXX) $^ -o $@ $(LIBRARY) $(LDFLAGS) - -client_async: $(PROTO_OBJ) client_async.o cookie_plugin/rpc_cookie.o - $(CXX) $^ -o $@ $(LIBRARY) $(LDFLAGS) - -%.pb.o: %.pb.cc - $(CXX) $(CXXFLAGS) -c $< -o $@ - -%.pb.cc: %.proto - $(PROTOBUF_DIR)/bin/protoc $(PROTO_OPTIONS) --cpp_out=. $< - -%.o: %.cc $(PROTO_OBJ) - $(CXX) $(CXXFLAGS) -c $< -o $@ - diff --git a/sample/echo_cookie/client_async.cc b/sample/echo_cookie/client_async.cc deleted file mode 100644 index 6bfdf66..0000000 --- a/sample/echo_cookie/client_async.cc +++ /dev/null @@ -1,87 +0,0 @@ -// Copyright (c) 2014 Baidu.com, Inc. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. -// -// Author: qinzuoyan01@baidu.com (Qin Zuoyan) - -#include -#include -#include "cookie_plugin/rpc_cookie.h" -#include "echo_service.pb.h" - - -typedef sofa::pbrpc::shared_ptr CookiePtr; -sofa::pbrpc::RpcCookieManager cookie_manager; - -void EchoCallback(sofa::pbrpc::RpcController* cntl, - sofa::pbrpc::test::EchoRequest* request, - sofa::pbrpc::test::EchoResponse* response, - bool* callbacked) -{ - SLOG(NOTICE, "RemoteAddress=%s", cntl->RemoteAddress().c_str()); - SLOG(NOTICE, "IsRequestSent=%s", cntl->IsRequestSent() ? "true" : "false"); - if (cntl->IsRequestSent()) - { - SLOG(NOTICE, "LocalAddress=%s", cntl->LocalAddress().c_str()); - SLOG(NOTICE, "SentBytes=%ld", cntl->SentBytes()); - } - - if (cntl->Failed()) { - SLOG(ERROR, "request failed: %s", cntl->ErrorText().c_str()); - } - else { - SLOG(NOTICE, "request succeed: %s", response->message().c_str()); - CookiePtr cookie(new sofa::pbrpc::Cookie(&cookie_manager)); - cntl->GetResponseAttachment(cookie.get()); - std::string version; - cookie->Get("version", version); - SLOG(NOTICE, "cookie version : %s", version.c_str()); - cookie->Store(); - } - - delete cntl; - delete request; - delete response; - *callbacked = true; -} - -int main() -{ - SOFA_PBRPC_SET_LOG_LEVEL(NOTICE); - - // Define an rpc server. - sofa::pbrpc::RpcClientOptions client_options; - sofa::pbrpc::RpcClient rpc_client(client_options); - - // Define an rpc channel. - sofa::pbrpc::RpcChannelOptions channel_options; - sofa::pbrpc::RpcChannel rpc_channel(&rpc_client, "127.0.0.1:12300", channel_options); - - CookiePtr cookie(new sofa::pbrpc::Cookie(&cookie_manager)); - cookie->Set("type", "async"); - cookie->Set("logid", "123456"); - - // Prepare parameters. - sofa::pbrpc::RpcController* cntl = new sofa::pbrpc::RpcController(); - cntl->SetTimeout(3000); - cntl->SetRequestAttachment(cookie.get()); - sofa::pbrpc::test::EchoRequest* request = new sofa::pbrpc::test::EchoRequest(); - request->set_message("Hello from qinzuoyan01"); - sofa::pbrpc::test::EchoResponse* response = new sofa::pbrpc::test::EchoResponse(); - bool callbacked = false; - google::protobuf::Closure* done = sofa::pbrpc::NewClosure( - &EchoCallback, cntl, request, response, &callbacked); - - // Async call. - sofa::pbrpc::test::EchoServer_Stub stub(&rpc_channel); - stub.Echo(cntl, request, response, done); - - // Wait call done. - while (!callbacked) { - usleep(100000); - } - - return EXIT_SUCCESS; -} - -/* vim: set ts=4 sw=4 sts=4 tw=100 */ diff --git a/sample/echo_cookie/client_sync.cc b/sample/echo_cookie/client_sync.cc deleted file mode 100644 index e029a94..0000000 --- a/sample/echo_cookie/client_sync.cc +++ /dev/null @@ -1,79 +0,0 @@ -// Copyright (c) 2014 Baidu.com, Inc. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. -// -// Author: qinzuoyan01@baidu.com (Qin Zuoyan) - -#include -#include "cookie_plugin/rpc_cookie.h" -#include "echo_service.pb.h" -#include - -typedef sofa::pbrpc::shared_ptr CookiePtr; -sofa::pbrpc::RpcCookieManager cookie_manager; - -// Using global RpcClient object can help share resources such as threads and buffers. -sofa::pbrpc::RpcClient g_rpc_client; - -int main() -{ - SOFA_PBRPC_SET_LOG_LEVEL(NOTICE); - - // Define an rpc channel. - sofa::pbrpc::RpcChannelOptions channel_options; - sofa::pbrpc::RpcChannel rpc_channel(&g_rpc_client, "127.0.0.1:12300", channel_options); - - // Prepare parameters. - sofa::pbrpc::RpcController* cntl = new sofa::pbrpc::RpcController(); - cntl->SetTimeout(30000); - CookiePtr cookie(new sofa::pbrpc::Cookie(&cookie_manager)); - cookie->Set("type", "sync"); - cookie->Set("logid", "123456"); - cntl->SetRequestAttachment(cookie.get()); - sofa::pbrpc::test::EchoRequest* request = - new sofa::pbrpc::test::EchoRequest(); - request->set_message("Hello from qinzuoyan01"); - sofa::pbrpc::test::EchoResponse* response = - new sofa::pbrpc::test::EchoResponse(); - - // Sync call. - sofa::pbrpc::test::EchoServer_Stub* stub = - new sofa::pbrpc::test::EchoServer_Stub(&rpc_channel); - stub->Echo(cntl, request, response, NULL); - - // Check if the request has been sent. - // If has been sent, then can get the sent bytes. - SLOG(NOTICE, "RemoteAddress=%s", cntl->RemoteAddress().c_str()); - SLOG(NOTICE, "IsRequestSent=%s", cntl->IsRequestSent() ? "true" : "false"); - if (cntl->IsRequestSent()) - { - SLOG(NOTICE, "LocalAddress=%s", cntl->LocalAddress().c_str()); - SLOG(NOTICE, "SentBytes=%ld", cntl->SentBytes()); - } - - // Check if failed. - if (cntl->Failed()) - { - SLOG(ERROR, "request failed: %s", cntl->ErrorText().c_str()); - } - else - { - SLOG(NOTICE, "request succeed: %s", response->message().c_str()); - cookie.reset(new sofa::pbrpc::Cookie(&cookie_manager)); - cntl->GetResponseAttachment(cookie.get()); - std::string version; - cookie->Get("version", version); - SLOG(NOTICE, "cookie version : %s", version.c_str()); - cookie->Store(); - } - - // Destroy objects. - delete cntl; - delete request; - delete response; - delete stub; - - return EXIT_SUCCESS; -} - -/* vim: set ts=4 sw=4 sts=4 tw=100 */ diff --git a/sample/echo_cookie/cookie_plugin/rpc_cookie.cc b/sample/echo_cookie/cookie_plugin/rpc_cookie.cc deleted file mode 100644 index e6f3a79..0000000 --- a/sample/echo_cookie/cookie_plugin/rpc_cookie.cc +++ /dev/null @@ -1,170 +0,0 @@ -// Copyright (c) 2014 Baidu.com, Inc. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. -// -// Author: zhangdi05@baidu.com (Zhang Di) - -#include -#include "rpc_cookie.h" - -namespace sofa { -namespace pbrpc { - -bool RpcCookieManager::Has(const std::string& key) -{ - ScopedLocker _(_lock); - return _kv_map.find(key) != _kv_map.end(); -} - -void RpcCookieManager::Load(MapType& kv_map) -{ - ScopedLocker _(_lock); - kv_map = _kv_map; -} - -void RpcCookieManager::Store(const MapType& kv_map) -{ - ScopedLocker _(_lock); - _kv_map.insert(kv_map.begin(), kv_map.end()); -} - -Cookie::Cookie() : _keep_type(ALWAYS_NEW) -{ } - -Cookie::Cookie(RpcCookieManager* manager, KeepType keep_type) : - _manager(manager), _keep_type(keep_type) -{ - if (_keep_type == ALWAYS_NEW) - { - return; - } - else if (_manager != NULL) - { - _manager->Load(_kv_map); - } -} - -bool Cookie::Get(const std::string& key, std::string& value) -{ - MapType::const_iterator it = _kv_map.find(key); - if (it == _kv_map.end()) - { - return false; - } - value = it->second; - return true; -} - -void Cookie::Set(const std::string& key, const std::string& value) -{ - if (_keep_type == MANAGER_FIRST) - { - if (!_manager->Has(key)) - { - _kv_map[key] = value; - } - return; - } - _kv_map[key] = value; -} - -void Cookie::Store() -{ - if (_manager != NULL) - { - _manager->Store(_kv_map); - } -} - -bool Cookie::Has(const std::string& key) -{ - return _kv_map.find(key) != _kv_map.end(); -} - -bool Cookie::Erase(const std::string& key) -{ - MapType::iterator it = _kv_map.find(key); - if (it == _kv_map.end()) - { - return false; - } - _kv_map.erase(key); - return true; -} - -void Cookie::Clear() -{ - _kv_map.clear(); -} - -bool Cookie::Serialize(sofa::pbrpc::ReadBufferPtr& attach_buffer) -{ - int size = _kv_map.size(); - SCHECK_LE(static_cast(size), SOFA_CONTAINER_MAX_SERIALIZE_SIZE); - WriteBufferPtr write_buffer(new WriteBuffer()); - Serializer serializer(write_buffer); - if (!serializer.serialize_varint(size)) - { - return false; - } - MapType::const_iterator it = _kv_map.begin(); - for (; it != _kv_map.end(); ++it) - { - if (!serializer.serialize_string(it->first)) - { - return false; - } - if (!serializer.serialize_string(it->second)) - { - return false; - } - } - write_buffer->SwapOut(attach_buffer.get()); - return true; -} - -int Cookie::SerializeLen() const -{ - int len = Serializer::varint_len(_kv_map.size()); - MapType::const_iterator it = _kv_map.begin(); - for (; it != _kv_map.end(); ++it) - { - len += Serializer::string_len(it->first); - len += Serializer::string_len(it->second); - } - return len; -} - -bool Cookie::Deserialize(sofa::pbrpc::ReadBufferPtr& attach_buffer) -{ - int size = 0; - Deserializer deserializer(attach_buffer); - if (!deserializer.deserialize_varint(size)) - { - return false; - } - if (static_cast(size) > SOFA_CONTAINER_MAX_SERIALIZE_SIZE) - { - return false; - } - _kv_map.clear(); - for (int i = 0; i < size; ++i) - { - std::string key; - if (!deserializer.deserialize_string(key)) - { - return false; - } - std::string value; - if (!deserializer.deserialize_string(value)) - { - return false; - } - _kv_map[key] = value; - } - return true; -} -} // namespace pbrpc -} // namespace sofa - -/* vim: set ts=4 sw=4 sts=4 tw=100 */ diff --git a/sample/echo_cookie/cookie_plugin/rpc_cookie.h b/sample/echo_cookie/cookie_plugin/rpc_cookie.h deleted file mode 100644 index 4608a10..0000000 --- a/sample/echo_cookie/cookie_plugin/rpc_cookie.h +++ /dev/null @@ -1,94 +0,0 @@ -// Copyright (c) 2014 Baidu.com, Inc. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. -// -// Author: zhangdi05@baidu.com (Zhang Di) - -#ifndef _RPC_COOKIE_H_ -#define _RPC_COOKIE_H_ - -#include -#include -#include -#include -#include -#include - -namespace sofa { -namespace pbrpc { - -enum KeepType -{ - ALWAYS_NEW = 0, - USER_FIRST = 1, - MANAGER_FIRST = 2 -}; - -typedef std::map MapType; - -//class RpcCookieManager -class RpcCookieManager -{ -public: - RpcCookieManager() - { } - - ~RpcCookieManager() - { } - - bool Has(const std::string& key); - - void Load(MapType& kv_map); - - void Store(const MapType& kv_map); - -private: - MapType _kv_map; - - MutexLock _lock; -}; - -//class Cookie -class Cookie : public sofa::pbrpc::RpcAttachment -{ -public: - Cookie(); - - Cookie(RpcCookieManager* manager, KeepType keep_type = USER_FIRST); - - virtual ~Cookie() { } - - bool Get(const std::string& key, std::string& value); - - void Set(const std::string& key, const std::string& value); - - bool Has(const std::string& key); - - bool Erase(const std::string& key); - - void Clear(); - - void Store(); - - virtual bool Serialize(sofa::pbrpc::ReadBufferPtr& attach_buffer); - - virtual bool Deserialize(sofa::pbrpc::ReadBufferPtr& attach_buffer); - - int SerializeLen() const; - -private: - MapType _kv_map; - - RpcCookieManager* _manager; - - KeepType _keep_type; - - SOFA_PBRPC_DISALLOW_EVIL_CONSTRUCTORS(Cookie); -}; //class RpcCookie - -} // namespace pbrpc -} // namespace sofa - -#endif // _SOFA_PBRPC_RPC_COOKIE_H_ - -/* vim: set ts=4 sw=4 sts=4 tw=100 */ diff --git a/sample/echo_cookie/echo_service.proto b/sample/echo_cookie/echo_service.proto deleted file mode 100644 index 7886d13..0000000 --- a/sample/echo_cookie/echo_service.proto +++ /dev/null @@ -1,20 +0,0 @@ -import "sofa/pbrpc/rpc_option.proto"; - -package sofa.pbrpc.test; - -option cc_generic_services = true; - -message EchoRequest { - required string message = 1; -} - -message EchoResponse { - required string message = 1; -} - -service EchoServer { - rpc Echo(EchoRequest) returns(EchoResponse) { - option (sofa.pbrpc.request_compress_type) = CompressTypeSnappy; - option (sofa.pbrpc.response_compress_type) = CompressTypeSnappy; - } -} diff --git a/sample/echo_cookie/server.cc b/sample/echo_cookie/server.cc deleted file mode 100644 index 2feb775..0000000 --- a/sample/echo_cookie/server.cc +++ /dev/null @@ -1,125 +0,0 @@ -// Copyright (c) 2014 Baidu.com, Inc. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. -// -// Author: qinzuoyan01@baidu.com (Qin Zuoyan) - -#include -#include -#include -#include "cookie_plugin/rpc_cookie.h" -#include "echo_service.pb.h" - -typedef sofa::pbrpc::shared_ptr CookiePtr; - -bool WebServlet(const sofa::pbrpc::HTTPRequest& request, sofa::pbrpc::HTTPResponse& response) -{ - SLOG(INFO, "WebServlet(): request message from %s:%u", - request.client_ip.c_str(), request.client_port); - SLOG(INFO, "HTTP-PATH=\"%s\"", request.path.c_str()); - std::map::const_iterator it; - const std::map& query_params = *request.query_params; - for (it = query_params.begin(); it != query_params.end(); ++it) { - SLOG(INFO, "QueryParam[\"%s\"]=\"%s\"", it->first.c_str(), it->second.c_str()); - } - const std::map& headers = *request.headers; - for (it = headers.begin(); it != headers.end(); ++it) { - SLOG(INFO, "Header[\"%s\"]=\"%s\"", it->first.c_str(), it->second.c_str()); - } - return response.content->Append("

Hello from sofa-pbrpc web server

"); -} - -class EchoServerImpl : public sofa::pbrpc::test::EchoServer -{ -public: - EchoServerImpl() {} - virtual ~EchoServerImpl() {} - -private: - virtual void Echo(google::protobuf::RpcController* controller, - const sofa::pbrpc::test::EchoRequest* request, - sofa::pbrpc::test::EchoResponse* response, - google::protobuf::Closure* done) - { - sofa::pbrpc::RpcController* cntl = static_cast(controller); - SLOG(INFO, "Echo(): request message from %s: %s", - cntl->RemoteAddress().c_str(), request->message().c_str()); - if (cntl->IsHttp()) { - SLOG(INFO, "HTTP-PATH=\"%s\"", cntl->HttpPath().c_str()); - std::map::const_iterator it; - const std::map& query_params = cntl->HttpQueryParams(); - for (it = query_params.begin(); it != query_params.end(); ++it) { - SLOG(INFO, "QueryParam[\"%s\"]=\"%s\"", it->first.c_str(), it->second.c_str()); - } - const std::map& headers = cntl->HttpHeaders(); - for (it = headers.begin(); it != headers.end(); ++it) { - SLOG(INFO, "Header[\"%s\"]=\"%s\"", it->first.c_str(), it->second.c_str()); - } - } - CookiePtr cookie(new sofa::pbrpc::Cookie()); - cntl->GetRequestAttachment(cookie.get()); - std::string type; - std::string logid; - cookie->Get("type", type); - cookie->Get("logid", logid); - SLOG(INFO, "cookie info : type=%s, logid=%s", type.c_str(), logid.c_str()); - response->set_message("echo message: " + request->message()); - cookie->Set("version", "1.00"); - cntl->SetResponseAttachment(cookie.get()); - done->Run(); - } -}; - -bool thread_init_func() -{ - sleep(1); - SLOG(INFO, "Init work thread succeed"); - return true; -} - -void thread_dest_func() -{ - SLOG(INFO, "Destroy work thread succeed"); -} - -int main() -{ - SOFA_PBRPC_SET_LOG_LEVEL(NOTICE); - - // Define an rpc server. - sofa::pbrpc::RpcServerOptions options; - options.work_thread_init_func = sofa::pbrpc::NewPermanentExtClosure(&thread_init_func); - options.work_thread_dest_func = sofa::pbrpc::NewPermanentExtClosure(&thread_dest_func); - sofa::pbrpc::RpcServer rpc_server(options); - - sofa::pbrpc::Servlet servlet = sofa::pbrpc::NewPermanentExtClosure(&WebServlet); - rpc_server.RegisterWebServlet("/hello", servlet); - - // Start rpc server. - if (!rpc_server.Start("0.0.0.0:12300")) { - SLOG(ERROR, "start server failed"); - return EXIT_FAILURE; - } - - // Register service. - sofa::pbrpc::test::EchoServer* echo_service = new EchoServerImpl(); - if (!rpc_server.RegisterService(echo_service)) { - SLOG(ERROR, "export service failed"); - return EXIT_FAILURE; - } - - // Wait signal. - rpc_server.Run(); - - // Stop rpc server. - rpc_server.Stop(); - - // Delete closures. - // Attention: should delete the closures after server stopped, or may be crash. - delete options.work_thread_init_func; - delete options.work_thread_dest_func; - - return EXIT_SUCCESS; -} - -/* vim: set ts=4 sw=4 sts=4 tw=100 */ diff --git a/src/sofa/pbrpc/binary_rpc_request.cc b/src/sofa/pbrpc/binary_rpc_request.cc index 1a67bfa..85bc7dc 100644 --- a/src/sofa/pbrpc/binary_rpc_request.cc +++ b/src/sofa/pbrpc/binary_rpc_request.cc @@ -83,12 +83,48 @@ void BinaryRpcRequest::ProcessRequest( bool parse_request_return = false; if (compress_type == CompressTypeNone) { - parse_request_return = request->ParseFromZeroCopyStream(_req_body.get()); + parse_request_return = + request->ParseFromBoundedZeroCopyStream(_req_body.get(), _req_header.data_size); } else { + ReadBufferPtr read_buffer(new ReadBuffer()); + int bytes_read = 0; + while (bytes_read < _req_header.data_size) + { + const char* read_pos = NULL; + int cur_size; + int bytes_remain = _req_header.data_size - bytes_read; + char* handle_data = _req_body->CurrentHandle(); + int handle_offset = _req_body->CurrentHandleOffset(); + if (!_req_body->Next(reinterpret_cast(&read_pos), &cur_size)) + { +#if defined( LOG ) + LOG(ERROR) << "ProcessRequest(): " << RpcEndpointToString(_remote_endpoint) + << ": {" << _req_meta.sequence_id() << "}" + << ": bad request buffer"; +#else + SLOG(ERROR, "ProcessRequest(): %s: {%lu}: bad request buffer", + RpcEndpointToString(_remote_endpoint).c_str(), + _req_meta.sequence_id()); +#endif + SendFailedResponse(stream, RPC_ERROR_PARSE_REQUEST_MESSAGE, "bad request buffer"); + return; + } + if (bytes_remain >= cur_size) + { + read_buffer->Append(BufHandle(handle_data, cur_size, handle_offset)); + bytes_read += cur_size; + } + else + { + _req_body->BackUp(cur_size - bytes_remain); + read_buffer->Append(BufHandle(handle_data, bytes_remain, handle_offset)); + bytes_read += bytes_remain; + } + } sofa::pbrpc::scoped_ptr is( - get_compressed_input_stream(_req_body.get(), compress_type)); + get_compressed_input_stream(read_buffer.get(), compress_type)); parse_request_return = request->ParseFromZeroCopyStream(is.get()); } if (!parse_request_return) @@ -119,11 +155,8 @@ void BinaryRpcRequest::ProcessRequest( cntl->SetResponseCompressType(_req_meta.has_expected_response_compress_type() ? _req_meta.expected_response_compress_type() : CompressTypeNone); - WriteBuffer write_buffer; - write_buffer.Append(_req_header.attach_buffer, _req_header.attach_size); - ReadBufferPtr read_buffer(new ReadBuffer()); - write_buffer.SwapOut(read_buffer.get()); - cntl->SetRequestAttachBuffer(read_buffer); + cntl->SetRequestSize(_req_header.data_size); + cntl->SetRequestAttachBuffer(_req_body); CallMethod(method_board, controller, request, response); } @@ -172,20 +205,9 @@ ReadBufferPtr BinaryRpcRequest::AssembleSucceedResponse( return ReadBufferPtr(); } header.data_size = write_buffer.ByteCount() - header_pos - header_size - header.meta_size; - header.message_size = header.meta_size + header.data_size; - - std::string response_attach_str = cntl->GetResponseAttachBuffer()->ToString(); - size_t attach_size = response_attach_str.size(); - if (attach_size > 0) - { - if (attach_size > ATTACH_BUFFER_SIZE) - { - err = "serialize response message failed for attach data exceed the limit"; - return ReadBufferPtr(); - } - memcpy(header.attach_buffer, response_attach_str.c_str(), attach_size); - header.attach_size = attach_size; - } + write_buffer.Append(cntl->GetResponseAttachBuffer()->ToString()); + int attach_size = write_buffer.ByteCount() - header_pos - header_size - header.meta_size - header.data_size; + header.message_size = header.meta_size + header.data_size + attach_size; write_buffer.SetData(header_pos, reinterpret_cast(&header), header_size); diff --git a/src/sofa/pbrpc/buffer.cc b/src/sofa/pbrpc/buffer.cc index 1a45435..91408ec 100644 --- a/src/sofa/pbrpc/buffer.cc +++ b/src/sofa/pbrpc/buffer.cc @@ -103,6 +103,16 @@ bool ReadBuffer::Next(const void** data, int* size) } } +char* ReadBuffer::CurrentHandle() +{ + return _cur_it->data; +} + +int ReadBuffer::CurrentHandleOffset() +{ + return _cur_it->offset + _cur_pos; +} + // BackUp() can only be called after a successful Next(). // "count" should be greater than or equal to 0. void ReadBuffer::BackUp(int count) diff --git a/src/sofa/pbrpc/buffer.h b/src/sofa/pbrpc/buffer.h index 42ae569..f9224ae 100644 --- a/src/sofa/pbrpc/buffer.h +++ b/src/sofa/pbrpc/buffer.h @@ -58,6 +58,8 @@ class ReadBuffer : public google::protobuf::io::ZeroCopyInputStream // implements ZeroCopyInputStream ---------------------------------- bool Next(const void** data, int* size); + char* CurrentHandle(); + int CurrentHandleOffset(); void BackUp(int count); bool Skip(int count); int64 ByteCount() const; diff --git a/src/sofa/pbrpc/rpc_client_impl.cc b/src/sofa/pbrpc/rpc_client_impl.cc index 7608c86..f04bcdf 100644 --- a/src/sofa/pbrpc/rpc_client_impl.cc +++ b/src/sofa/pbrpc/rpc_client_impl.cc @@ -317,26 +317,9 @@ void RpcClientImpl::CallMethod(const google::protobuf::Message* request, return; } header.data_size = write_buffer.ByteCount() - header_pos - header_size - header.meta_size; - header.message_size = header.meta_size + header.data_size; - std::string request_attach_str = cntl->GetRequestAttachBuffer()->ToString(); - size_t attach_size = request_attach_str.size(); - if (attach_size > 0) - { - if (attach_size > ATTACH_BUFFER_SIZE) - { -#if defined( LOG ) - LOG(ERROR) << "CallMethod(): " << RpcEndpointToString(cntl->RemoteEndpoint()) - << ":attach data exceed the limit"; -#else - SLOG(ERROR, "CallMethod(): %s: attach data exceed the limit", - RpcEndpointToString(cntl->RemoteEndpoint()).c_str()); -#endif - cntl->Done(RPC_ERROR_SERIALIZE_REQUEST, "attach data exceed the limit"); - return; - } - memcpy(header.attach_buffer, request_attach_str.c_str(), attach_size); - header.attach_size = attach_size; - } + write_buffer.Append(cntl->GetRequestAttachBuffer()->ToString()); + int attach_size = write_buffer.ByteCount() - header_pos - header_size - header.meta_size - header.data_size; + header.message_size = header.meta_size + header.data_size + attach_size; write_buffer.SetData(header_pos, reinterpret_cast(&header), header_size); @@ -450,15 +433,49 @@ void RpcClientImpl::DoneCallback(google::protobuf::Message* response, SCHECK(cntl->ResponseBuffer()); ReadBufferPtr buffer = cntl->ResponseBuffer(); CompressType compress_type = cntl->ResponseCompressType(); + int data_size = cntl->ResponseSize(); bool parse_response_return = false; if (compress_type == CompressTypeNone) { - parse_response_return = response->ParseFromZeroCopyStream(buffer.get()); + parse_response_return = response->ParseFromBoundedZeroCopyStream(buffer.get(), data_size); } else { + ReadBufferPtr read_buffer(new ReadBuffer()); + int bytes_read = 0; + while (bytes_read < data_size) + { + const char* read_pos = NULL; + int cur_size; + int bytes_remain = data_size - bytes_read; + char* handle_data = buffer->CurrentHandle(); + int handle_offset = buffer->CurrentHandleOffset(); + if (!buffer->Next(reinterpret_cast(&read_pos), &cur_size)) + { +#if defined ( LOG ) + LOG(ERROR) << "DoneCallback(): " << RpcEndpointToString(cntl->RemoteEndpoint()) + << ": bad response buffer"; +#else + SLOG(ERROR, "DoneCallback(): %s: bad response buffer", + RpcEndpointToString(cntl->RemoteEndpoint()).c_str()); +#endif + cntl->SetFailed(RPC_ERROR_PARSE_RESPONSE_MESSAGE, "bad response buffer"); + return; + } + if (bytes_remain >= cur_size) + { + read_buffer->Append(BufHandle(handle_data, cur_size, handle_offset)); + bytes_read += cur_size; + } + else + { + buffer->BackUp(cur_size - bytes_remain); + read_buffer->Append(BufHandle(handle_data, bytes_remain, handle_offset)); + bytes_read += bytes_remain; + } + } ::sofa::pbrpc::scoped_ptr is( - get_compressed_input_stream(buffer.get(), compress_type)); + get_compressed_input_stream(read_buffer.get(), compress_type)); parse_response_return = response->ParseFromZeroCopyStream(is.get()); } if (!parse_response_return) @@ -471,7 +488,9 @@ void RpcClientImpl::DoneCallback(google::protobuf::Message* response, RpcEndpointToString(cntl->RemoteEndpoint()).c_str()); #endif cntl->SetFailed(RPC_ERROR_PARSE_RESPONSE_MESSAGE, "parse response message pb failed"); + return; } + cntl->SetResponseAttachBuffer(buffer); } } diff --git a/src/sofa/pbrpc/rpc_client_stream.h b/src/sofa/pbrpc/rpc_client_stream.h index fecd9cb..afe65b4 100644 --- a/src/sofa/pbrpc/rpc_client_stream.h +++ b/src/sofa/pbrpc/rpc_client_stream.h @@ -10,6 +10,7 @@ #include #include #include +#include namespace sofa { namespace pbrpc { @@ -150,7 +151,7 @@ class RpcClientStream : public RpcMessageStream const ReadBufferPtr& message, int meta_size, int64 data_size, - const std::string& attach_buffer) + int attach_size) { SOFA_PBRPC_FUNCTION_TRACE; @@ -258,13 +259,9 @@ class RpcClientStream : public RpcMessageStream } else // !meta.failed() { - SCHECK_EQ(data_size, message->TotalCount() - message->ByteCount()); - WriteBuffer write_buffer; - write_buffer.Append(attach_buffer); - ReadBufferPtr read_buffer(new ReadBuffer()); - write_buffer.SwapOut(read_buffer.get()); - cntl->SetResponseAttachBuffer(read_buffer); + SCHECK_EQ(data_size + attach_size, message->TotalCount() - message->ByteCount()); cntl->SetResponseBuffer(message); + cntl->SetResponseSize(data_size); cntl->SetResponseCompressType(meta.has_compress_type() ? meta.compress_type() : CompressTypeNone); cntl->Done(RPC_SUCCESS, "succeed"); diff --git a/src/sofa/pbrpc/rpc_controller_impl.h b/src/sofa/pbrpc/rpc_controller_impl.h index 7ab17a5..0aa2c80 100644 --- a/src/sofa/pbrpc/rpc_controller_impl.h +++ b/src/sofa/pbrpc/rpc_controller_impl.h @@ -467,6 +467,26 @@ class RpcControllerImpl : public sofa::pbrpc::enable_shared_from_thisAppend(BufHandle(_tran_buf, consume_size, data - _tran_buf)); } - std::string response_attach_str(_receiving_header.attach_buffer, _receiving_header.attach_size); + int attach_size = _receiving_header.message_size - _receiving_header.meta_size - _receiving_header.data_size; received_messages->push_back(ReceivedItem(_receiving_message, - _receiving_header.meta_size, _receiving_header.data_size, response_attach_str)); + _receiving_header.meta_size, _receiving_header.data_size, attach_size)); reset_receiving_env(); data += consume_size; size -= consume_size; @@ -652,22 +652,20 @@ class RpcMessageStream : public RpcByteStream #endif return -1; } - if (_receiving_header.meta_size + _receiving_header.data_size != _receiving_header.message_size) + if (_receiving_header.meta_size + _receiving_header.data_size > _receiving_header.message_size) { #if defined( LOG ) LOG(ERROR) << "identify_message_header(): " << RpcEndpointToString(_remote_endpoint) << ": check size in header failed" << ": meta_size=" << _receiving_header.meta_size << ", data_size=" << _receiving_header.data_size - << ", attach_size=" << _receiving_header.attach_size << ", message_size=" << _receiving_header.message_size; #else SLOG(ERROR, "identify_message_header(): %s: " - "check size in header failed: meta_size=%d, data_size=%lld, attach_size=%lld, message_size=%lld", + "check size in header failed: meta_size=%d, data_size=%lld, message_size=%lld", RpcEndpointToString(_remote_endpoint).c_str(), _receiving_header.meta_size, _receiving_header.data_size, - _receiving_header.attach_size, _receiving_header.message_size); #endif return -1; @@ -741,15 +739,15 @@ class RpcMessageStream : public RpcByteStream ReadBufferPtr message; int meta_size; int64 data_size; - std::string attach_buffer; + int attach_size; ReceivedItem(const ReadBufferPtr& _message, int _meta_size, int64 _data_size, - const std::string& _attach_buffer) + int _attach_size) : message(_message) , meta_size(_meta_size) , data_size(_data_size) - , attach_buffer(_attach_buffer) {} + , attach_size(_attach_size) {} }; // TODO improve sync queue performance diff --git a/src/sofa/pbrpc/serialize.h b/src/sofa/pbrpc/serialize.h index 68d16ca..28ff9c9 100644 --- a/src/sofa/pbrpc/serialize.h +++ b/src/sofa/pbrpc/serialize.h @@ -4,8 +4,8 @@ // // Author: qinzuoyan01@baidu.com (Qin Zuoyan) -#ifndef _SOFA_PBRPC_RPC_COOKIE_H_ -#define _SOFA_PBRPC_RPC_COOKIE_H_ +#ifndef _SOFA_PBRPC_SERIALIZE_H_ +#define _SOFA_PBRPC_SERIALIZE_H_ #include #include @@ -341,6 +341,6 @@ class Deserializer } // namespace pbrpc } // namespace sofa -#endif // _SOFA_PBRPC_RPC_COOKIE_H_ +#endif // _SOFA_PBRPC_SERIALIZE_H_ /* vim: set ts=4 sw=4 sts=4 tw=100 */ From b942f9b8090004f0c54f44b4f81e84e2cf9382a0 Mon Sep 17 00:00:00 2001 From: zd-double Date: Mon, 4 Apr 2016 21:57:53 +0800 Subject: [PATCH 10/13] update the interface about attachment in rpc controller --- sample/echo/client_async.cc | 12 +++++++----- sample/echo/client_sync.cc | 12 +++++++----- sample/echo/server.cc | 16 +++++++++------- src/sofa/pbrpc/buffer.h | 5 +++++ src/sofa/pbrpc/rpc_client_stream.h | 1 - src/sofa/pbrpc/rpc_controller.cc | 16 ++++++++-------- src/sofa/pbrpc/rpc_controller.h | 20 ++++++++++++++------ src/sofa/pbrpc/rpc_controller_impl.h | 16 ++++++++-------- 8 files changed, 58 insertions(+), 40 deletions(-) diff --git a/sample/echo/client_async.cc b/sample/echo/client_async.cc index a29a63a..d927c9f 100644 --- a/sample/echo/client_async.cc +++ b/sample/echo/client_async.cc @@ -31,11 +31,13 @@ void EchoCallback(sofa::pbrpc::RpcController* cntl, else { SLOG(NOTICE, "request succeed: %s", response->message().c_str()); CookiePtr cookie(new sofa::pbrpc::Cookie(&cookie_manager)); - cntl->GetResponseAttachment(cookie.get()); - std::string version; - cookie->Get("version", version); - SLOG(NOTICE, "cookie version : %s", version.c_str()); - cookie->Store(); + if (cntl->GetResponseAttachment(cookie.get())) + { + std::string version; + cookie->Get("version", version); + SLOG(NOTICE, "cookie version=%s", version.c_str()); + cookie->Store(); + } } delete cntl; diff --git a/sample/echo/client_sync.cc b/sample/echo/client_sync.cc index c59641a..2e637c5 100644 --- a/sample/echo/client_sync.cc +++ b/sample/echo/client_sync.cc @@ -60,11 +60,13 @@ int main() { SLOG(NOTICE, "request succeed: %s", response->message().c_str()); cookie.reset(new sofa::pbrpc::Cookie(&cookie_manager)); - cntl->GetResponseAttachment(cookie.get()); - std::string version; - cookie->Get("version", version); - SLOG(NOTICE, "cookie version : %s", version.c_str()); - cookie->Store(); + if (cntl->GetResponseAttachment(cookie.get())) + { + std::string version; + cookie->Get("version", version); + SLOG(NOTICE, "cookie version=%s", version.c_str()); + cookie->Store(); + } } // Destroy objects. diff --git a/sample/echo/server.cc b/sample/echo/server.cc index 59177b4..9b489b4 100644 --- a/sample/echo/server.cc +++ b/sample/echo/server.cc @@ -57,15 +57,17 @@ class EchoServerImpl : public sofa::pbrpc::test::EchoServer } } CookiePtr cookie(new sofa::pbrpc::Cookie()); - cntl->GetRequestAttachment(cookie.get()); - std::string type; - std::string logid; - cookie->Get("type", type); - cookie->Get("logid", logid); - SLOG(INFO, "cookie info : type=%s, logid=%s", type.c_str(), logid.c_str()); - response->set_message("echo message: " + request->message()); + if (cntl->GetRequestAttachment(cookie.get())) + { + std::string type; + std::string logid; + cookie->Get("type", type); + cookie->Get("logid", logid); + SLOG(INFO, "cookie info : type=%s, logid=%s", type.c_str(), logid.c_str()); + } cookie->Set("version", "1.00"); cntl->SetResponseAttachment(cookie.get()); + response->set_message("echo message: " + request->message()); done->Run(); } }; diff --git a/src/sofa/pbrpc/buffer.h b/src/sofa/pbrpc/buffer.h index f9224ae..c51ec4e 100644 --- a/src/sofa/pbrpc/buffer.h +++ b/src/sofa/pbrpc/buffer.h @@ -58,8 +58,13 @@ class ReadBuffer : public google::protobuf::io::ZeroCopyInputStream // implements ZeroCopyInputStream ---------------------------------- bool Next(const void** data, int* size); + + // Get the address of current buffer handle. char* CurrentHandle(); + + // Get the offset of current buffer handle. int CurrentHandleOffset(); + void BackUp(int count); bool Skip(int count); int64 ByteCount() const; diff --git a/src/sofa/pbrpc/rpc_client_stream.h b/src/sofa/pbrpc/rpc_client_stream.h index afe65b4..3d34fec 100644 --- a/src/sofa/pbrpc/rpc_client_stream.h +++ b/src/sofa/pbrpc/rpc_client_stream.h @@ -10,7 +10,6 @@ #include #include #include -#include namespace sofa { namespace pbrpc { diff --git a/src/sofa/pbrpc/rpc_controller.cc b/src/sofa/pbrpc/rpc_controller.cc index a3d5389..8bfbfee 100644 --- a/src/sofa/pbrpc/rpc_controller.cc +++ b/src/sofa/pbrpc/rpc_controller.cc @@ -119,24 +119,24 @@ void RpcController::NotifyOnCancel(google::protobuf::Closure* callback) _impl->NotifyOnCancel(callback); } -void RpcController::SetRequestAttachment(RpcAttachment* request_attachment) +bool RpcController::SetRequestAttachment(RpcAttachment* request_attachment) { - _impl->SetRequestAttachment(request_attachment); + return _impl->SetRequestAttachment(request_attachment); } -void RpcController::GetRequestAttachment(RpcAttachment* request_attachment) +bool RpcController::GetRequestAttachment(RpcAttachment* request_attachment) { - _impl->GetRequestAttachment(request_attachment); + return _impl->GetRequestAttachment(request_attachment); } -void RpcController::SetResponseAttachment(RpcAttachment* response_attachment) +bool RpcController::SetResponseAttachment(RpcAttachment* response_attachment) { - _impl->SetResponseAttachment(response_attachment); + return _impl->SetResponseAttachment(response_attachment); } -void RpcController::GetResponseAttachment(RpcAttachment* response_attachment) +bool RpcController::GetResponseAttachment(RpcAttachment* response_attachment) { - _impl->GetResponseAttachment(response_attachment); + return _impl->GetResponseAttachment(response_attachment); } } // namespace pbrpc diff --git a/src/sofa/pbrpc/rpc_controller.h b/src/sofa/pbrpc/rpc_controller.h index 1ceedd1..63f35a5 100644 --- a/src/sofa/pbrpc/rpc_controller.h +++ b/src/sofa/pbrpc/rpc_controller.h @@ -124,6 +124,14 @@ class RpcController : public google::protobuf::RpcController // // Not supported now. virtual void StartCancel(); + + // If true, indicates that the attachment data on the client side has been + // serialized successfully. + bool SetRequestAttachment(RpcAttachment* request_attachment); + + // If true, indicates that the attachment data from the server side has been + // deserialized successfully. + bool GetResponseAttachment(RpcAttachment* response_attachment); // -------- used only by server side --------- // These calls should be made from the server side only. Their results @@ -165,13 +173,13 @@ class RpcController : public google::protobuf::RpcController // NotifyOnCancel() must be called no more than once per request. virtual void NotifyOnCancel(google::protobuf::Closure* callback); - void SetRequestAttachment(RpcAttachment* request_attachment); - - void GetRequestAttachment(RpcAttachment* attachment); - - void SetResponseAttachment(RpcAttachment* response_attachment); + // If true, indicates that the attachment data from the client side has been + // deserialized successfully. + bool GetRequestAttachment(RpcAttachment* request_attachment); - void GetResponseAttachment(RpcAttachment* attachment); + // If true, indicates that the attachment data on the server side has been + // serialized successfully. + bool SetResponseAttachment(RpcAttachment* response_attachment); public: const sofa::pbrpc::shared_ptr& impl() const diff --git a/src/sofa/pbrpc/rpc_controller_impl.h b/src/sofa/pbrpc/rpc_controller_impl.h index 0aa2c80..1f36b83 100644 --- a/src/sofa/pbrpc/rpc_controller_impl.h +++ b/src/sofa/pbrpc/rpc_controller_impl.h @@ -332,24 +332,24 @@ class RpcControllerImpl : public sofa::pbrpc::enable_shared_from_thisSerialize(_request_attach_buffer); + return request_attachment->Serialize(_request_attach_buffer); } - void GetRequestAttachment(RpcAttachment* request_attachment) + bool GetRequestAttachment(RpcAttachment* request_attachment) { - request_attachment->Deserialize(_request_attach_buffer); + return request_attachment->Deserialize(_request_attach_buffer); } - void SetResponseAttachment(RpcAttachment* response_attachment) + bool SetResponseAttachment(RpcAttachment* response_attachment) { - response_attachment->Serialize(_response_attach_buffer); + return response_attachment->Serialize(_response_attach_buffer); } - void GetResponseAttachment(RpcAttachment* response_attachment) + bool GetResponseAttachment(RpcAttachment* response_attachment) { - response_attachment->Deserialize(_response_attach_buffer);; + return response_attachment->Deserialize(_response_attach_buffer);; } void SetRequestAttachBuffer(const ReadBufferPtr& request_attach_buffer) From 9aeb14c0589ee3cef7fe202ba1939577eb97ba25 Mon Sep 17 00:00:00 2001 From: zd-double Date: Sat, 9 Apr 2016 15:00:37 +0800 Subject: [PATCH 11/13] improve the interfaces about cookie --- sample/echo/Makefile | 2 +- sample/echo/client_async.cc | 11 +-- sample/echo/client_sync.cc | 8 +- sample/echo/server.cc | 4 +- src/sofa/pbrpc/binary_rpc_request.cc | 2 +- src/sofa/pbrpc/buffer.cc | 5 -- src/sofa/pbrpc/buffer.h | 3 - src/sofa/pbrpc/plugin/cookie/rpc_cookie.cc | 89 +++++++++++++--------- src/sofa/pbrpc/plugin/cookie/rpc_cookie.h | 27 +++---- src/sofa/pbrpc/rpc_attachment.h | 29 +++---- src/sofa/pbrpc/rpc_client_impl.cc | 2 +- 11 files changed, 88 insertions(+), 94 deletions(-) diff --git a/sample/echo/Makefile b/sample/echo/Makefile index 62e2d68..1a6db23 100644 --- a/sample/echo/Makefile +++ b/sample/echo/Makefile @@ -60,7 +60,7 @@ check_depends: @if [ ! -f "$(SOFA_PBRPC)/lib/libsofa-pbrpc-full.a" ]; then echo "ERROR: need sofa-pbrpc-full lib"; exit 1; fi clean: - @rm -f $(BIN) *.o *.pb.* + @rm -f $(BIN) *.o *.pb.* rebuild: clean all diff --git a/sample/echo/client_async.cc b/sample/echo/client_async.cc index d927c9f..16d0523 100644 --- a/sample/echo/client_async.cc +++ b/sample/echo/client_async.cc @@ -9,8 +9,8 @@ #include #include "echo_service.pb.h" -typedef sofa::pbrpc::shared_ptr CookiePtr; -sofa::pbrpc::RpcCookieManager cookie_manager; +typedef sofa::pbrpc::shared_ptr RpcCookiePtr; +sofa::pbrpc::RpcCookieManager g_cookie_manager; void EchoCallback(sofa::pbrpc::RpcController* cntl, sofa::pbrpc::test::EchoRequest* request, @@ -28,9 +28,10 @@ void EchoCallback(sofa::pbrpc::RpcController* cntl, if (cntl->Failed()) { SLOG(ERROR, "request failed: %s", cntl->ErrorText().c_str()); } - else { + else + { SLOG(NOTICE, "request succeed: %s", response->message().c_str()); - CookiePtr cookie(new sofa::pbrpc::Cookie(&cookie_manager)); + RpcCookiePtr cookie(new sofa::pbrpc::RpcCookie(&g_cookie_manager)); if (cntl->GetResponseAttachment(cookie.get())) { std::string version; @@ -58,7 +59,7 @@ int main() sofa::pbrpc::RpcChannelOptions channel_options; sofa::pbrpc::RpcChannel rpc_channel(&rpc_client, "127.0.0.1:12321", channel_options); - CookiePtr cookie(new sofa::pbrpc::Cookie(&cookie_manager)); + RpcCookiePtr cookie(new sofa::pbrpc::RpcCookie(&g_cookie_manager)); cookie->Load(); cookie->Set("type", "async"); cookie->Set("logid", "123456"); diff --git a/sample/echo/client_sync.cc b/sample/echo/client_sync.cc index 2e637c5..193ccec 100644 --- a/sample/echo/client_sync.cc +++ b/sample/echo/client_sync.cc @@ -8,8 +8,8 @@ #include #include "echo_service.pb.h" -typedef sofa::pbrpc::shared_ptr CookiePtr; -sofa::pbrpc::RpcCookieManager cookie_manager; +typedef sofa::pbrpc::shared_ptr RpcCookiePtr; +sofa::pbrpc::RpcCookieManager g_cookie_manager; // Using global RpcClient object can help share resources such as threads and buffers. sofa::pbrpc::RpcClient g_rpc_client; @@ -25,7 +25,7 @@ int main() // Prepare parameters. sofa::pbrpc::RpcController* cntl = new sofa::pbrpc::RpcController(); cntl->SetTimeout(3000); - CookiePtr cookie(new sofa::pbrpc::Cookie(&cookie_manager)); + RpcCookiePtr cookie(new sofa::pbrpc::RpcCookie(&g_cookie_manager)); cookie->Load(); cookie->Set("type", "sync"); cookie->Set("logid", "123456"); @@ -59,7 +59,7 @@ int main() else { SLOG(NOTICE, "request succeed: %s", response->message().c_str()); - cookie.reset(new sofa::pbrpc::Cookie(&cookie_manager)); + cookie.reset(new sofa::pbrpc::RpcCookie(&g_cookie_manager)); if (cntl->GetResponseAttachment(cookie.get())) { std::string version; diff --git a/sample/echo/server.cc b/sample/echo/server.cc index 9b489b4..b7c7010 100644 --- a/sample/echo/server.cc +++ b/sample/echo/server.cc @@ -10,7 +10,7 @@ #include #include "echo_service.pb.h" -typedef sofa::pbrpc::shared_ptr CookiePtr; +typedef sofa::pbrpc::shared_ptr RpcCookiePtr; bool WebServlet(const sofa::pbrpc::HTTPRequest& request, sofa::pbrpc::HTTPResponse& response) { @@ -56,7 +56,7 @@ class EchoServerImpl : public sofa::pbrpc::test::EchoServer SLOG(INFO, "Header[\"%s\"]=\"%s\"", it->first.c_str(), it->second.c_str()); } } - CookiePtr cookie(new sofa::pbrpc::Cookie()); + RpcCookiePtr cookie(new sofa::pbrpc::RpcCookie()); if (cntl->GetRequestAttachment(cookie.get())) { std::string type; diff --git a/src/sofa/pbrpc/binary_rpc_request.cc b/src/sofa/pbrpc/binary_rpc_request.cc index cd47601..e3bfdad 100644 --- a/src/sofa/pbrpc/binary_rpc_request.cc +++ b/src/sofa/pbrpc/binary_rpc_request.cc @@ -95,7 +95,6 @@ void BinaryRpcRequest::ProcessRequest( const char* read_pos = NULL; int cur_size; int bytes_remain = _req_header.data_size - bytes_read; - char* handle_data = _req_body->CurrentHandle(); int handle_offset = _req_body->CurrentHandleOffset(); if (!_req_body->Next(reinterpret_cast(&read_pos), &cur_size)) { @@ -111,6 +110,7 @@ void BinaryRpcRequest::ProcessRequest( SendFailedResponse(stream, RPC_ERROR_PARSE_REQUEST_MESSAGE, "bad request buffer"); return; } + char* handle_data = const_cast(read_pos) - handle_offset; if (bytes_remain >= cur_size) { read_buffer->Append(BufHandle(handle_data, cur_size, handle_offset)); diff --git a/src/sofa/pbrpc/buffer.cc b/src/sofa/pbrpc/buffer.cc index 91408ec..bff002b 100644 --- a/src/sofa/pbrpc/buffer.cc +++ b/src/sofa/pbrpc/buffer.cc @@ -103,11 +103,6 @@ bool ReadBuffer::Next(const void** data, int* size) } } -char* ReadBuffer::CurrentHandle() -{ - return _cur_it->data; -} - int ReadBuffer::CurrentHandleOffset() { return _cur_it->offset + _cur_pos; diff --git a/src/sofa/pbrpc/buffer.h b/src/sofa/pbrpc/buffer.h index c51ec4e..164dc73 100644 --- a/src/sofa/pbrpc/buffer.h +++ b/src/sofa/pbrpc/buffer.h @@ -58,9 +58,6 @@ class ReadBuffer : public google::protobuf::io::ZeroCopyInputStream // implements ZeroCopyInputStream ---------------------------------- bool Next(const void** data, int* size); - - // Get the address of current buffer handle. - char* CurrentHandle(); // Get the offset of current buffer handle. int CurrentHandleOffset(); diff --git a/src/sofa/pbrpc/plugin/cookie/rpc_cookie.cc b/src/sofa/pbrpc/plugin/cookie/rpc_cookie.cc index 5723088..1718d70 100644 --- a/src/sofa/pbrpc/plugin/cookie/rpc_cookie.cc +++ b/src/sofa/pbrpc/plugin/cookie/rpc_cookie.cc @@ -4,7 +4,7 @@ // // Author: zhangdi05@baidu.com (Zhang Di) -#include +#include #include #include @@ -17,59 +17,74 @@ bool RpcCookieManager::Has(const std::string& key) return _kv_map.find(key) != _kv_map.end(); } -void RpcCookieManager::Load(MapType& kv_map) +void RpcCookieManager::Load(std::map& kv_map) { ScopedLocker _(_lock); kv_map = _kv_map; } -void RpcCookieManager::Store(const MapType& kv_map) +void RpcCookieManager::Store(const std::map& kv_map) { ScopedLocker _(_lock); - _kv_map.insert(kv_map.begin(), kv_map.end()); - - std::vector remove_list; - MapType::const_iterator old_it = _kv_map.begin(); - MapType::const_iterator old_end = _kv_map.end(); - MapType::const_iterator new_it = kv_map.begin(); - MapType::const_iterator new_end = kv_map.end(); - while (old_it != old_end && new_it != new_end) + std::set remove_keys; + std::map add_or_modify_pairs; + std::map::const_iterator old_it = _kv_map.begin(); + std::map::const_iterator new_it = kv_map.begin(); + while (old_it != _kv_map.end() && new_it != kv_map.end()) { if(old_it->first < new_it->first) { - remove_list.push_back(old_it->first); + remove_keys.insert(old_it->first); ++old_it; } - else + else if (old_it->first > new_it->first) { + add_or_modify_pairs.insert(*new_it); ++new_it; + } + else + { + if (old_it->second != new_it->second) + { + add_or_modify_pairs.insert(*new_it); + } ++old_it; + ++new_it; } } - while (old_it != old_end) + while (old_it != _kv_map.end()) { - remove_list.push_back(old_it->first); + remove_keys.insert(old_it->first); ++old_it; } - if (!remove_list.empty()) + if (new_it != kv_map.end()) { - for (std::vector::const_iterator it = remove_list.begin(); - it != remove_list.end(); ++it) - { - _kv_map.erase(*it); - } + add_or_modify_pairs.insert(new_it, kv_map.end()); + } + for (std::set::const_iterator it = remove_keys.begin(); + it != remove_keys.end(); ++it) + { + _kv_map.erase(*it); + } + for (std::map::const_iterator it = add_or_modify_pairs.begin(); + it != add_or_modify_pairs.end(); ++it) + { + _kv_map[it->first] = it->second; } } -Cookie::Cookie() : _manager(NULL) +RpcCookie::RpcCookie() : _manager(NULL) +{ } + +RpcCookie::RpcCookie(RpcCookieManager* manager) : _manager(manager) { } -Cookie::Cookie(RpcCookieManager* manager) : _manager(manager) +RpcCookie::~RpcCookie() { } -bool Cookie::Get(const std::string& key, std::string& value) +bool RpcCookie::Get(const std::string& key, std::string& value) { - MapType::const_iterator it = _kv_map.find(key); + std::map::const_iterator it = _kv_map.find(key); if (it == _kv_map.end()) { return false; @@ -78,12 +93,12 @@ bool Cookie::Get(const std::string& key, std::string& value) return true; } -void Cookie::Set(const std::string& key, const std::string& value) +void RpcCookie::Set(const std::string& key, const std::string& value) { _kv_map[key] = value; } -void Cookie::Load() +void RpcCookie::Load() { if (_manager != NULL) { @@ -91,7 +106,7 @@ void Cookie::Load() } } -void Cookie::Store() +void RpcCookie::Store() { if (_manager != NULL) { @@ -99,17 +114,17 @@ void Cookie::Store() } } -bool Cookie::Has(const std::string& key) +bool RpcCookie::Has(const std::string& key) { return _kv_map.find(key) != _kv_map.end(); } -bool Cookie::Empty() +bool RpcCookie::Empty() { return _kv_map.empty(); } -bool Cookie::Erase(const std::string& key) +bool RpcCookie::Erase(const std::string& key) { if (_kv_map.find(key) == _kv_map.end()) { @@ -119,12 +134,12 @@ bool Cookie::Erase(const std::string& key) return true; } -void Cookie::Clear() +void RpcCookie::Clear() { _kv_map.clear(); } -bool Cookie::Serialize(sofa::pbrpc::ReadBufferPtr& attach_buffer) +bool RpcCookie::Serialize(sofa::pbrpc::ReadBufferPtr& attach_buffer) { int size = _kv_map.size(); SCHECK_LE(static_cast(size), SOFA_CONTAINER_MAX_SERIALIZE_SIZE); @@ -134,7 +149,7 @@ bool Cookie::Serialize(sofa::pbrpc::ReadBufferPtr& attach_buffer) { return false; } - MapType::const_iterator it = _kv_map.begin(); + std::map::const_iterator it = _kv_map.begin(); for (; it != _kv_map.end(); ++it) { if (!serializer.serialize_string(it->first)) @@ -150,10 +165,10 @@ bool Cookie::Serialize(sofa::pbrpc::ReadBufferPtr& attach_buffer) return true; } -int Cookie::SerializeLen() const +int RpcCookie::SerializeLen() const { int len = Serializer::varint_len(_kv_map.size()); - MapType::const_iterator it = _kv_map.begin(); + std::map::const_iterator it = _kv_map.begin(); for (; it != _kv_map.end(); ++it) { len += Serializer::string_len(it->first); @@ -162,7 +177,7 @@ int Cookie::SerializeLen() const return len; } -bool Cookie::Deserialize(sofa::pbrpc::ReadBufferPtr& attach_buffer) +bool RpcCookie::Deserialize(sofa::pbrpc::ReadBufferPtr& attach_buffer) { int size = 0; Deserializer deserializer(attach_buffer); diff --git a/src/sofa/pbrpc/plugin/cookie/rpc_cookie.h b/src/sofa/pbrpc/plugin/cookie/rpc_cookie.h index 43fa1a9..375bf8c 100644 --- a/src/sofa/pbrpc/plugin/cookie/rpc_cookie.h +++ b/src/sofa/pbrpc/plugin/cookie/rpc_cookie.h @@ -9,7 +9,6 @@ #include #include -#include #include #include #include @@ -17,9 +16,6 @@ namespace sofa { namespace pbrpc { -typedef std::map MapType; - -//class RpcCookieManager class RpcCookieManager { public: @@ -31,25 +27,24 @@ class RpcCookieManager bool Has(const std::string& key); - void Load(MapType& kv_map); + void Load(std::map& kv_map); - void Store(const MapType& kv_map); + void Store(const std::map& kv_map); private: - MapType _kv_map; + std::map _kv_map; MutexLock _lock; -}; +}; // class RpcCookieManager -//class Cookie -class Cookie : public sofa::pbrpc::RpcAttachment +class RpcCookie : public sofa::pbrpc::RpcAttachment { public: - Cookie(); + RpcCookie(); - Cookie(RpcCookieManager* manager); + explicit RpcCookie(RpcCookieManager* manager); - virtual ~Cookie() { } + virtual ~RpcCookie(); bool Get(const std::string& key, std::string& value); @@ -74,12 +69,12 @@ class Cookie : public sofa::pbrpc::RpcAttachment int SerializeLen() const; private: - MapType _kv_map; + std::map _kv_map; RpcCookieManager* _manager; - SOFA_PBRPC_DISALLOW_EVIL_CONSTRUCTORS(Cookie); -}; //class RpcCookie + SOFA_PBRPC_DISALLOW_EVIL_CONSTRUCTORS(RpcCookie); +}; // class RpcCookie } // namespace pbrpc } // namespace sofa diff --git a/src/sofa/pbrpc/rpc_attachment.h b/src/sofa/pbrpc/rpc_attachment.h index 8ba6f0d..40ba10d 100644 --- a/src/sofa/pbrpc/rpc_attachment.h +++ b/src/sofa/pbrpc/rpc_attachment.h @@ -1,20 +1,11 @@ -/*************************************************************************** - * - * Copyright (c) 2016 Baidu.com, Inc. All Rights Reserved - * $Id$ - * - **************************************************************************/ - - /** - * @file rpc_attachment.h - * @author zhangdi05(zhangdi05@baidu.com) - * @date 2016/03/16 11:14:32 - * @version $Revision$ - * @brief - * - **/ -#ifndef RPC_ATTACHMENT_H -#define RPC_ATTACHMENT_H +// Copyright (c) 2014 Baidu.com, Inc. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. +// +// Author: zhangdi05@baidu.com (Zhang Di) + +#ifndef _SOFA_PBRPC_RPC_ATTACHMENT_H_ +#define _SOFA_PBRPC_RPC_ATTACHMENT_H_ #include #include @@ -33,11 +24,11 @@ class RpcAttachment virtual bool Serialize(ReadBufferPtr& attachment_buffer) = 0; virtual bool Deserialize(ReadBufferPtr& attachment_buffer) = 0; -}; +}; // class RpcAttachment } // namespace pbrpc } // namespace sofa -#endif // RPC_ATTACHMENT_H +#endif // _SOFA_PBRPC_RPC_ATTACHMENT_H_ /* vim: set ts=4 sw=4 sts=4 tw=100 */ diff --git a/src/sofa/pbrpc/rpc_client_impl.cc b/src/sofa/pbrpc/rpc_client_impl.cc index 66046ac..7446aa9 100644 --- a/src/sofa/pbrpc/rpc_client_impl.cc +++ b/src/sofa/pbrpc/rpc_client_impl.cc @@ -452,7 +452,6 @@ void RpcClientImpl::DoneCallback(google::protobuf::Message* response, const char* read_pos = NULL; int cur_size; int bytes_remain = data_size - bytes_read; - char* handle_data = buffer->CurrentHandle(); int handle_offset = buffer->CurrentHandleOffset(); if (!buffer->Next(reinterpret_cast(&read_pos), &cur_size)) { @@ -466,6 +465,7 @@ void RpcClientImpl::DoneCallback(google::protobuf::Message* response, cntl->SetFailed(RPC_ERROR_PARSE_RESPONSE_MESSAGE, "bad response buffer"); return; } + char* handle_data = const_cast(read_pos) - handle_offset; if (bytes_remain >= cur_size) { read_buffer->Append(BufHandle(handle_data, cur_size, handle_offset)); From 4fcca0898f1c8207397fc2eedaf70ac152318e55 Mon Sep 17 00:00:00 2001 From: zd-double Date: Fri, 3 Jun 2016 22:41:33 +0800 Subject: [PATCH 12/13] remove redundant func --- src/sofa/pbrpc/serialize.cc | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/sofa/pbrpc/serialize.cc b/src/sofa/pbrpc/serialize.cc index 28a5c56..a0d6a0b 100644 --- a/src/sofa/pbrpc/serialize.cc +++ b/src/sofa/pbrpc/serialize.cc @@ -5,7 +5,7 @@ // Author: qinzuoyan01@baidu.com (Qin Zuoyan) #include -#include "serialize.h" +#include namespace sofa { namespace pbrpc { @@ -352,11 +352,7 @@ bool Deserializer::deserialize_varint(uint64_t& d) return deserialize_raw_data(d); } -} // namespace io +} // namespace pbrpc } // namespace sofa -void sofa_touch_sofa_io_serializer() -{ -} - /* vim: set ts=4 sw=4 sts=4 tw=100 */ From 3ca5c4828e8fab02a13b0f4cd51d8412a338306d Mon Sep 17 00:00:00 2001 From: zd-double Date: Mon, 27 Jun 2016 17:31:00 +0800 Subject: [PATCH 13/13] improve implementation of append attach buffer --- src/sofa/pbrpc/binary_rpc_request.cc | 6 +++--- src/sofa/pbrpc/rpc_client_impl.cc | 6 +++--- src/sofa/pbrpc/rpc_controller_impl.h | 8 ++++++-- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/src/sofa/pbrpc/binary_rpc_request.cc b/src/sofa/pbrpc/binary_rpc_request.cc index e3bfdad..04fd16e 100644 --- a/src/sofa/pbrpc/binary_rpc_request.cc +++ b/src/sofa/pbrpc/binary_rpc_request.cc @@ -209,14 +209,14 @@ ReadBufferPtr BinaryRpcRequest::AssembleSucceedResponse( return ReadBufferPtr(); } header.data_size = write_buffer.ByteCount() - header_pos - header_size - header.meta_size; - write_buffer.Append(cntl->GetResponseAttachBuffer()->ToString()); - int attach_size = write_buffer.ByteCount() - header_pos - header_size - header.meta_size - header.data_size; - header.message_size = header.meta_size + header.data_size + attach_size; + ReadBufferPtr response_attach_buffer = cntl->GetResponseAttachBuffer(); + header.message_size = header.meta_size + header.data_size + response_attach_buffer->TotalCount(); write_buffer.SetData(header_pos, reinterpret_cast(&header), header_size); ReadBufferPtr read_buffer(new ReadBuffer()); write_buffer.SwapOut(read_buffer.get()); + read_buffer->Append(response_attach_buffer.get()); return read_buffer; } diff --git a/src/sofa/pbrpc/rpc_client_impl.cc b/src/sofa/pbrpc/rpc_client_impl.cc index 7446aa9..b52e123 100644 --- a/src/sofa/pbrpc/rpc_client_impl.cc +++ b/src/sofa/pbrpc/rpc_client_impl.cc @@ -322,14 +322,14 @@ void RpcClientImpl::CallMethod(const google::protobuf::Message* request, return; } header.data_size = write_buffer.ByteCount() - header_pos - header_size - header.meta_size; - write_buffer.Append(cntl->GetRequestAttachBuffer()->ToString()); - int attach_size = write_buffer.ByteCount() - header_pos - header_size - header.meta_size - header.data_size; - header.message_size = header.meta_size + header.data_size + attach_size; + ReadBufferPtr request_attach_buffer = cntl->GetRequestAttachBuffer(); + header.message_size = header.meta_size + header.data_size + request_attach_buffer->TotalCount(); write_buffer.SetData(header_pos, reinterpret_cast(&header), header_size); ReadBufferPtr read_buffer(new ReadBuffer()); write_buffer.SwapOut(read_buffer.get()); + read_buffer->Append(request_attach_buffer.get()); cntl->SetRequestBuffer(read_buffer); // push callback diff --git a/src/sofa/pbrpc/rpc_controller_impl.h b/src/sofa/pbrpc/rpc_controller_impl.h index 1b723ef..2d93f9c 100644 --- a/src/sofa/pbrpc/rpc_controller_impl.h +++ b/src/sofa/pbrpc/rpc_controller_impl.h @@ -335,21 +335,25 @@ class RpcControllerImpl : public sofa::pbrpc::enable_shared_from_thisSerialize(_request_attach_buffer); } bool GetRequestAttachment(RpcAttachment* request_attachment) { + SCHECK(request_attachment); return request_attachment->Deserialize(_request_attach_buffer); } bool SetResponseAttachment(RpcAttachment* response_attachment) { + SCHECK(response_attachment); return response_attachment->Serialize(_response_attach_buffer); } bool GetResponseAttachment(RpcAttachment* response_attachment) { + SCHECK(response_attachment); return response_attachment->Deserialize(_response_attach_buffer);; } @@ -358,7 +362,7 @@ class RpcControllerImpl : public sofa::pbrpc::enable_shared_from_this