gRPC and Protocol Buffers

Protobuf Basics

Protocol Buffers (protobuf) is a language-neutral binary serialization format. Define schemas in .proto files.

syntax = "proto3";
package myapp.v1;
option go_package = "github.com/myorg/myapp/gen/myappv1";

message User {
  string id = 1;
  string name = 2;
  string email = 3;
  int64 created_at = 4;
  repeated string roles = 5;       // ordered list
  Address address = 6;             // nested message
  UserStatus status = 7;           // enum
  map<string, string> metadata = 8; // key-value pairs
}

message Address {
  string street = 1;
  string city = 2;
  string country = 3;
}

enum UserStatus {
  USER_STATUS_UNSPECIFIED = 0;  // required zero value
  USER_STATUS_ACTIVE = 1;
  USER_STATUS_INACTIVE = 2;
}

Scalar Types

Protobuf	Go	Python	Node	Java
double	float64	float	number	double
float	float32	float	number	float
int32	int32	int	number	int
int64	int64	int	number	long
bool	bool	bool	boolean	boolean
string	string	str	string	String
bytes	[]byte	bytes	Buffer	ByteString

Field numbers 1-15 use one byte on the wire -- reserve them for frequent fields. Never reuse field numbers. Use wrapper types (google.protobuf.StringValue) to distinguish unset from default.

Service Definitions

service UserService {
  rpc GetUser(GetUserRequest) returns (GetUserResponse);           // unary
  rpc ListUsers(ListUsersRequest) returns (stream User);           // server streaming
  rpc UploadUsers(stream User) returns (UploadUsersResponse);      // client streaming
  rpc SyncUsers(stream SyncRequest) returns (stream SyncResponse); // bidirectional
}

message GetUserRequest { string id = 1; }
message GetUserResponse { User user = 1; }
message ListUsersRequest { int32 page_size = 1; string page_token = 2; }

Code Generation with protoc

# Install: brew install protobuf (macOS) / apt install protobuf-compiler (Linux)

# Go
protoc --go_out=. --go-grpc_out=. proto/user.proto

# Python
python -m grpc_tools.protoc -I. --python_out=. --grpc_python_out=. proto/user.proto

# Node.js
grpc_tools_node_protoc --js_out=import_style=commonjs:. --grpc_out=. proto/user.proto

buf Tool (Modern protoc Replacement)

# buf.yaml
version: v2
modules:
  - path: proto
lint:
  use: [STANDARD]
breaking:
  use: [FILE]

# buf.gen.yaml
version: v2
plugins:
  - remote: buf.build/protocolbuffers/go
    out: gen/go
    opt: paths=source_relative
  - remote: buf.build/grpc/go
    out: gen/go
    opt: paths=source_relative

buf lint                                    # Lint proto files
buf breaking --against '.git#branch=main'   # Detect breaking changes
buf generate                                # Generate code

buf enforces style: package names match directories, enum zero values end in _UNSPECIFIED, service names end in Service.

gRPC Patterns

• Unary: one request, one response. Use for CRUD, lookups.
• Server streaming: one request, stream of responses. Use for large result sets, feeds.
• Client streaming: stream of requests, one response. Use for uploads, batch ingestion.
• Bidirectional: both sides stream independently. Use for chat, real-time sync.

gRPC in Node.js

const grpc = require("@grpc/grpc-js");
const protoLoader = require("@grpc/proto-loader");
const packageDef = protoLoader.loadSync("proto/user.proto", {
  keepCase: true, longs: String, enums: String, defaults: true, oneofs: true,
});
const proto = grpc.loadPackageDefinition(packageDef).myapp.v1;

// Server
function getUser(call, callback) {
  callback(null, { user: { id: call.request.id, name: "Alice" } });
}
function listUsers(call) {
  for (let i = 0; i < 10; i++) call.write({ id: String(i), name: `User ${i}` });
  call.end();
}
const server = new grpc.Server();
server.addService(proto.UserService.service, { getUser, listUsers });
server.bindAsync("0.0.0.0:50051", grpc.ServerCredentials.createInsecure(), () => {});

// Client
const client = new proto.UserService("localhost:50051", grpc.credentials.createInsecure());
client.getUser({ id: "123" }, (err, res) => {
  if (err) return console.error(`Code: ${err.code}, Message: ${err.details}`);
  console.log(res.user);
});
const stream = client.listUsers({ page_size: 10 });
stream.on("data", (user) => console.log(user));
stream.on("end", () => console.log("done"));

gRPC in Python

import grpc
from concurrent import futures
import user_pb2, user_pb2_grpc

class UserServiceServicer(user_pb2_grpc.UserServiceServicer):
    def GetUser(self, request, context):
        if not request.id:
            context.abort(grpc.StatusCode.INVALID_ARGUMENT, "id required")
        return user_pb2.GetUserResponse(user=user_pb2.User(id=request.id, name="Alice"))

    def ListUsers(self, request, context):
        for i in range(10):
            yield user_pb2.User(id=str(i), name=f"User {i}")

def serve():
    server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
    user_pb2_grpc.add_UserServiceServicer_to_server(UserServiceServicer(), server)
    server.add_insecure_port("[::]:50051")
    server.start()
    server.wait_for_termination()

# Client
channel = grpc.insecure_channel("localhost:50051")
stub = user_pb2_grpc.UserServiceStub(channel)
try:
    response = stub.GetUser(user_pb2.GetUserRequest(id="123"), timeout=5)
except grpc.RpcError as e:
    print(f"Code: {e.code()}, Details: {e.details()}")
for user in stub.ListUsers(user_pb2.ListUsersRequest(page_size=10)):
    print(user)

gRPC in Go

package main

import (
    "context"
    "fmt"
    "log"
    "net"
    "google.golang.org/grpc"
    "google.golang.org/grpc/codes"
    "google.golang.org/grpc/status"
    pb "github.com/myorg/myapp/gen/myappv1"
)

type server struct{ pb.UnimplementedUserServiceServer }

func (s *server) GetUser(ctx context.Context, req *pb.GetUserRequest) (*pb.GetUserResponse, error) {
    if req.Id == "" {
        return nil, status.Error(codes.InvalidArgument, "id required")
    }
    return &pb.GetUserResponse{User: &pb.User{Id: req.Id, Name: "Alice"}}, nil
}

func (s *server) ListUsers(req *pb.ListUsersRequest, stream pb.UserService_ListUsersServer) error {
    for i := 0; i < 10; i++ {
        if err := stream.Send(&pb.User{Id: fmt.Sprintf("%d", i)}); err != nil {
            return err
        }
    }
    return nil
}

func main() {
    lis, _ := net.Listen("tcp", ":50051")
    s := grpc.NewServer()
    pb.RegisterUserServiceServer(s, &server{})
    log.Fatal(s.Serve(lis))
}

Go client:

conn, _ := grpc.Dial("localhost:50051", grpc.WithTransportCredentials(insecure.NewCredentials()))
defer conn.Close()
client := pb.NewUserServiceClient(conn)
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
resp, err := client.GetUser(ctx, &pb.GetUserRequest{Id: "123"})
if err != nil {
    st, _ := status.FromError(err)
    log.Printf("Code: %s, Message: %s", st.Code(), st.Message())
}

grpcurl for Testing

# Install: brew install grpcurl
# Server must have reflection enabled, or use -import-path/-proto flags

grpcurl -plaintext localhost:50051 list                              # list services
grpcurl -plaintext localhost:50051 describe myapp.v1.UserService     # describe service
grpcurl -plaintext -d '{"id":"123"}' localhost:50051 myapp.v1.UserService/GetUser
grpcurl -plaintext -H 'authorization: Bearer tok' -d '{"id":"123"}' \
  localhost:50051 myapp.v1.UserService/GetUser                       # with metadata
grpcurl -plaintext -import-path ./proto -proto user.proto \
  -d '{"page_size":10}' localhost:50051 myapp.v1.UserService/ListUsers

Enable reflection: Go reflection.Register(s), Python grpc_reflection.v1alpha.reflection.enable_server_reflection(names, server), Node @grpc/reflection.

Error Handling: gRPC Status Codes

Code	Num	Use For
OK	0	Success
CANCELLED	1	Client cancelled
INVALID_ARGUMENT	3	Bad input
DEADLINE_EXCEEDED	4	Timeout
NOT_FOUND	5	Resource missing
ALREADY_EXISTS	6	Conflict on create
PERMISSION_DENIED	7	Lacks permission
RESOURCE_EXHAUSTED	8	Rate limit / quota
FAILED_PRECONDITION	9	System not in required state
UNIMPLEMENTED	12	Method not implemented
INTERNAL	13	Unexpected server error
UNAVAILABLE	14	Transient -- client should retry
UNAUTHENTICATED	16	No valid credentials

Do not use UNKNOWN as a catch-all. Return UNAVAILABLE for transient failures so clients retry.

Metadata

Metadata is the gRPC equivalent of HTTP headers. Keys are strings; binary values use keys ending in -bin.

// Go: read incoming metadata
md, _ := metadata.FromIncomingContext(ctx)
token := md.Get("authorization")
// Go: send outgoing metadata
ctx = metadata.AppendToOutgoingContext(ctx, "authorization", "Bearer tok")

# Python: read
token = context.invocation_metadata()  # list of (key, value) tuples
# Python: send
stub.GetUser(request, metadata=[("authorization", "Bearer tok")])

Deadlines and Timeouts

Always set deadlines. A missing deadline holds resources indefinitely. Propagate the incoming context in chained calls so the overall deadline is respected.

ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()

response = stub.GetUser(request, timeout=5)

const deadline = new Date(Date.now() + 5000);
client.getUser({ id: "123" }, { deadline }, callback);

Health Checking

Use the standard grpc.health.v1.Health protocol. Do not invent your own.

import "google.golang.org/grpc/health"
import healthpb "google.golang.org/grpc/health/grpc_health_v1"
hs := health.NewServer()
healthpb.RegisterHealthServer(s, hs)
hs.SetServingStatus("myapp.v1.UserService", healthpb.HealthCheckResponse_SERVING)

grpcurl -plaintext localhost:50051 grpc.health.v1.Health/Check

Common Patterns

Microservice API Structure

proto/myapp/
  user/v1/user.proto
  user/v1/user_service.proto
  order/v1/order.proto
  order/v1/order_service.proto

Keep request/response messages with the service. Share domain messages via imports. Use v1 package suffix for versioning.

Streaming Data Feed

service PriceFeed {
  rpc Subscribe(SubscribeRequest) returns (stream PriceUpdate);
}
message SubscribeRequest { repeated string symbols = 1; }
message PriceUpdate { string symbol = 1; double price = 2; int64 timestamp = 3; }

Client reconnects on UNAVAILABLE with exponential backoff. Use keepalive settings to detect dead connections.

gRPC vs REST

Concern	gRPC	REST
Serialization	Protobuf (binary, compact)	JSON (text, readable)
Schema	Required (.proto)	Optional (OpenAPI)
Streaming	Native (4 patterns)	SSE / WebSocket workarounds
Browser	Needs grpc-web proxy	Native
Tooling	protoc/buf + plugins	curl, any HTTP client
Performance	Lower latency, smaller payload	Higher latency, larger payload
Code gen	Built-in, strongly typed	Varies
Load balancing	L7 HTTP/2-aware required	Any HTTP LB

Use gRPC for internal service-to-service calls where performance, type safety, and streaming matter. Use REST for public APIs and browser clients. Both coexist well -- REST at the edge, gRPC internally.