Go Grpc部署到 k8s【端口共享】

书接上文Go Grpc部署到 k8s【负载均衡】 grpc server端我们暴露了9090和8080端口，这次我们的http服务用iris,并且绑定到9090端口。

GO文件

服务端代码main.go：

package main

import (
    "context"
    "fmt"
    pb "grpcdemo/protos"
    "net"

    "github.com/kataras/iris/v12"
    "github.com/kataras/iris/v12/middleware/logger"
    "github.com/kataras/iris/v12/middleware/recover"
    "github.com/soheilhy/cmux"
    "google.golang.org/grpc"
)

func main() {
    l, err := net.Listen("tcp", ":9090")
    fmt.Println(err)
    m := cmux.New(l)

    // grpc
    grpcL := m.MatchWithWriters(cmux.HTTP2MatchHeaderFieldSendSettings("content-type", "application/grpc"))
    grpcServer := grpc.NewServer()                  // 创建GRPC
    pb.RegisterGreeterServer(grpcServer, &server{}) // 在GRPC服务端注册服务

    go grpcServer.Serve(grpcL)

    // http
    app := iris.New()
    app.Use(recover.New())
    app.Use(logger.New())

    app.Handle("GET", "/", func(ctx iris.Context) {
        ctx.WriteString("pong")
    })

    go func() {
        httpL := m.Match(cmux.HTTP1Fast())
        app.Run(iris.Listener(httpL))
    }()

    // Start serving!
    m.Serve()
}

type server struct{}

func NewServer() *server {
    return &server{}
}

func (s *server) SayHello(ctx context.Context, in *pb.HelloRequest) (*pb.HelloReply, error) {
    msg := "Resuest By:" + in.Name + " Response By :" + LocalIp()
    fmt.Println("GRPC Send: ", msg)
    return &pb.HelloReply{Message: msg}, nil
}

func LocalIp() string {
    addrs, _ := net.InterfaceAddrs()
    var ip string = "localhost"
    for _, address := range addrs {
        if ipnet, ok := address.(*net.IPNet); ok && !ipnet.IP.IsLoopback() {
            if ipnet.IP.To4() != nil {
                ip = ipnet.IP.String()
            }
        }
    }
    return ip
}

服务端代码deploy.yaml：

apiVersion: apps/v1
kind: Deployment
metadata:
  name: grpcserver
  namespace: go
  labels:
    app: grpcserver
    version: v1
spec:
  replicas: 1
  minReadySeconds: 10 
  selector:
    matchLabels:
      app: grpcserver
      version: v1
  template:
    metadata:
      labels:
        app: grpcserver
        version: v1
    spec:
      imagePullSecrets:
      - name: regsecret
      containers:
      - name: grpcserver
        image: 192.168.100.30:8080/go/grpcserver:2022
        ports:
        - containerPort: 9090
        imagePullPolicy: Always

---
apiVersion: v1 
kind: Service 
metadata:
  name: grpcserver
  namespace: go 
  labels:
    app: grpcserver
    version: v1
spec:
  ports:
    - port: 9090 
      targetPort: 9090 
      name: grpcserver
      protocol: TCP
  selector:
    app: grpcserver

服务端build.sh

#!/bin/bash
#cd $WORKSPACE
export GOPROXY=https://goproxy.io
 
 #根据 go.mod 文件来处理依赖关系。
go mod tidy
 
# linux环境编译
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o main
 
# 构建docker镜像，项目中需要在当前目录下有dockerfile，否则构建失败

docker build -t grpcserver .
docker tag  grpcserver 192.168.100.30:8080/go/grpcserver:2022

docker login -u admin -p '123456' 192.168.100.30:8080
docker push 192.168.100.30:8080/go/grpcserver
 
docker rmi  grpcserver
docker rmi 192.168.100.30:8080/go/grpcserver:2022

服务端Dockerfile

FROM golang:1.15.6

RUN mkdir -p /app

WORKDIR /app
 
ADD main /app/main

EXPOSE 9090
 
CMD ["./main"]

客户端 default.go

package controllers

import (
    "context"
    "fmt"
    pb "grpcclient/protos"
    "io/ioutil"
    "net"
    "net/http"
    "time"

    "github.com/astaxie/beego"
    "google.golang.org/grpc"
)

type MainController struct {
    beego.Controller
}

func (c *MainController) Get() {
    address := "grpcserver:9090"
    // GRPC
    conn, _ := grpc.Dial(address, grpc.WithInsecure())
    defer conn.Close()
    grpcClient := pb.NewGreeterClient(conn)
    req := pb.HelloRequest{Name: "gavin_" + LocalIp()}

    res, err := grpcClient.SayHello(context.Background(), &req)
    if err != nil {
        msg := fmt.Sprintf("grpc client  client.SayHello  has err:%v
", err)
        c.Ctx.WriteString(msg)
        return
    }

    //http
    var httpClient = http.Client{
        Timeout: 10 * time.Second,
    }
    resp, err := httpClient.Get("http://" + address)
    if err != nil {
        msg := fmt.Sprintf("http get  has err:%v
", err)
        c.Ctx.WriteString(msg)
        return
    }

    defer resp.Body.Close()
    bytes, _ := ioutil.ReadAll(resp.Body)
    message := fmt.Sprintf("GRPC Clinet Received:%v ,Http Get:%v ", res.Message, string(bytes))
    //
    c.Ctx.WriteString(message)
}

func LocalIp() string {
    addrs, _ := net.InterfaceAddrs()
    var ip string = "localhost"
    for _, address := range addrs {
        if ipnet, ok := address.(*net.IPNet); ok && !ipnet.IP.IsLoopback() {
            if ipnet.IP.To4() != nil {
                ip = ipnet.IP.String()
            }
        }
    }
    return ip
}

客户端 deploy.yaml

apiVersion: apps/v1
kind: Deployment
metadata:
  name: grpcclient
  namespace: go
  labels:
    app: grpcclient
    version: v1
spec:
  replicas: 1
  minReadySeconds: 10 
  selector:
    matchLabels:
      app: grpcclient
      version: v1
  template:
    metadata:
      labels:
        app: grpcclient
        version: v1
    spec:
      imagePullSecrets:
      - name: regsecret
      containers:
      - name: grpcclient
        image: 192.168.100.30:8080/go/grpcclient:2022
        ports:
        - containerPort: 8080
        imagePullPolicy: Always

---

apiVersion: v1 
kind: Service 
metadata:
  name: grpcclient
  namespace: go 
  labels:
    app: grpcclient
    version: v1
spec:
  type: ClusterIP
  ports:
    - port: 8080 
      targetPort: 8080 
      protocol: TCP
  selector:
    app: grpcclient

客户端 build.sh

#!/bin/bash
#cd $WORKSPACE
export GOPROXY=https://goproxy.io
 
 #根据 go.mod 文件来处理依赖关系。
go mod tidy
 
# linux环境编译
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o main
 
# 构建docker镜像，项目中需要在当前目录下有dockerfile，否则构建失败

docker build -t grpcserver .
docker tag  grpcserver 192.168.100.30:8080/go/grpcserver:2022

docker login -u admin -p '123456' 192.168.100.30:8080
docker push 192.168.100.30:8080/go/grpcserver
 
docker rmi  grpcserver
docker rmi 192.168.100.30:8080/go/grpcserver:2022

客户端 Dockerfile

FROM golang:1.15.6

RUN mkdir -p /app
RUN mkdir -p /app/conf
RUN mkdir -p /app/logs

WORKDIR /app
 
ADD main /app/main

EXPOSE 8080
 
CMD ["./main"]

Deploy

默认的k8s 部署运行结果：

Istio部署

​

需要先启用拦截 kubectl label namespace go istio-injection=enabled

Istio想要启用的话 就必须修改svc的 端口名称如：

 但是这样http返回失败：

 所以用一般的方式是不能能搞定grpc和http协议.我后面尝试了很多方法 还是没搞定。。。。，估计只能放弃了

截图中的name 建议不要用 http 和http2 以及grpc-  打头， 他们有特别的意义， 同样gateway那边的name也要注意， istio会有helth检测【比如我的name以http或者http2 可以过健康检测， 但是grpc没法用， 如果用grpc作为name ， 健康检测都过不了， 当然实际 http 和egrpc是不通的】

记录以下我的错误记录吧， 上面的name,我试过grpcserver,grpc-port,http-port，http2,test[http2 和test是我gateway定义的]，结果pod启动不起来错误信息：

Readiness probe failed: Get "http://10.244.1.128:15021/healthz/ready": dial tcp 10.244.1.128:15021: connect: connection refused

 把name改为http后pod可以启动【我先说以下的图是虚假的吧】

 

我的gateway.yaml如下：【9090是我增加的】

apiVersion: networking.istio.io/v1alpha3
kind: Gateway
metadata:
  name: grpcserver-gateway
  namespace: go
spec:
  selector:
    istio: ingressgateway # use istio default controller
  servers:
  - port:
      number: 9090  
      name: test
      protocol: TCP
    hosts:
    - "*"
---
apiVersion: networking.istio.io/v1alpha3
kind: VirtualService
metadata:
  name: grpcserver
  namespace: go
spec:
  hosts:
  - "*"
  gateways:
  - grpcserver-gateway
  http:
  - match:
    - port: 9090
    route:
    - destination:
        host: grpcserver.go.svc.cluster.local
        port:
          number: 9090

查看地址信息：

export INGRESS_PORT=$(kubectl -n istio-system get service istio-ingressgateway -o jsonpath='{.spec.ports[?(@.name=="test")].nodePort}')
export INGRESS_HOST=$(kubectl get po -l istio=ingressgateway -n istio-system -o 'jsonpath={.items[0].status.hostIP}')
echo $INGRESS_HOST:$INGRESS_PORT

结果是失败的

 为了证明是是istio的问题，我的gateway.yaml文件不变，新将deploy2.yaml文件【沿用上文的grpc服务，8080和9090端口】：

apiVersion: v1
kind: Service
metadata:
  name: grpcserver
  namespace: go
  labels:
    app: grpcserver
spec:
  ports:
    - name: grpc-port
      port: 9090
    - name: http-port
      port: 8080
  selector:
    app: grpcserver    
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: grpcserver
  namespace: go
  labels:
    app: grpcserver
    version: v1
spec:
  replicas: 1
  selector:
    matchLabels:
      app: grpcserver
      version: v1
  template:
    metadata:
      labels:
        app: grpcserver
        version: v1
    spec:
      containers:
      - name: grpcserver
        image: 192.168.100.30:8080/go/grpcserver:2021
        imagePullPolicy: IfNotPresent
        ports:
          - containerPort: 8080
          - containerPort: 9090

gateway2.yaml文件如下：

apiVersion: networking.istio.io/v1alpha3
kind: Gateway
metadata:
  name: grpcserver-gateway
  namespace: go 
spec:
  selector:
    istio: ingressgateway # use istio default controller
  servers:
  - port:
      number: 80  
      name: http
      protocol: HTTP
    hosts:
    - "*"
  - port:
      number: 9090  
      name: test
      protocol: HTTP
    hosts:
    - "*"
---
apiVersion: networking.istio.io/v1alpha3
kind: VirtualService
metadata:
  name: grpcserver
  namespace: go 
spec:
  hosts:
  - "*"
  gateways:
  - grpcserver-gateway
  http:
  - match:
    - port: 80
    route:
    - destination:
        host: grpcserver.go.svc.cluster.local
        port:
          number: 8080
  - match:
    - port: 9090
    route:
    - destination:
        host: grpcserver.go.svc.cluster.local
        port: