我的第一个微服务系列（五）：使用开源服务容错处理库Polly来跨服务调用

　　在微服务中，我们把系统拆分成粒度更小的服务，而服务与服务之间的调用就越频繁，伴随之网络故障、依赖服务崩溃、超时、服务器内存与CPU等问题也无法避免。所以在进行系统设计的时候要以“Design For Failure”为指导原则。把一些边缘场景以及服务之间的调用发生的异常和超时当成一定会发生的情况来预先进行处理。

　　得益于.Net社区的日渐活跃，Polly已经为我们实现了服务容错的大部分功能。Polly是一个.NET弹性和瞬态故障处理库，允许开发人员以流畅和线程安全的方式来表示重试，熔断，超时，舱壁隔离和回退等策略。在Polly中，对这些服务容错模式分为两类：错误处理fault handling ：重试、熔断、回退 ； 弹性应变resilience：超时、舱壁、缓存。错误处理是当错误已经发生时，防止由于该错误对整个系统造成更坏的影响而设置。而弹性应变，则在是错误发生前，针对有可能发生错误的地方进行预先处理，从而达到保护整个系统的目地。

　　关于Polly更详细的资料和用法可以参考本篇文章最后的参考资料，这里我们只讲在我的第一个微服务系列中项目是如何使用Polly的。

　　为了在项目中实现共用，新建项目Resilience类库，参照 eShopOnContainer 定义自己的IHttpClient，其主要提供Get、Post、Put、Delete方法。

using System;
using System.Collections.Generic;
using System.Net.Http;
using System.Threading.Tasks;

namespace Resilience
{
    public interface IHttpClient
    {
        Task<string> GetStringAsync(string url, string authorizationToken = null, string authorizationMethod = "Bearer");

        Task<HttpResponseMessage> PostAsync<T>(string url, T item, string authorizationToken=null, 
            string requestId = null, string authorizationMethod = "Bearer");

        Task<HttpResponseMessage> PostAsync(string url, Dictionary<string,string> form, string authorizationToken=null,
            string requestId = null, string authorizationMethod = "Bearer");


        Task<HttpResponseMessage> PutAsync<T>(string url, T item, string authorizationToken = null,
            string requestId = null, string authorizationMethod = "Bearer");
    }
}

　　在项目中引用Polly 

Install-Package Polly

　　创建实现IHttpClient的类ResilienceHttpClient

using System;
using System.Collections.Generic;
using System.Net.Http;
using System.Text;
using System.Threading.Tasks;
using Polly;
using System.Collections.Concurrent;
using Polly.Wrap;
using Microsoft.Extensions.Logging;
using Microsoft.AspNetCore.Http;
using System.Linq;
using Newtonsoft.Json;
using zipkin4net.Transport.Http;

namespace Resilience
{
    /// <summary>
    /// Description: ResilienceHttpClient
    /// Author: Jesen
    /// Version: 1.0
    /// Date: 2019/7/30 22:32:34
    /// </summary>
    public class ResilienceHttpClient : IHttpClient
    {
        private readonly HttpClient _httpClient;
        // 根据url origin 去创建 policy
        private readonly Func<string, IEnumerable<AsyncPolicy>> _policyCreator;
        //把policy打包成组合policy wraper，进行本地缓存
        private readonly ConcurrentDictionary<string, AsyncPolicyWrap> _policyWrappers;
        private readonly ILogger<ResilienceHttpClient> _logger;
        private IHttpContextAccessor _httpContextAccessor;

        public ResilienceHttpClient(string applicationName,Func<string, IEnumerable<AsyncPolicy>> policyCreator
            //, ConcurrentDictionary<string, AsyncPolicy> policyWrap
            , ILogger<ResilienceHttpClient> logger
            , IHttpContextAccessor httpContextAccessor)
        {
            _policyCreator = policyCreator;
            _httpClient = new HttpClient(new TracingHandler(applicationName));
            _policyWrappers = new ConcurrentDictionary<string, AsyncPolicyWrap>();
            _logger = logger;
            _httpContextAccessor = httpContextAccessor;
        }

        public async Task<HttpResponseMessage> PostAsync<T>(string url, T item, string authorizationToken=null, string requestId = null, string authorizationMethod = "Bearer")
        {
            HttpContent httpContent = new StringContent(JsonConvert.SerializeObject(item), System.Text.Encoding.UTF8, "application/json");
            return await DoPostAsync(HttpMethod.Post,url, httpContent, authorizationToken, requestId, authorizationMethod);
        }

        public async Task<HttpResponseMessage> PostAsync(string url, Dictionary<string, string> form, string authorizationToken=null, string requestId = null, string authorizationMethod = "Bearer")
        {
            HttpContent httpContent = new FormUrlEncodedContent(form);
            return await DoPostAsync(HttpMethod.Post, url, httpContent, authorizationToken, requestId, authorizationMethod);
        }

        private Task<HttpResponseMessage> DoPostAsync(HttpMethod method,string url,HttpContent httpContent, string authorizationToken, string requestId = null, string authorizationMethod = "Bearer")
        {
            if(method != HttpMethod.Post && method != HttpMethod.Put)
            {
                throw new ArgumentException("Value must be either post or put.", nameof(method));
            }

            var origin = GetOriginFromUri(url);
            return HttpInvoker(origin, async (context) => {

                var requestMessage = CreateHttpRequestMessage(method, url, httpContent);

                SetAuthorizationHeader(requestMessage);

                if(authorizationToken != null)
                {
                    requestMessage.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue(authorizationMethod, authorizationToken);
                }

                if(requestId != null)
                {
                    requestMessage.Headers.Add("x-requestid", requestId);
                }

                var response = await _httpClient.SendAsync(requestMessage);
                if(response.StatusCode == System.Net.HttpStatusCode.InternalServerError)
                {
                    throw new HttpRequestException();
                }
                return response;
            });
        }

        private HttpRequestMessage CreateHttpRequestMessage(HttpMethod method,string url,HttpContent httpContent)
        {
            return new HttpRequestMessage(method, url) { Content = httpContent };
        }

        private async Task<T> HttpInvoker<T>(string origin,Func<Context,Task<T>> action)
        {
            var normalizedOrigin = NormalizeOrigin(origin);

            if(!_policyWrappers.TryGetValue(normalizedOrigin,out AsyncPolicyWrap policyWrap))
            {
                policyWrap = Policy.WrapAsync(_policyCreator(normalizedOrigin).ToArray());
                _policyWrappers.TryAdd(normalizedOrigin, policyWrap);
            }

            return await policyWrap.ExecuteAsync(action, new Context(normalizedOrigin));
        }

        private static string NormalizeOrigin(string origin)
        {
            return origin?.Trim().ToLower();
        }

        private static string GetOriginFromUri(string uri)
        {
            var url = new Uri(uri);

            var origin = $"{url.Scheme}://{url.DnsSafeHost}:{url.Port}";

            return origin;
        }

        private void SetAuthorizationHeader(HttpRequestMessage requestMessage)
        {
            var authorizationHeader = _httpContextAccessor.HttpContext.Request.Headers["Authorization"];
            if (!string.IsNullOrEmpty(authorizationHeader))
            {
                requestMessage.Headers.Add("Authorization", new List<string> { authorizationHeader });
            }
        }

        public async Task<string> GetStringAsync(string url, string authorizationToken = null, string authorizationMethod = "Bearer")
        {
            var origin = GetOriginFromUri(url);

            return await HttpInvoker(origin, async (context) => {
                var requestMessage = new HttpRequestMessage(HttpMethod.Get, url);

                SetAuthorizationHeader(requestMessage);

                if(authorizationToken != null)
                {
                    requestMessage.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue(authorizationMethod, authorizationToken);
                }

                var response = await _httpClient.SendAsync(requestMessage);

                if(response.StatusCode == System.Net.HttpStatusCode.InternalServerError)
                {
                    throw new HttpRequestException();
                }

                if (!response.IsSuccessStatusCode)
                {
                    return null;
                }

                return await response.Content.ReadAsStringAsync();
            });
        }

        public async Task<HttpResponseMessage> PutAsync<T>(string url, T item, string authorizationToken = null, string requestId = null, string authorizationMethod = "Bearer")
        {
            HttpContent httpContent = new StringContent(JsonConvert.SerializeObject(item), System.Text.Encoding.UTF8, "application/json");
            return await DoPostAsync(HttpMethod.Put, url, httpContent, authorizationToken, requestId, authorizationMethod);
        }
    }
}

　　前面讲到要在User.Identity项目中调用User.Api，所以我们新建IUserService和UserService

using System.Threading.Tasks;
using User.Identity.Dtos;

namespace User.Identity.Services
{
    public interface IUserService
    {
        /// <summary>
        /// 检查手机号是否已注册，如果没有注册的话就注册一个用户
        /// </summary>
        /// <param name="phone"></param>
        Task<UserInfo> CheckOrCreateAsync(string phone);
    }
}

using DnsClient;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using Newtonsoft.Json;
using Resilience;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Threading.Tasks;
using User.Identity.Dtos;

namespace User.Identity.Services
{
    public class UserService : IUserService
    {
        private IHttpClient _httpClient;

        private string _userServiceUrl;

        private ILogger<UserService> _logger;

        public UserService(IHttpClient httpClient,
            IOptions<ServiceDiscoveryOptions> options,
            IDnsQuery dnsQuery,
            ILogger<UserService> logger)
        {
            _httpClient = httpClient;
            var address = dnsQuery.ResolveService("service.consul", options.Value.UserServiceName);
            var addressList = address.First().AddressList;
            var host = addressList.Any() ? addressList.First().ToString() : address.First().HostName;
            var port = address.First().Port;
            _userServiceUrl = $"http://{host}:{port}";

            _logger = logger;

        }

        public async Task<UserInfo> CheckOrCreateAsync(string phone)
        {
            _logger.LogTrace($"Enter into CheckOrCreate:{phone}");
            var form = new Dictionary<string, string>
            {
                {"phone",phone }
            };

            try
            {
                var response = await _httpClient.PostAsync(_userServiceUrl + "/api/user/check-or-create", form);
                if (response.StatusCode == HttpStatusCode.OK)
                {
                    var result = await response.Content.ReadAsStringAsync();

                    var userInfo = JsonConvert.DeserializeObject<UserInfo>(result);

                    _logger.LogTrace($"Completed CheckOrCreate with UserId:{userInfo.Id}");
                    return userInfo;
                }
            }
            catch(Exception ex)
            {
                _logger.LogError($"{nameof(CheckOrCreateAsync) }在重试之后失败，{ex.Message}{ex.StackTrace}");
            }
            

            return null;
        }
    }
}

　　添加Resilience类库，新建ResilienceClientFactory类来创建Polly的Policy。

using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Linq;
using System.Net.Http;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Http;
using Microsoft.Extensions.Logging;
using Polly;
using Polly.Wrap;
using Resilience;

namespace User.Identity.Infrastructure
{
    public class ResilienceClientFactory
    {
        private ILogger<ResilienceHttpClient> _logger;
        private IHttpContextAccessor _httpContextAccessor;
        // 重试次数
        private int _retryCount;
        //熔断之前允许的异常次数
        private int _exceptionCountAllowBeforeBreaking;

        public ResilienceClientFactory(ILogger<ResilienceHttpClient> logger
            , IHttpContextAccessor httpContextAccessor
            ,int retryCount,int exceptionCountAllowBeforeBreaking)
        {
            _logger = logger;
            _httpContextAccessor = httpContextAccessor;
            _retryCount = retryCount;
            _exceptionCountAllowBeforeBreaking = exceptionCountAllowBeforeBreaking;
        }

        public ResilienceHttpClient GetResilienceHttpClient =>
            new ResilienceHttpClient("IdentityServer", origin => CreatePolicy(origin), _logger, _httpContextAccessor);


        private AsyncPolicy[] CreatePolicy(string origin)
        {
            return new AsyncPolicy[] {
                Policy.Handle<HttpRequestException>()
                .WaitAndRetryAsync(
                    _retryCount,
                    retryAttempt => TimeSpan.FromSeconds( Math.Pow(2,retryAttempt)),
                    (exception, timespan, retryCount, context) =>
                    {
                        var msg = $"第 {retryCount} 次重试 " +
                        $"of {context.PolicyKey} " +
                        $"at {context.OperationKey} " +
                        $"due to : {exception} .";

                        _logger.LogWarning(msg);
                        _logger.LogDebug(msg);
                    }),

                Policy.Handle<HttpRequestException>()
                .CircuitBreakerAsync(
                    _exceptionCountAllowBeforeBreaking,
                    TimeSpan.FromMinutes(1),
                    (exception,duration)=>{
                        _logger.LogWarning("熔断器打开");
                    },()=>{
                        _logger.LogWarning("熔断器关闭");
                    })
            };
        }
    }
}

　　由于User.Api注册在Consul上，User.Identity需要通过Consul发现User.Api，需要引入DnsClient包

Install-Package DnsClient

　　和User.Api一样需要添加Consul的配置

  "ServiceDiscovery": {
    "UserServiceName": "userapi",
    "Consul": {
      "HttpEndpoint": "http://127.0.0.1:8500",
      "DnsEndpoint": {
        "Address": "127.0.0.1",
        "Port": 8600
      }
    }
  }

namespace User.Identity.Dtos
{
    public class ServiceDiscoveryOptions
    {
        public string UserServiceName { get; set; }

        public ConsulOptions Consul { get; set; }
    }
}

namespace User.Identity.Dtos
{
    public class DnsEndpoint
    {
        public string Address { get; set; }

        public int Port { get; set; }

        public IPEndPoint ToIPEndPoint()
        {
            return new IPEndPoint(IPAddress.Parse(Address), Port);
        }
    }
}

namespace User.Identity.Dtos
{
    public class ConsulOptions
    {
        public string HttpEndpoint { get; set; }

        public DnsEndpoint DnsEndpoint { get; set; }
    }
}

　　最后需要在Startup中注入这些服务

services.AddScoped<IAuthCodeService, TestAuthCodeService>()
    .AddScoped<IUserService, UserService>();

//注册全局单例的ResilienceClientFactory
services.AddSingleton(typeof(ResilienceClientFactory), sp => {
    var logger = sp.GetRequiredService<ILogger<ResilienceHttpClient>>();
    var httpContextAccesser = sp.GetRequiredService<IHttpContextAccessor>();
    var retryCount = 5;
    var exceptionCountAllowBeforeBreaking = 5;
    return new ResilienceClientFactory(logger,httpContextAccesser,retryCount,exceptionCountAllowBeforeBreaking);
});

//注册全局单例IHttpClient
//services.AddSingleton(new HttpClient());
services.AddSingleton<IHttpClient>(sp =>
{
   return sp.GetRequiredService<ResilienceClientFactory>().GetResilienceHttpClient;
});

//配置文件注入
services.AddOptions();
services.Configure<ServiceDiscoveryOptions>(Configuration.GetSection("ServiceDiscovery"));

//从consul获取IP
services.AddSingleton<IDnsQuery>(p =>
{
    var serviceConfiguration = p.GetRequiredService<IOptions<ServiceDiscoveryOptions>>().Value;
    return new LookupClient(serviceConfiguration.Consul.DnsEndpoint.ToIPEndPoint());
});

　　到这里，User.Identity调用User.Api实现认证就完成了。

参考资料：

　　关于服务容错模式可以参考美团技术团队的文章：https://tech.meituan.com/2016/11/11/service-fault-tolerant-pattern.html 。

　　关于Polly的官方文档：https://github.com/App-vNext/Polly 。

　　中文翻译文档可参考jesse的文章：https://www.cnblogs.com/jesse2013/archive/2018/03/29/8647581.html 。