C#开源磁盘/内存缓存引擎

C#开源磁盘/内存缓存引擎

前言

昨天写了个 《基于STSdb和fastJson的磁盘/内存缓存》，大家可以先看看。下午用到业务系统时候，觉得可以改进一下，昨晚想了一个晚上，刚才重新实现一下。

更新

1. 增加了对批量处理的支持，写操作速度提升5倍，读操作提升100倍

2. 增加了一个存储provider，可以选择不用STSdb做存储，而用物理文件/Dictionary。

3. 增加了对并发的支持

需求

业务系统用的是数据库，数据量大，部分只读或相对稳定业务查询复杂，每次页面加载都要花耗不少时间(不讨论异步)，觉得可以做一下高速缓存，譬如用nosql那种key/value快速存取结果

目的

提供一个简单易用的解决缓存方案，可以根据数据的大小缓存到内存或者磁盘。

实现

存取

方法1. 基于STSdb，提供高效的Key/Value存取，支持磁盘/内存，对Key无限制

方法2. 基于直接物理文件/Dictionary。Key必须是基本类型，譬如int/long/uint/ulong/DateTime/string等。

代码

代码比较简单，花了2个小时写的，很多情况没考虑，譬如磁盘空间/内存不足，自动回收过期缓存等，这些留给大家做家庭作业吧。另外，为了发布方便，STSdb和fastJson的代码都合并到一个项目里。

BaseCahce.cs

这是一个抽象基类，提供存取接口。

?

using System;

using System.Collections.Generic;

using System.Linq;

using System.Text;

 

namespace Com.SuperCache.Engine

{

    public abstract class BaseCache

    {

        protected internal const string KeyExpiration = "Expiration";

 

        public abstract void Add<K>(string Category, K Key, object Data);

        public abstract void Add<K, V>(string Category, IEnumerable<KeyValuePair<K, V>> Items, DateTime? ExpirationDate);

        public abstract void Add<K>(string Category, K Key, object Data, DateTime? ExpirationDate);

        public abstract List<KeyValuePair<K, V>> Get<K, V>(string Category, IEnumerable<K> Keys);

        public abstract V Get<K, V>(string Category, K Key);

    }

}

　　

CahceEngine.cs

主要调用缓存引擎

?

using System;

using System.Collections.Generic;

using System.Linq;

using System.Text;

using STSdb4.Database;

using fastJSON;

using System.IO;

 

namespace Com.SuperCache.Engine

{

    public enum CacheProviders

    {

        Default = 1,

        Raw = 2

    }

 

    public class CacheEngine

    {

        private BaseCache cacheProvider = null;

 

        public CacheEngine(string DataPath): this(CacheProviders.Default, DataPath)

        {

        }

 

        public CacheEngine(CacheProviders Provider, string DataPath)

        {

            switch (Provider)

            {

                case CacheProviders.Default:

                    cacheProvider = new STSdbCache(DataPath);

                    break;

                case CacheProviders.Raw:

                    cacheProvider = new RawCache(DataPath);

                    break;

                default:

                    break;

            }

        }

 

        public void Add<K>(string Category, K Key, object Data)

        {

            cacheProvider.Add<K>(Category, Key, Data);

        }

 

        public void Add<K, V>(string Category, IEnumerable<KeyValuePair<K, V>> Items, DateTime? ExpirationDate)

        {

            cacheProvider.Add<K, V>(Category, Items, ExpirationDate);

        }

 

        public void Add<K>(string Category, K Key, object Data, DateTime? ExpirationDate)

        {

            cacheProvider.Add<K>(Category, Key, Data, ExpirationDate);

        }

 

        public List<KeyValuePair<K, V>> Get<K, V>(string Category, IEnumerable<K> Keys)

        {

            return cacheProvider.Get<K, V>(Category, Keys);

        }

 

        public V Get<K, V>(string Category, K Key)

        {

            return cacheProvider.Get<K, V>(Category, Key);

        }

    }

}

　　

　　

STSdbCache.cs

STSdb存储引擎　　

?

using System;

using System.Collections.Generic;

using System.Linq;

using System.Text;

using STSdb4.Database;

using fastJSON;

using System.IO;

 

namespace Com.SuperCache.Engine

{

    public class STSdbCache : BaseCache

    {

        private string dataPath;

        private static IStorageEngine memoryInstance = null;

        private static object syncRoot = new object();

        private bool isMemory = false;

 

        public STSdbCache(string DataPath)

        {

            dataPath = DataPath;

            if (!dataPath.EndsWith(Path.DirectorySeparatorChar.ToString()))

                dataPath += Path.DirectorySeparatorChar;

 

            isMemory = string.IsNullOrEmpty(DataPath);

        }

 

        public override void Add<K>(string Category, K Key, object Data)

        {

            Add(Category, Key, Data, null);

        }

 

        private IStorageEngine Engine

        {

            get

            {

                if (isMemory)

                {

                    lock (syncRoot)

                    {

                        if (memoryInstance == null)

                            memoryInstance = STSdb.FromMemory();

                    }

                    return memoryInstance;

                }

                else

                    return STSdb.FromFile(GetFile(false), GetFile(true));

            }

        }

 

        private string GetExpirationTable(string Category)

        {

            return KeyExpiration + "_" + Category;

        }

 

        public override void Add<K, V>(string Category, IEnumerable<KeyValuePair<K, V>> Items, DateTime? ExpirationDate)

        {

            lock (syncRoot)

            {

                var engine = Engine;

                var table = engine.OpenXIndex<K, string>(Category);

                Items.ForEach(i =>

                    {

                        var key = i.Key;

                        var data = i.Value;

                        //will only serialize object other than string

                        var result = typeof(V) == typeof(string) ? data as string: JSON.Instance.ToJSON(data);

                        table[key] = result;

                        table.Flush();

 

                        //specify expiration

                        var expiration = engine.OpenXIndex<K, DateTime>(GetExpirationTable(Category));

                        //default 30 mins to expire from now

                        var expirationDate = ExpirationDate == null || ExpirationDate <= DateTime.Now ? DateTime.Now.AddMinutes(30) : (DateTime)ExpirationDate;

                        expiration[key] = expirationDate;

                        expiration.Flush();

                    });

                engine.Commit();

 

                //only dispose disk-based engine

                if (!isMemory)

                    engine.Dispose();

            }

        }

 

        public override void Add<K>(string Category, K Key, object Data, DateTime? ExpirationDate)

        {

            Add<K, object>(Category, new List<KeyValuePair<K, object>> { new KeyValuePair<K, object>(Key, Data) }, ExpirationDate);

        }

 

        private string GetFile(bool IsData)

        {

            if (!Directory.Exists(dataPath))

                Directory.CreateDirectory(dataPath);

            return dataPath + "SuperCache." + (IsData ? "dat" : "sys");

        }

 

        public override List<KeyValuePair<K, V>> Get<K, V>(string Category, IEnumerable<K> Keys)

        {

            var result = new List<KeyValuePair<K, V>>();

            lock (syncRoot)

            {

                var engine = Engine;

                var table = engine.OpenXIndex<K, string>(Category);

                var expiration = engine.OpenXIndex<K, DateTime>(GetExpirationTable(Category));

                var isCommitRequired = false;

 

                Keys.ForEach(key =>

                    {

                        string buffer;

                        V value;

                        if (table.TryGet(key, out buffer))

                        {

                            //will only deserialize object other than string

                            value = typeof(V) == typeof(string) ? (V)(object)buffer : JSON.Instance.ToObject<V>(buffer);

                            DateTime expirationDate;

                            //get expiration date

                            if (expiration.TryGet(key, out expirationDate))

                            {

                                //expired

                                if (expirationDate < DateTime.Now)

                                {

                                    value = default(V);

                                    table.Delete(key);

                                    table.Flush();

                                    expiration.Delete(key);

                                    expiration.Flush();

                                    isCommitRequired = true;

                                }

                            }

                        }

                        else

                            value = default(V);

 

                        result.Add(new KeyValuePair<K, V>(key, value));

                    });

 

                //only need to commit write actions

                if (isCommitRequired)

                    engine.Commit();

 

                //only dispose disk-based engine

                if (!isMemory)

                    engine.Dispose();

            }

            return result;

        }

 

        public override V Get<K, V>(string Category, K Key)

        {

            var buffer = Get<K, V>(Category, new K[] { Key });

            var result = buffer.FirstOrDefault();

            return result.Value;

        }

    }

}

　　

RawCache.cs

物理文件/Dictionary引擎

?

using System;

using System.Collections.Generic;

using System.Linq;

using System.Text;

using STSdb4.Database;

using fastJSON;

using System.IO;

 

namespace Com.SuperCache.Engine

{

    public class RawCache : BaseCache

    {

        private string dataPath;

        private static Dictionary<string, object> memoryData = new Dictionary<string, object>();

        private static Dictionary<string, DateTime?> memoryExpiration = new Dictionary<string, DateTime?>();

        private static object syncRoot = new object();

        private bool isMemory = false;

 

        public RawCache(string DataPath)

        {

            dataPath = DataPath;

            if (!dataPath.EndsWith(Path.DirectorySeparatorChar.ToString()))

                dataPath += Path.DirectorySeparatorChar;

 

            isMemory = string.IsNullOrEmpty(DataPath);

        }

 

        public override void Add<K>(string Category, K Key, object Data)

        {

            Add(Category, Key, Data, null);

        }

 

        private string GetExpirationTable(string Category)

        {

            return KeyExpiration + "_" + Category;

        }

 

        public override void Add<K, V>(string Category, IEnumerable<KeyValuePair<K, V>> Items, DateTime? ExpirationDate)

        {

            lock (syncRoot)

            {

                Items.ForEach(i =>

                    {

                        var key = i.Key;

                        var data = i.Value;

 

                        if (isMemory)

                        {

                            var memKey = GetKey(Category, key.ToString());

                            memoryData[memKey] = data;

                            memoryExpiration[memKey] = ExpirationDate;

                        }

                        else

                        {

                            //will only serialize object other than string

                            var result = typeof(V) == typeof(string) ? data as string : JSON.Instance.ToJSON(data);

                            File.WriteAllText(GetFile(Category, key.ToString(), true), result);

 

                            //specify expiration

                            //default 30 mins to expire from now

                            var expirationDate = ExpirationDate == null || ExpirationDate <= DateTime.Now ? DateTime.Now.AddMinutes(30) : (DateTime)ExpirationDate;

                            File.WriteAllText(GetFile(Category, key.ToString(), false), expirationDate.ToString());

                        }

                    });

            }

        }

 

        public override void Add<K>(string Category, K Key, object Data, DateTime? ExpirationDate)

        {

            Add<K, object>(Category, new List<KeyValuePair<K, object>> { new KeyValuePair<K, object>(Key, Data) }, ExpirationDate);

        }

 

        private string GetFile(string Category, string FileName, bool IsData)

        {

            var path = dataPath + Category.NormalizeFileName() + @"";

            if (!Directory.Exists(path))

                Directory.CreateDirectory(path);

            return path + FileName.NormalizeFileName() + "." + (IsData ? "dat" : "exp");

        }

 

        private string GetKey(string Category, string Key)

        {

            return Category + "_" + Key;

        }

 

        public override List<KeyValuePair<K, V>> Get<K, V>(string Category, IEnumerable<K> Keys)

        {

            var result = new List<KeyValuePair<K, V>>();

            lock (syncRoot)

            {

                Keys.ForEach(key =>

                    {

                        string buffer;

                        V value;

                        if (isMemory)

                        {

                            var memKey = GetKey(Category, key.ToString());

                            object memBuffer;

                            if (memoryData.TryGetValue(memKey, out memBuffer))

                            {

                                value = (V)memBuffer;

                                DateTime? expirationDate;

                                if (memoryExpiration.TryGetValue(memKey, out expirationDate))

                                {

                                    //expired

                                    if (expirationDate != null && (DateTime)expirationDate < DateTime.Now)

                                    {

                                        value = default(V);

                                        memoryData.Remove(memKey);

                                        memoryExpiration.Remove(memKey);

                                    }

                                }

                            }

                            else

                                value = default(V);

                        }

                        else

                        {

                            var dataFilePath = GetFile(Category, key.ToString(), true);

                            if (File.Exists(dataFilePath))

                            {

                                buffer = File.ReadAllText(dataFilePath);

 

                                //will only deserialize object other than string

                                value = typeof(V) == typeof(string) ? (V)(object)buffer : JSON.Instance.ToObject<V>(buffer);

                                DateTime expirationDate;

                                var expirationFilePath = GetFile(Category, key.ToString(), false);

                                if (File.Exists(expirationFilePath))

                                {

                                    buffer = File.ReadAllText(expirationFilePath);

                                    expirationDate = Convert.ToDateTime(buffer);

                                    //expired

                                    if (expirationDate < DateTime.Now)

                                    {

                                        value = default(V);

                                        File.Delete(dataFilePath);

                                        File.Delete(expirationFilePath);

                                    }

                                }

                            }

                            else

                                value = default(V);

                        }

 

                        result.Add(new KeyValuePair<K, V>(key, value));

                    });

            }

            return result;

        }

 

        public override V Get<K, V>(string Category, K Key)

        {

            var buffer = Get<K, V>(Category, new K[] { Key });

            var result = buffer.FirstOrDefault();

            return result.Value;

        }

    }

}

　　

　　

Extensions.cs

扩展函数

?

using System;

using System.Collections.Generic;

using System.Linq;

using System.Text;

using System.IO;

 

namespace Com.SuperCache.Engine

{

    public static class Extensions

    {

        public static void ForEach<T>(this IEnumerable<T> source, Action<T> action)

        {

            if (source != null)

            {

                foreach (var item in source)

                {

                    action(item);

                }

            }

        }

 

        public static string NormalizeFileName(this string FileName)

        {

            var result = FileName;

            Path.GetInvalidFileNameChars().ForEach(c =>

                {

                    result = result.Replace(c.ToString(), string.Empty);

                });

            return result;

        }

    }

}

　　

新建

构造CacheEngine需要传递缓存保存到哪个文件夹。

基于内存

如果你不喜欢基于磁盘的缓存，可以使用基于内存，构造函数传递空字符串便可。

增加/更新

同一个方法：Add。用户可以指定类型（Category），譬如User，Employee等。键（Key）支持泛型，值（Data）是object。有一个overload是过期日期（ExpirationDate），默认当前时间30分钟后

获取

Get方法需要指定类型（Category）和键（Key）。

 

例子

?

using System;

using System.Collections.Generic;

using System.Linq;

using System.Text;

using System.Threading;

using System.Threading.Tasks;

using System.Diagnostics;

using Com.SuperCache.Engine;

 

namespace Com.SuperCache.Test

{

    public class Foo

    {

        public string Name { get; set; }

        public int Age { get; set; }

        public double? Some { get; set; }

        public DateTime? Birthday { get; set; }

    }

 

    class Program

    {

        static void Main(string[] args)

        {

            //TestAddGet();

 

            //Thread.Sleep(4000);

            //TestExpiration();

 

            TestDefaultDiskPerformance();

            TestRawMemoryPerformance();

            TestRawDiskPerformance();

            TestRawMemoryPerformance();

 

            //TestConcurrent();

 

            Console.Read();

        }

 

        private static void TestConcurrent()

        {

            var w = new Stopwatch();

            w.Start();

            Parallel.For(1, 3, (a) =>

                {

                    var employees = Enumerable.Range((a - 1) * 1000, a * 1000).Select(i => new KeyValuePair<int, string>(i, "Wilson " + i + " Chen"));

                    var engine = new CacheEngine(@"....data");

                    engine.Add<int, string>("Employee", employees, DateTime.Now.AddMinutes(1));

                });

            w.Stop();

            Console.WriteLine("add:" + w.Elapsed);

 

            var engine2 = new CacheEngine(@"....data");

            var o = engine2.Get<int, string>("Employee", 1005);

            Console.WriteLine(o);

        }

 

        private static void TestDefaultDiskPerformance()

        {

            TestPerformance(CacheProviders.Default, @"....data");

        }

 

        private static void TestDefaultMemoryPerformance()

        {

            TestPerformance(CacheProviders.Default, string.Empty);

        }

 

        private static void TestRawDiskPerformance()

        {

            TestPerformance(CacheProviders.Raw, @"....data");

        }

 

        private static void TestRawMemoryPerformance()

        {

            TestPerformance(CacheProviders.Raw, string.Empty);

        }

 

        private static void TestPerformance(CacheProviders Provider, string DataPath)

        {

            Console.WriteLine("Performance Test: " + Provider.ToString() + ", " + (string.IsNullOrEmpty(DataPath) ? "Memory" : DataPath));

            var engine = new CacheEngine(Provider, DataPath);

            var w = new Stopwatch();

 

            w.Start();

            var employees = Enumerable.Range(0, 1000).Select(i => new KeyValuePair<int, string>(i, "Wilson " + i + " Chen"));

            engine.Add<int, string>("Employee", employees, DateTime.Now.AddMinutes(1));

            w.Stop();

            Console.WriteLine("add:" + w.Elapsed);

 

            /*w.Restart();

            employees.ForEach(key =>

                {

                    var o1 = engine.Get<int, string>("Employee", key.Key);

                });

            w.Stop();

            Console.WriteLine("individual get:" + w.Elapsed);*/

 

            w.Restart();

            var keys = employees.Select(i => i.Key);

            var o = engine.Get<int, string>("Employee", keys);

            w.Stop();

            Debug.Assert(o.Count == keys.Count());

            Console.WriteLine("get:" + w.Elapsed);

            Console.WriteLine();

        }

 

        private static void TestExpiration()

        {

            var engine = new CacheEngine(@"....data");

            var o = engine.Get<string, Foo>("User", "wchen");

            Console.WriteLine(o != null ? o.Name : "wchen does not exist or expired");

        }

 

        private static void TestAddGet()

        {

            var engine = new CacheEngine(@"....data");

            var f = new Foo { Name = "Wilson Chen", Age = 30, Birthday = DateTime.Now, Some = 123.456 };

            engine.Add("User", "wchen", f, DateTime.Now.AddSeconds(5));

 

            var o = engine.Get<string, Foo>("User", "wchen");

            Console.WriteLine(o.Name);

 

            var o4 = engine.Get<string, Foo>("User", "foo");

            Console.WriteLine(o4 != null ? o4.Name : "foo does not exist");

 

            var o3 = engine.Get<string, string>("PlainText", "A");

            Console.WriteLine(o3 ?? "A does not exist");

        }

    }

}

　　

性能

通过上述性能测试例子，你会发现STSdb的磁盘存取速度要比一个记录对应一个物理文件快。想了解更多，请访问官方网站。

测试条件：1000条记录，7200RPM磁盘，i7。

引擎	介质	写入	读取
STSdb	磁盘	1.3s	0.06s
	内存	0.01s	0.008s
Raw	磁盘	1.0s	0.16s
	内存	0.0004s	0.0008s

 　　

说明

项目中引用了System.Management是因为STSdb支持内存数据库，需要判断最大物理内存。如果不喜欢，大家可以移除引用，并且去掉STSdb4.Database.STSdb.FromMemory方法便可。

下载

点击这里下载

 

 

分类: 1.每日一得, 3.开源项目