zoukankan      html  css  js  c++  java
  • 延迟添加队列

    using System;
    using System.Collections.Generic;
    using System.Diagnostics.Contracts;
    using System.Linq;
    using System.Net;
    using System.Text;
    using System.Threading.Tasks;
    
    namespace AnfleCrawler.Common
    {
        /// <summary>
        /// 延迟添加队列
        /// </summary>
        internal class LagPattern : Disposable
        {
            private static readonly List<LagPattern> Set = new List<LagPattern>();
            private const double PerCount = 50d;
            private WeakReference _weakRef;
            private int _depth;
            private object _state;
            private JobTimer _job;
    
            private PageCrawler Weak
            {
                get
                {
                    if (!_weakRef.IsAlive)
                    {
                        throw new InvalidOperationException("Wrap Dead");
                    }
                    return (PageCrawler)_weakRef.Target;
                }
            }
    
            public LagPattern(PageCrawler wrap, StringPatternGenerator urlGen, int depth, object state)
            {
                Contract.Requires(wrap != null && urlGen != null);
    
                _weakRef = new WeakReference(wrap);
                _depth = depth;
                _state = state;
    
                var tor = urlGen.GetEnumerator();
                int totalCount = urlGen.Count();
                if (totalCount < PerCount)
                {
                    Proc(tor);
                    return;
                }
    
                double span = Math.Ceiling(360d / (totalCount / PerCount));
                //double span = 16d;
                App.LogInfo("LagPattern Span:{0}", span);
                _job = new JobTimer(Proc, TimeSpan.FromSeconds(span));
                _job.Start(tor);
                //App.DisposeService.Register(this.GetType(), this);
                lock (Set)
                {
                    Set.Add(this);
                }
            }
    
            protected override void DisposeInternal(bool disposing)
            {
                if (disposing)
                {
                    if (_job != null)
                    {
                        _job.Dispose();
                    }
                    //App.DisposeService.Release(this.GetType(), this);
                    lock (Set)
                    {
                        Set.Remove(this);
                    }
                }
            }
    
            private void Proc(object state)
            {
                var wrap = this.Weak;
                var urlGen = (IEnumerator<string>)state;
                for (int i = 0; i < PerCount; i++)
                {
                    if (!urlGen.MoveNext())
                    {
                        this.Dispose();
                        return;
                    }
                    wrap.PushUrl(new Uri(urlGen.Current), _depth, _state);
                }
            }
        }
    }
  • 相关阅读:
    centos ftp搭建
    python_模块
    python_如何建立包
    python_类
    python_递归
    python_函数
    python_字典
    python_条件、循环语句
    python_中文乱码问题
    Viola–Jones object detection framework--Rapid Object Detection using a Boosted Cascade of Simple Features中文翻译 及 matlab实现(见文末链接)
  • 原文地址:https://www.cnblogs.com/Googler/p/4287383.html
Copyright © 2011-2022 走看看