Kinect 开发 —— 图片浏览

总体思路

首先运用WPF编写一个简单的支持多点触控的图片浏览程序，这方面您可以参看MSDN上的这篇文章，上面有代码，可能需要FQ才能下载。中文的话，您可以参考Gnie同学关于在WPF上面多点触屏(MultiTouch)应用程序的相关文章，这些是基础。

然后，将从Kinect骨骼信息中提取到的双手的位置信息，模拟为触摸屏上的点击，这个有点麻烦，也是本文的重点。这方面我参考了candescentnui这一开源项目。

---

具体步骤

完成多点触摸类逻辑的编写

WPF本身支持触摸屏设备和多点触控，在System.Windows.Input 下有一个TouchDevice 类，它表示 触摸屏上一个手指的产生的单个触摸输入。我们需要继承这个类，并对其定制将Kienct骨骼点数据转换为触摸屏上的单个输入。为此新建一个名为KinectTouchDevice

的类并继承 TouchDevice类和Idisposable接口。

internal class KinectTouchDevice : TouchDevice, IDisposable
{
    private DateTime? firstTouch;
    public Point Position { get; private set; }
    internal TouchState TouchState { get; private set; }


    public KinectTouchDevice(int id, PresentationSource source): base(id)
    {
        this.Position = new Point();
        this.TouchState = TouchState.Up;
        this.SetActiveSource(source);
    }

    public void Touch(Point position)
    {
        //记录第一次触摸时间
        if (!this.firstTouch.HasValue)
        {
            this.firstTouch = DateTime.Now;
            return;
        }//如果不是第一次点击，但两次间隔小于100毫秒，则认为是一次点击，不做处理
        else if (DateTime.Now.Subtract(this.firstTouch.Value).TotalMilliseconds < 100)
        {
            return;
        }
        this.Position = position;
        if (!this.IsActive)
        {
            this.Activate();
        }
        if (this.TouchState != TouchState.Down)
        {
            this.Dispatcher.Invoke(new Func<bool>(this.ReportDown));
            this.TouchState = TouchState.Down;
        }
        else
        {
            this.Dispatcher.Invoke(new Func<bool>(this.ReportMove));
        }
    }

    public void NoTouch()
    {
        this.firstTouch = null;
        if (TouchState == TouchState.Down)
        {
            this.Dispatcher.Invoke(new Func<bool>(this.ReportUp));
        }
        this.TouchState = TouchState.Up;
    }

    public override TouchPointCollection GetIntermediateTouchPoints(IInputElement relativeTo)
    {
        return new TouchPointCollection();
    }

    public override TouchPoint GetTouchPoint(IInputElement relativeTo)
    {
        var point = this.Position;
        if (relativeTo != null)
        {
            //获取当前点击位置
            point = this.ActiveSource.RootVisual.TransformToDescendant((Visual)relativeTo).Transform(point);
        }
        return new TouchPoint(this, point, new Rect(point, new Size(1, 1)), TouchAction.Move);
    }

    public void Dispose()
    {
        if (this.IsActive)
        {
            this.Deactivate();
        }
    }
}

这是一个点，如何模拟一个面板呢，所以需要建立包含这一个点的集合的新类，名为KinectTouchDevice

public class KinectMultiTouchDevice : IDisposable
{
    //触控数据源
    private HandDataSource handDataSource;
    private PresentationSource presentationSource;
    //触控点集合，每一个点对应一个id
    private IDictionary<int, KinectTouchDevice> touchDevices;
    public Size TargetSize { get; set; }

    public KinectMultiTouchDevice(HandDataSource handDataSource, PresentationSource presentationSource, Size targetSize)
    {
        this.presentationSource = presentationSource;
        this.TargetSize = targetSize;
    }

    public KinectMultiTouchDevice(HandDataSource handDataSource, FrameworkElement area)
    {
        this.touchDevices = new Dictionary<int, KinectTouchDevice>();
        this.TargetSize = new Size(area.ActualWidth, area.ActualHeight);
        this.presentationSource = PresentationSource.FromVisual(area);
        this.handDataSource = handDataSource;
        //当数据源有新数据时，触发处理事件
        this.handDataSource.NewDataAvailable += handDataSource_NewDataAvailable;
        area.SizeChanged += area_SizeChanged;
    }

    private void handDataSource_NewDataAvailable(Object sender, HandCollectionEventArgs data)
    {
        if (data.IsEmpty)
        {
            ReportNoTouch(this.touchDevices.Values);
            return;
        }

        var touchedDevices = this.ReportTouches(data);
        this.ReportNoTouch(this.touchDevices.Values.Except(touchedDevices));
    }

    private void area_SizeChanged(object sender, SizeChangedEventArgs e)
    {
        this.TargetSize = e.NewSize;
    }


    private IList<KinectTouchDevice> ReportTouches(HandCollectionEventArgs data)
    {
        var touchedDevices = new List<KinectTouchDevice>();
        foreach (var hand in data.Hands)
        {
                var device = this.GetDevice(hand.Id);
                var pointOnPresentationArea = this.MapToPresentationArea(hand, new Size(this.handDataSource.Width, this.handDataSource.Height));
                device.Touch(pointOnPresentationArea);
                touchedDevices.Add(device);
        }
        return touchedDevices;
    }

    private void ReportNoTouch(IEnumerable<KinectTouchDevice> devices)
    {
        foreach (var device in devices)
        {
            device.NoTouch();
        }
    }

    private KinectTouchDevice GetDevice(int index)
    {
        if (!this.touchDevices.ContainsKey(index))
        {
            this.presentationSource.Dispatcher.Invoke(new Action(() =>
            {
                if (!this.touchDevices.ContainsKey(index))
                this.touchDevices.Add(index, new KinectTouchDevice(index, this.presentationSource));
            }));
        }
        return this.touchDevices[index];
    }

    private Point MapToPresentationArea(HandData fingerPoint, Size originalSize)
    {
        // return new Point(fingerPoint.X / originalSize.Width * this.TargetSize.Width, fingerPoint.Y / originalSize.Height * this.TargetSize.Height);
        return new Point(fingerPoint.X, fingerPoint.Y);
    }

    public void Dispose()
    {
        this.handDataSource.NewDataAvailable -= handDataSource_NewDataAvailable;
        foreach (var device in this.touchDevices.Values)
        {
            device.Dispose();
        }
    }
}

需要注意的是,上面代码中，touchDevices 是一个IDictionary<int, KinectTouchDevice> 型的对象,表示所有触控点的集合，每一个触控点有一个int型的id。代码中HandDataSource 类型的handDataSource,表示触发触控的数据源,在KinectMultiTouchDevice类的构造函数中,我们注册了handDataSource的NewDataAvailable事件,该事件会在每当从Kinect中获取每一帧数据,且数据符合特定条件就会触发。HandDataSource类如下：

public class HandDataSource 
{
    public delegate void NewDataHandler<HandCollectionEventArgs>(Object sender,HandCollectionEventArgs data);

    public event NewDataHandler<HandCollectionEventArgs> NewDataAvailable;

    public int Width { get; set; }

    public int Height { get; set; }

    protected virtual void OnNewDataAvailable(HandCollectionEventArgs e)
    {
        NewDataHandler<HandCollectionEventArgs> temp = NewDataAvailable;
        if (temp != null)
        {
            temp(this, e);
        }
    }

    public void RaiseNewDataEvent(List<HandData> handData) {
        HandCollectionEventArgs e = new HandCollectionEventArgs(handData);
        OnNewDataAvailable(e);
    }
}

---

界面逻辑的编写

下面我们来看应用程序的前台代码。为了在界面上显示手的位置，这里我们建立一个名为TouchControl的自定义控件，该控件很简单，里面包含一个椭圆形和一个label对象，用以表示当前手在屏幕上的位置，代码如下：

<UserControl x:Class="KinectImageView.MultiTouch.TouchControl"
             xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation"
             xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
             xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006" 
             xmlns:d="http://schemas.microsoft.com/expression/blend/2008" 
             mc:Ignorable="d" 
             d:DesignHeight="40" d:DesignWidth="40">
    <Grid Width="40" Height="40">
        <Ellipse Stroke="White" StrokeThickness="3"/>
        <Label Foreground="White" Name="Label" HorizontalAlignment="Center" VerticalAlignment="Center"/>
    </Grid>
</UserControl>

后台逻辑代码也很简单，只有一个带参的构造函数。

public partial class TouchControl : UserControl
{
    public TouchControl()
    {
        InitializeComponent();
    }

    public TouchControl(int id)
        : this()
    {
        this.Label.Content = id;
    }
}

接下来就是主界面了，为了简便，主界面上随意摆放了三张图片，用于我们使用Kinect来进行缩放平移旋转等操作，在页面的最底层添加了一个TouchControl自定义控件，用来显示手所在的位置。整个界面前端代码如下：

<Window x:Class="KinectImageView.MainWindow"
        xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation"
        xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
        Title="MainWindow" xmlns:c="clr-namespace:KinectImageView" Closing="Window_Closing" 
        Loaded="Window_Loaded" 
        mc:Ignorable="d" xmlns:d="http://schemas.microsoft.com/expression/blend/2008" 
        xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006" 
        SizeToContent="WidthAndHeight">
    <Grid x:Name="LayoutRoot"  Height="750" Width="1000" >
        <Canvas Name="mainCanvas">
            <Image Name="image" Panel.ZIndex="642" IsManipulationEnabled="True" Width="200" Source="Images/flower.jpg" >
                <Image.RenderTransform>
                    <MatrixTransform Matrix="1.5929750047527,0.585411309251951,-0.585411309251951,1.5929750047527,564.691807426081,79.4658072348299"/>
                </Image.RenderTransform>
            </Image>
            <Image Name="image1" Panel.ZIndex="641"  IsManipulationEnabled="True"   Width="200" Source="Images/flower2.jpg" >
                <Image.RenderTransform>
                    <MatrixTransform  Matrix="1.79780224775912,-1.1136472330559,1.1136472330559,1.79780224775912,45.6962327448951,205.029554723656" />
                </Image.RenderTransform>
            </Image>
            <Image Name="image2" Panel.ZIndex="644"  IsManipulationEnabled="True"  Width="200"   Source="Images/flower3.jpg" >
                <Image.RenderTransform>
                    <MatrixTransform Matrix="2.41806325085411,-0.0527474549128994,0.0527474549128994,2.41806325085411,280.737615796121,292.420001677231"/>
                </Image.RenderTransform>
            </Image>         
        </Canvas>
        <Canvas Name="fingerCanvas"></Canvas>
    </Grid>
</Window>

下面来看看后台代码，WPF默认支持开发多点触控的程序，只需要从写下面三个方法即可：

protected override void OnManipulationStarting(ManipulationStartingEventArgs e)
{
    base.OnManipulationStarting(e);
    e.ManipulationContainer = mainCanvas;
    e.Handled = true;
}

protected override void OnManipulationDelta(ManipulationDeltaEventArgs e)
{
    base.OnManipulationDelta(e);
    var element = e.Source as FrameworkElement;
    var transformation = element.RenderTransform as MatrixTransform;
    //获取缩放的中心点
    Point center = new Point(element.ActualWidth / 2, element.ActualHeight / 2);
    var matrix = transformation == null ? Matrix.Identity : transformation.Matrix;
    center = matrix.Transform(center);
    //缩放
    if (e.DeltaManipulation.Scale.X > 0.5 && e.DeltaManipulation.Scale.Y > 0.5
        && e.DeltaManipulation.Scale.X < 2 && e.DeltaManipulation.Scale.Y < 2)
        matrix.ScaleAt(e.DeltaManipulation.Scale.X, e.DeltaManipulation.Scale.Y, center.X, center.Y);
    //旋转
    matrix.RotateAt(e.DeltaManipulation.Rotation, center.X, center.Y);
    //移动
    if (center.X > 0 && center.Y > 0
        && center.X < this.mainCanvas.ActualWidth
        && center.Y < this.mainCanvas.ActualHeight)
        matrix.Translate(e.DeltaManipulation.Translation.X, e.DeltaManipulation.Translation.Y);

    element.RenderTransform = new MatrixTransform(matrix);
}

protected override void OnManipulationInertiaStarting(ManipulationInertiaStartingEventArgs e)
{
    base.OnManipulationInertiaStarting(e);
    e.TranslationBehavior.DesiredDeceleration = 0.001;
    e.RotationBehavior.DesiredDeceleration = 0.01;
    e.ExpansionBehavior.DesiredDeceleration = 0.01;
}

除此之外，为了使用Kinect数据模拟触控，我们还需要重载OnTouchMove，OnTouchDown和OnTouchUp这三个方法，详细代码如下：

protected override void OnTouchMove(TouchEventArgs e)
{
    base.OnTouchMove(e);
    HandleTouch(e);
}

protected override void OnTouchDown(TouchEventArgs e)
{
    base.OnTouchDown(e);
    HandleTouch(e);
}

protected override void OnTouchUp(TouchEventArgs e)
{
    base.OnTouchUp(e);
    this.fingerCanvas.Children.Remove(this.touchPoints[e.TouchDevice.Id]);
    this.touchPoints.Remove(e.TouchDevice.Id);
}

private void HandleTouch(TouchEventArgs e)
{
    var visual = GetTouchVisual(e.TouchDevice.Id);
    var point = e.GetTouchPoint(this.fingerCanvas).Position;
    visual.SetValue(Canvas.LeftProperty, point.X);
    visual.SetValue(Canvas.TopProperty, point.Y);
}

private TouchControl GetTouchVisual(int deviceId)
{
    if (this.touchPoints.ContainsKey(deviceId))
    {
        return this.touchPoints[deviceId];
    }

    var touchControl = new TouchControl(deviceId);
    this.touchPoints.Add(deviceId, touchControl);
    this.fingerCanvas.Children.Add(touchControl);
    return touchControl;
}

以上工作做好之后，我们现在需要从Kinect中获取数据，然后发起事件，传递参数，根据数据来模拟屏幕点击。如何建立Kinect连接，以及如何获取数据这里不详细讲解了，你可以参考之前Kinect for Windows SDK入门系列文章。这里就如何从Kinect获取数据以及如何发起事件来进行详细讨论。从Kinect中获取数据最简单的方法就是注册相应事件，在本例中，我们需要骨骼数据，所以需要注册KinectSensor对象的SkeletonFrameReady事件。具体的事件中处理代码如下：

private void KinectDevice_SkeletonFrameReady(object sender, SkeletonFrameReadyEventArgs e)
{
    using (SkeletonFrame frame = e.OpenSkeletonFrame())
    {
        if (frame != null)
        {
            frame.CopySkeletonDataTo(this.frameSkeletons);
            Skeleton skeleton = GetPrimarySkeleton(this.frameSkeletons);

            if (skeleton != null)
            {
                Joint head = skeleton.Joints[JointType.Head];

                Joint leftHand = skeleton.Joints[JointType.HandLeft];
                Joint leftWrist = skeleton.Joints[JointType.WristLeft];

                Joint rightHand = skeleton.Joints[JointType.HandRight];
                Joint rightWrist = skeleton.Joints[JointType.WristRight];

                Point leftHandPos = GetPosition(leftHand);
                Point leftWristPos = GetPosition(leftWrist);

                Point rightHandPos = GetPosition(rightHand);
                Point rightWristPos = GetPosition(rightWrist);


                if (rightHandPos.Y < rightWristPos.Y && leftHandPos.Y < leftWristPos.Y)
                {
                    leftHandTarget = GetHitTarget(skeleton.Joints[JointType.HandLeft], mainCanvas);
                    rightHandTarget = GetHitTarget(skeleton.Joints[JointType.HandRight], mainCanvas);

                    if (rightHandTarget != null)
                    {
                        dics.Clear();
                        foreach (Image element in mainCanvas.Children)
                        {
                            dics.Add(element, Canvas.GetZIndex(element));
                        }
                        ResetZIndex(dics, rightHandTarget);
                    }
                    if (leftHandTarget != null && rightHandTarget != null)
                    {
                        Image leftHandHitImage = leftHandTarget as System.Windows.Controls.Image;
                        Image rightHnadHitImage = rightHandTarget as System.Windows.Controls.Image;
                        if (leftHandHitImage != null && rightHnadHitImage != null)
                        {

                            String leftHandName = leftHandHitImage.Name;
                            String rightHandName = leftHandHitImage.Name;
                            if (rightHandName.Equals(leftHandName))
                            {
                                List<HandData> list = new List<HandData>()
                        {
                            new HandData{ Id=1,X=leftHandPos.X,Y=leftHandPos.Y},
                            new HandData{ Id=2,X=rightHandPos.X,Y=rightHandPos.Y}
                        };
                                handDataSource.RaiseNewDataEvent(list);
                            }
                        }
                    }
                    else
                    {
                        handDataSource.RaiseNewDataEvent(new List<HandData>());
                    }
                }
                else
                {
                    handDataSource.RaiseNewDataEvent(new List<HandData>());
                }
            }
        }
    }
}

在该方法中，我们从骨骼数据中获取左右手的具体位置，然后当左右手的手部(hand)高于肘部(wrist)时,则认为用户是要进行操作；然后根据左右手所在的位置，获取当前左右手所在的对象，将该对象置于最前，以便于我们进行操作。然后判断左右手是否位于同一个对象之上，如果是，则将左右手的坐标点存储到list中，触发事件handDataSource.RaiseNewDataEvent(list)，提醒有新的触摸点产生。这里handDataSource对象是在Window_Loaded方法中初始化的。

private void Window_Loaded(object sender, RoutedEventArgs e)
{
    handDataSource = new HandDataSource();
    handDataSource.Width = kinectDevice.DepthStream.FrameWidth;
    handDataSource.Height = kinectDevice.DepthStream.FrameHeight;
    this.device = new KinectMultiTouchDevice(handDataSource, this);
    this.touchPoints = new Dictionary<int, TouchControl>();
}

从上面的方法中可以看到，我们初始化KinectMultiTouchDevice类型的device对象的时候传入了handDataSource，所以在上面我们触发handDataSource的RaiseNewDataEvent事件时，device的构造函数中注册了该事件，所以会模拟触控点击。