UWP简单示例（一）：快速合成音乐MV

说明

　　本文发布较早，查看最新动态，请关注 TypeScript 版本。（2020 年 1 月 注）

在线演示： 音频可视化（TypeScript）

准备

　　IDE：Visual Studio

　　Nuget包：SharpDx.XAudio2

　　Nuget包：Win2D.UWP

　　了解学习：Win2D 官方博客

　　了解学习：Win2D 官方示例

第一节 波形

　　获取实时时域数据。

Imports SharpDX.Multimedia
Imports SharpDX.XAudio2
Public Class AudioPlayer
    Public Event WavePlaying(e As WavePlayingEventArgs)
    Public Property Device As XAudio2
    Public Property Voice As SourceVoice
    Private CurrentFormat As WaveFormat
    Private CurrentBuffer As AudioBuffer
    Private PacketsInfo As UInteger()
    Public Sub New()
        Device = New XAudio2()
        Device.StartEngine()
        Dim mv As New MasteringVoice(Device)
    End Sub
    Public Async Function LoadFile(fileName As String) As Task(Of Boolean)
        Try
            Voice = Await CreateVoiceFromFile(Device, fileName)
            LoadBuffer()
            Return True
        Catch
            Return False
        End Try
    End Function
    Public Sub Play(Optional volume As Single = 1.0F)
        Voice?.SetVolume(volume)
        Voice?.Start()
        ReadBuffer()
    End Sub
    Public Sub [Stop]()
        Voice?.Stop()
    End Sub
    Protected Async Function CreateVoiceFromFile(device As XAudio2, fileName As String) As Task(Of SourceVoice)
        Dim file = Await Package.Current.InstalledLocation.GetFileAsync(fileName)
        Dim streamWithContentType = Await file.OpenReadAsync()
        Dim st = streamWithContentType.AsStreamForRead()
        Using stream = New SoundStream(st)
            CurrentFormat = stream.Format
            CurrentBuffer = New AudioBuffer() With {
            .Stream = stream.ToDataStream(),
            .AudioBytes = CInt(stream.Length),
            .Flags = BufferFlags.EndOfStream
        }
            PacketsInfo = stream.DecodedPacketsInfo
        End Using
        Dim sourceVoice = New SourceVoice(device, CurrentFormat, True)
        Return sourceVoice
    End Function
    Protected Sub LoadBuffer()
        Voice?.FlushSourceBuffers()
        Voice?.SubmitSourceBuffer(CurrentBuffer, PacketsInfo)
    End Sub
    ''' <summary>
    ''' 从流中读取当前播放的数据
    ''' </summary>
    Private Async Sub ReadBuffer()
        Try
            Dim count As Integer = CurrentFormat?.AverageBytesPerSecond / 10
            While Voice.State.BuffersQueued > 0
                If Voice.State.SamplesPlayed * CurrentFormat.BlockAlign > CurrentBuffer.Stream.Position + count Then
                    Dim byteArr(count - 1) As Byte
                    Await CurrentBuffer.Stream.ReadAsync(byteArr, 0, count)
                    RaiseEvent WavePlaying(New WavePlayingEventArgs(byteArr, CurrentFormat))
                Else
                    Await Task.Delay(10)
                End If
            End While
        Catch
            Return
        End Try
    End Sub
    Public Function Position() As Integer
        Return Voice.State.SamplesPlayed / CurrentFormat.SampleRate
    End Function
    Protected Overrides Sub Finalize()
        Try
            Dispose(False)
        Finally
            MyBase.Finalize()
        End Try
    End Sub
    Public Sub Dispose()
        Dispose(True)
        GC.SuppressFinalize(Me)
    End Sub
    Private Sub Dispose(isDisposing As Boolean)
        If Not isDisposing Then
            Return
        End If
        Voice.DestroyVoice()
        Voice.Dispose()
        CurrentBuffer.Stream.Dispose()
    End Sub
End Class

//由于在线转换工具对异步代码段支持不好，该处代码请参考VB.

图1-1 实时波谱

第二节 频谱

　　通过快速傅里叶变换将时域信号转换为频域信号。

    ''' <summary>
    ''' 快速傅里叶变换
    ''' </summary>
    ''' <param name="data">指定的数据</param>
    ''' <param name="count"></param>
    ''' <returns></returns>
    Public Shared Function FFT(ByVal data() As Single, ByVal count As Integer) As Single() '快速傅里叶变换
        Dim fftCount As Integer = count
        Dim j As Integer
        Dim k As Integer
        Dim NM1 As Integer
        Dim ND2 As Integer
        Dim M, L, LE, LE2, IP, JM1 As Integer
        Dim TR, TI, SR, SI, UR, UI As Double
        Dim IMX(fftCount - 1) As Double
        Dim REX(fftCount - 1) As Double
        Dim n As Integer = fftCount
        Array.Copy(data, REX, fftCount)
        NM1 = n - 1
        ND2 = n / 2
        M = CInt(Math.Log(n) / Math.Log(2))
        j = ND2

        For i = 1 To n - 2
            If i < j Then
                TR = REX(j)
                TI = IMX(j)
                REX(j) = REX(i)
                IMX(j) = IMX(i)
                REX(i) = TR
                IMX(i) = TI
            End If
            k = ND2
            While (k <= j)
                j = j - k
                k = k / 2
            End While
            j = j + k
        Next i

        For L = 1 To M
            LE = CInt(2 ^ L)
            LE2 = LE / 2
            UR = 1
            UI = 0
            SR = Math.Cos(Math.PI / LE2)
            SI = -Math.Sin(Math.PI / LE2)
            For j = 1 To LE2
                JM1 = j - 1
                For i = JM1 To NM1 Step LE
                    IP = i + LE2
                    TR = REX(IP) * UR - IMX(IP) * UI
                    TI = REX(IP) * UI + IMX(IP) * UR
                    REX(IP) = REX(i) - TR
                    IMX(IP) = IMX(i) - TI
                    REX(i) = REX(i) + TR
                    IMX(i) = IMX(i) + TI
                Next i
                TR = UR
                UR = TR * SR - UI * SI
                UI = TR * SI + UI * SR
            Next j
        Next L
        '
        Dim w() As Single
        ReDim w(fftCount / 2)    '取有效值
        ''公式:F=Kf/N F=频率;k=位置;f=取样频率;N=样本数;有效数据(N/2+1)个.
        For i = 0 To fftCount / 2
            w(i) = Math.Sqrt(REX(i) * REX(i) + IMX(i) * IMX(i))
        Next
        Return w
    End Function

/// <summary>
/// 快速傅里叶变换
/// </summary>
/// <param name="data">指定的数据</param>
/// <param name="count"></param>
/// <returns></returns>
public static float[] FFT(float[] data, int count)
{
    //快速傅里叶变换
    int fftCount = count;
    int j = 0;
    int k = 0;
    int NM1 = 0;
    int ND2 = 0;
    int M = 0;
    int L = 0;
    int LE = 0;
    int LE2 = 0;
    int IP = 0;
    int JM1 = 0;
    double TR = 0;
    double TI = 0;
    double SR = 0;
    double SI = 0;
    double UR = 0;
    double UI = 0;
    double[] IMX = new double[fftCount];
    double[] REX = new double[fftCount];
    int n = fftCount;
    Array.Copy(data, REX, fftCount);
    NM1 = n - 1;
    ND2 = n / 2;
    M = Convert.ToInt32(Math.Log(n) / Math.Log(2));
    j = ND2;

    for (i = 1; i <= n - 2; i++) {
        if (i < j) {
            TR = REX[j];
            TI = IMX[j];
            REX[j] = REX[i];
            IMX[j] = IMX[i];
            REX[i] = TR;
            IMX[i] = TI;
        }
        k = ND2;
        while ((k <= j)) {
            j = j - k;
            k = k / 2;
        }
        j = j + k;
    }

    for (L = 1; L <= M; L++) {
        LE = Convert.ToInt32(Math.Pow(2, L));
        LE2 = LE / 2;
        UR = 1;
        UI = 0;
        SR = Math.Cos(Math.PI / LE2);
        SI = -Math.Sin(Math.PI / LE2);
        for (j = 1; j <= LE2; j++) {
            JM1 = j - 1;
            for (i = JM1; i <= NM1; i += LE) {
                IP = i + LE2;
                TR = REX[IP] * UR - IMX[IP] * UI;
                TI = REX[IP] * UI + IMX[IP] * UR;
                REX[IP] = REX[i] - TR;
                IMX[IP] = IMX[i] - TI;
                REX[i] = REX[i] + TR;
                IMX[i] = IMX[i] + TI;
            }
            TR = UR;
            UR = TR * SR - UI * SI;
            UI = TR * SI + UI * SR;
        }
    }
    //
    float[] w = null;
    w = new float[fftCount / 2 + 1];
    //取有效值
    //'公式:F=Kf/N F=频率;k=位置;f=取样频率;N=样本数;有效数据(N/2+1)个.
    for (i = 0; i <= fftCount / 2; i++) {
        w[i] = Math.Sqrt(REX[i] * REX[i] + IMX[i] * IMX[i]);
    }
    return w;
}

图2-1 实时频谱

第三节 简化

　　简化频域数据，将相邻的若干组值相加后取平均值。

　　频谱也可以绘制成圆环状。

    Public DataWave As ConcurrentQueue(Of Single)
    Public DataFFT() As Single
    Public DataFFTExtra() As Single
    Private Sub CalcFFT()
        Dim count As Integer = 2048 '最小时域数据的长度
        If DataWave.Count < count Then Return
        Dim tempD(count - 1) As Single
        For i = 0 To count - 1
            tempD(i) = DataWave(DataWave.Count - count + i)
        Next
        DataFFT = SignalMath.FFT(tempD, count) '原始频谱

        Dim extraCount As Integer = 64 '简化的长度
        Dim TempList As New List(Of Single)
        Dim sCount As Integer = DataFFT.Count / extraCount
        TempList.Add(0)
        For i = 1 To extraCount - 1
            Dim TempSingle As Single = 0
            For j = 0 To sCount - 1
                TempSingle += DataFFT(i * sCount + j)
            Next
            TempSingle = TempSingle / sCount / 10
            TempList.Add(TempSingle)
        Next
        DataFFTExtra = TempList.ToArray '简化后的频谱
        WaveSignal.Energy = DataFFTExtra.ToList.IndexOf(DataFFTExtra.Max) '不精确的基音位置
    End Sub

public ConcurrentQueue<float> DataWave;
public float[] DataFFT;
public float[] DataFFTExtra;
private void CalcFFT()
{
    int count = 2048;
    //最小时域数据的长度
    if (DataWave.Count < count)
        return;
    float[] tempD = new float[count];
    for (i = 0; i <= count - 1; i++) {
        tempD[i] = DataWave(DataWave.Count - count + i);
    }
    DataFFT = SignalMath.FFT(tempD, count);
    //原始频谱

    int extraCount = 64;
    //简化的长度
    List<float> TempList = new List<float>();
    int sCount = DataFFT.Count / extraCount;
    TempList.Add(0);
    for (i = 1; i <= extraCount - 1; i++) {
        float TempSingle = 0;
        for (j = 0; j <= sCount - 1; j++) {
            TempSingle += DataFFT[i * sCount + j];
        }
        TempSingle = TempSingle / sCount / 10;
        TempList.Add(TempSingle);
    }
    DataFFTExtra = TempList.ToArray();//简化后的频谱
}

图3-1 频谱变形

第四节 场景

　　粒子

　　加入飞舞的粒子。粒子运动速度与实时音频的基音大小相关。

　　若用贴图取代圆点，可生成效果逼真的烟雾或者飞雪等场景。

Imports System.Numerics
Imports Windows.UI
''' <summary>
''' 粒子类，表示一个拥有加速度、加速度和位置矢量的抽象粒子
''' </summary>
Public Class Partical
    Public Property Location As Vector2 '位置矢量
    Public Property Velocity As Vector2 '速度
    Public Property Acceleration As Vector2 '加速度
    Public Property Mass As Single = 10.0 '质量大小
    Public Property Age As Single = 0 '生命周期
    Public Property Alpha As Single = 255
    Public Property Size As Single = 1
    Public Property Radius As Single = 0
    Public Property RadiusX As Single = 0
    Public Property RadiusY As Single = 0
    Public Property ImageSize As Single = 1 '粒子图像的大小
    Public Property Color As Color '粒子颜色
    Public Shared Rnd As New Random
    ''' <summary>
    ''' 初始化一个粒子
    ''' </summary>
    Public Sub New(loc As Vector2)
        Location = loc
        Velocity = New Vector2(0, 0)
        Acceleration = New Vector2(0, 0)
    End Sub
    ''' <summary>
    ''' 指定的力作用于当前对象
    ''' </summary>
    ''' <param name="forceVec">指定的力</param>
    Public Sub ApplyForce(forceVec As Vector2)
        Acceleration = Acceleration + forceVec / Mass
    End Sub
    ''' <summary>
    ''' 更新粒子位置，重绘每帧图像前调用该方法
    ''' </summary>
    Public Sub Move()
        Velocity += Acceleration * WaveSignal.Energy
        'Velocity.LimitMag(20)
        Location += Velocity '更新位置
        Acceleration = Vector2.Zero
    End Sub
    Public Sub StartNew(Loc As Vector2)
        Location = Loc
        Velocity.SetMag(0)
    End Sub
End Class

using Microsoft.VisualBasic;
using System;
using System.Collections;
using System.Collections.Generic;
using System.Data;
using System.Diagnostics;
using System.Numerics;
using Windows.UI;
/// <summary>
/// 粒子类，表示一个拥有加速度、加速度和位置矢量的抽象粒子
/// </summary>
public class Partical
{
    public Vector2 Location { get; set; }
    //位置矢量
    public Vector2 Velocity { get; set; }
    //速度
    public Vector2 Acceleration { get; set; }
    //加速度
    public float Mass { get; set; }
    //质量大小
    public float Age { get; set; }
    //生命周期
    public float Alpha { get; set; }
    public float Size { get; set; }
    public float Radius { get; set; }
    public float RadiusX { get; set; }
    public float RadiusY { get; set; }
    public float ImageSize { get; set; }
    //粒子图像的大小
    public Color Color { get; set; }
    //粒子颜色
    public static Random Rnd = new Random();
    /// <summary>
    /// 初始化一个粒子
    /// </summary>
    public Partical(Vector2 loc)
    {
        Location = loc;
        Velocity = new Vector2(0, 0);
        Acceleration = new Vector2(0, 0);
    }
    /// <summary>
    /// 指定的力作用于当前对象
    /// </summary>
    /// <param name="forceVec">指定的力</param>
    public void ApplyForce(Vector2 forceVec)
    {
        Acceleration = Acceleration + forceVec / Mass;
    }
    /// <summary>
    /// 更新粒子位置，重绘每帧图像前调用该方法
    /// </summary>
    public void Move()
    {
        Velocity += Acceleration * WaveSignal.Energy;//与基音位置相关
        //Velocity.LimitMag(20)
        Location += Velocity;
        //更新位置
        Acceleration = Vector2.Zero;
    }
    public void StartNew(Vector2 Loc)
    {
        Location = Loc;
        Velocity.SetMag(0);
    }
}

　　背景

　　加入背景图片。背景高斯模糊与实时音频的基音大小相关。

　　也可以让背景随机抖动或者旋转。

    Private Sub DrawBackGround(DrawingSession As CanvasDrawingSession)
        Using cmdList = New CanvasCommandList(DrawingSession)
            Using dl = cmdList.CreateDrawingSession
                dl.DrawImage(ImageManager.GetResource(ImageResourceID.back1))
            End Using
            Using blur1 = New Effects.GaussianBlurEffect With {.Source = cmdList, .BlurAmount = 1 + WaveSignal.Energy / 10}
                DrawingSession.DrawImage(blur1)
            End Using
        End Using
    End Sub

private void DrawBackGround(CanvasDrawingSession DrawingSession)
{
    using (cmdList == new CanvasCommandList(DrawingSession)) {
        using (dl == cmdList.CreateDrawingSession) {
            dl.DrawImage(ImageManager.GetResource(ImageResourceID.back1));
        }
        using (blur1 == new Effects.GaussianBlurEffect {Source = cmdList,BlurAmount = 1 + WaveSignal.Energy / 10}) {
            DrawingSession.DrawImage(blur1);
        }
    }
}

　　文字

　　加入歌曲名称信息。

　　具体的文字效果由你设置，请参考 Win2D 的 Microsoft.Graphics.Canvas.Effects 文档。

Imports Microsoft.Graphics.Canvas
Imports Windows.UI
Public Class StaticStringView
    Inherits TypedGameView(Of StaticString)
    Public Sub New(Target As StaticString)
        MyBase.New(Target)
    End Sub
    Public Overrides Sub Draw(DrawingSession As CanvasDrawingSession)
        Using cmdList = New CanvasCommandList(DrawingSession)
            Using dl = cmdList.CreateDrawingSession
                DrawText(dl)
            End Using
            Using blur1 = New Effects.GaussianBlurEffect With {.Source = cmdList, .BlurAmount = 3}
                DrawingSession.DrawImage(blur1)
                DrawingSession.DrawImage(cmdList)
            End Using
        End Using
    End Sub
    Dim TextFormat = New Text.CanvasTextFormat() With {.FontFamily = "微软雅黑",
                                              .FontSize = 12,
                                              .HorizontalAlignment = Microsoft.Graphics.Canvas.Text.CanvasHorizontalAlignment.Center,
                                              .VerticalAlignment = Microsoft.Graphics.Canvas.Text.CanvasVerticalAlignment.Center}
    Public Sub DrawText(Dl As CanvasDrawingSession)
        Dl.DrawText(Target.Str, Target.Location, Colors.White, TextFormat)
    End Sub
End Class

using Microsoft.VisualBasic;
using System;
using System.Collections;
using System.Collections.Generic;
using System.Data;
using System.Diagnostics;
using Microsoft.Graphics.Canvas;
using Windows.UI;
public class StaticStringView : TypedGameView<StaticString>
{
    public StaticStringView(StaticString Target) : base(Target)
    {
    }
    public override void Draw(CanvasDrawingSession DrawingSession)
    {
        using (cmdList == new CanvasCommandList(DrawingSession)) {
            using (dl == cmdList.CreateDrawingSession) {
                DrawText(dl);
            }
            using (blur1 == new Effects.GaussianBlurEffect {Source = cmdList,BlurAmount = 3}) {
                DrawingSession.DrawImage(blur1);
                DrawingSession.DrawImage(cmdList);
            }
        }
    }
     TextFormat = new Text.CanvasTextFormat {
        FontFamily = "微软雅黑",
        FontSize = 12,
        HorizontalAlignment = Microsoft.Graphics.Canvas.Text.CanvasHorizontalAlignment.Center,
        VerticalAlignment = Microsoft.Graphics.Canvas.Text.CanvasVerticalAlignment.Center
    };
    public void DrawText(CanvasDrawingSession Dl)
    {
        Dl.DrawText(Target.Str, Target.Location, Colors.White, TextFormat);
    }
}

图4-1 最终效果

附录

　　开源：MusicVideoGenerator （已失效）

　　相同类型的MV：

　　Over The Horizon

　　Tuesdays

　　Supernova(Original Mix)

　　Lovers

　　Life