Is it possible to pre-generate an array from Spectrum Data?

I'd like to implement something similar to. this in my game. While browsing through Unity's documentation I noticed it says:
Returns a block of the currently playing source's spectrum data

Code (csharp):

1. GetSpectrumData(samples: float[], channel: int, window: FFTWindow): void;
2. [B]Description[/B]
3. Returns a block of the currently playing source's spectrum data.

Would it be possible to pre-analyze a song's spectrum data and then map the floats as an array to later iterate through a procedural mesh?

something along the lines of

Code (csharp):

1. foreach (float in spectrumdata)
2. {
3. terrain.setHeight(xPosition,yPosition, BaseHeight * spectrumdata[i]);
4. }
5.  

should get the job done but I'm developing for mobile so analyzing in real-time seems very expensive. Pre-analyzing would also allow me to smooth/average out the floats from the beginning to the end of the song, for a smoother experience.

 

Seems like you can't get spectrum data unless the audio is ACTUALLY playing (ie, i tried setting source time, and then GetSpectrum, but its all zeros)
So your probably stuck with (something like) playing the clip in edit mode, and using editor scripts to capture the data, here's a starting point for you:

Code (csharp):

1. using UnityEngine;
2. using UnityEditor;
3. using System.Collections;
4.  
5. public class MeshSpectrum : EditorWindow {
6.  
7.     Mesh mesh;
8.     AudioSource source;
9.     AudioClip clip;
10.  
11.     Vector3[] meshVerts;
12.     float readInterval, startTime, lastTime, min, max;
13.     bool reading;
14.     Vector3 meshSize = new Vector3(10,2,50), meshSizeOld;
15.     int currentIndex, widthResolution = 64, lengthResolution = 100;
16.     int[] widthOptions = new int[] { 64, 128, 256, 512, 1024, 2048, 4096, 8192 };
17.     string[] widthOptionsStrings = new string[] { "64", "128", "256", "512", "1024", "2048", "4096", "8192" };
18.  
19.     [MenuItem( "EDITORS/Mesh Spectrum" )]
20.     static void Init () {
21.         MeshSpectrum window = (MeshSpectrum) EditorWindow.GetWindow( typeof( MeshSpectrum ) );
22.         window.Show();
23.         window.position = new Rect( 20, 80, 200, 193 );
24.     }
25.  
26.     void Update () {
27.         if ( reading ) {
28.             if ( Time.realtimeSinceStartup - readInterval > lastTime ) {
29.                 float[] spectrum = new float[widthResolution];
30.                 source.GetSpectrumData( spectrum, 0, FFTWindow.BlackmanHarris );
31.                 for ( int x = 0; x < widthResolution; x++ ) {
32.                     float f = Mathf.Max( Mathf.Log( spectrum[x] ), -10 );
33.                     if ( f > max ) max = f;
34.                     if ( f < min ) min = f;
35.                     meshVerts[currentIndex * widthResolution + x] = new Vector3( ( x / ( widthResolution - 1f ) ) * meshSize.x, f, ( currentIndex / ( lengthResolution - 1f ) ) * meshSize.z );
36.                 }
37.                 SceneView.RepaintAll();
38.                 lastTime = Time.realtimeSinceStartup;
39.                 currentIndex++;
40.                 if ( currentIndex == lengthResolution || !source.isPlaying ) {
41.                     float scale = 1f / ( max - min );
42.                     Vector2[] uvs = new Vector2[lengthResolution * widthResolution];
43.                     for ( int y = 0; y < lengthResolution; y++ ) {
44.                         for ( int x = 0; x < widthResolution; x++ ) {
45.                             meshVerts[y * widthResolution + x].y = ( meshVerts[y * widthResolution + x].y - min ) * scale * meshSize.y;
46.                             uvs[y * widthResolution + x] = new Vector2( x / ( widthResolution - 1f ), y / ( lengthResolution - 1f ) );
47.                         }
48.                     }
49.  
50.  
51.                     int[] tris = new int[( widthResolution - 1 ) * ( lengthResolution - 1 ) * 6];
52.  
53.                     for ( int y = 0; y < lengthResolution - 1; y++ ) {
54.                         for ( int x = 0; x < widthResolution - 1; x++ ) {
55.                             tris[( ( y * ( widthResolution - 1 ) ) + x ) * 6 + 0] = y * widthResolution + x;
56.                             tris[( ( y * ( widthResolution - 1 ) ) + x ) * 6 + 1] = ( y + 1 ) * widthResolution + x;
57.                             tris[( ( y * ( widthResolution - 1 ) ) + x ) * 6 + 2] = y * widthResolution + x + 1;
58.  
59.                             tris[( ( y * ( widthResolution - 1 ) ) + x ) * 6 + 3] = y * widthResolution + x + 1;
60.                             tris[( ( y * ( widthResolution - 1 ) ) + x ) * 6 + 4] = ( y + 1 ) * widthResolution + x;
61.                             tris[( ( y * ( widthResolution - 1 ) ) + x ) * 6 + 5] = ( y + 1 ) * widthResolution + x + 1;
62.                         }
63.                     }
64.                     mesh.Clear();
65.                     mesh.vertices = meshVerts;
66.                     mesh.triangles = tris;
67.                     mesh.uv = uvs;
68.                     mesh.RecalculateNormals();
69.                     reading = false;
70.                     source.Stop();
71.                     currentIndex = 0;
72.                 }
73.             }
74.         }
75.     }
76.  
77.     void OnFocus () {
78.         SceneView.onSceneGUIDelegate -= this.OnSceneGUI;
79.         SceneView.onSceneGUIDelegate += this.OnSceneGUI;
80.     }
81.  
82.     void OnDestroy () {
83.         SceneView.onSceneGUIDelegate -= this.OnSceneGUI;
84.     }
85.  
86.     void OnSceneGUI ( SceneView sceneView ) {
87.         Vector3[] verts = new Vector3[] { new Vector3( 0, 0, 0 ), new Vector3( meshSize.x, 0, 0 ), new Vector3( meshSize.x, 0, meshSize.z ), new Vector3( 0, 0, meshSize.z ), new Vector3( 0, meshSize.y, meshSize.z ), meshSize, new Vector3( meshSize.x, meshSize.y, 0 ), new Vector3( 0, meshSize.y, 0 ) };
88.         for ( int i = 0; i < 8; i++ ) {
89.             Handles.DrawLine( verts[i], verts[( i + 1 ) % 8] );
90.         }
91.         Handles.DrawLine( verts[0], verts[3] );
92.         Handles.DrawLine( verts[4], verts[7] );
93.         Handles.DrawLine( verts[2], verts[5] );
94.         Handles.DrawLine( verts[1], verts[6] );
95.     }
96.  
97.     void OnGUI () {
98.         EditorGUIUtility.labelWidth = 90;
99.         source = (AudioSource) EditorGUILayout.ObjectField( "Audio source:", source, typeof( AudioSource ), true );
100.         mesh = (Mesh) EditorGUILayout.ObjectField( "Terrain:", mesh, typeof( Mesh ), true );
101.         meshSize = EditorGUILayout.Vector3Field( "Approx size of mesh", meshSize );
102.         if ( meshSizeOld != meshSize ) {
103.             meshSizeOld = meshSize;
104.             SceneView.RepaintAll();
105.         }
106.         widthResolution = EditorGUILayout.IntPopup( "X resolution", widthResolution, widthOptionsStrings, widthOptions );
107.         lengthResolution = Mathf.Max( EditorGUILayout.IntField( "Z resolution", lengthResolution ), 3 );
108.         if ( GUILayout.Button( "I need a mesh!" ) ) {
109.             mesh = new Mesh();
110.             AssetDatabase.CreateAsset( mesh, "Assets/SpectrumMesh.asset" );
111.             Selection.activeObject = mesh;
112.             EditorGUIUtility.PingObject( mesh );
113.         }
114.         GUI.enabled = mesh != null  source != null;
115.         if ( GUILayout.Button( "Play and Create Mesh!" ) ) {
116.             float sourceLength = source.clip.length;
117.             readInterval = sourceLength / lengthResolution;
118.             lastTime = Time.realtimeSinceStartup;
119.             currentIndex = 0;
120.             max = float.MinValue;
121.             min = float.MaxValue;
122.             meshVerts = new Vector3[lengthResolution * widthResolution];
123.             reading = true;
124.             source.Play();
125.         }
126.         GUI.enabled = reading;
127.         if ( GUILayout.Button( "Stop" ) ) {
128.             source.Stop();
129.             reading = false;
130.             currentIndex = 0;
131.         }
132.  
133.         Rect rect = GUILayoutUtility.GetRect( new GUIContent( "" ), "Button" );
134.         GUI.Box( rect, "" );
135.         if ( reading ) {
136.             GUI.color = new Color(0.45f, 0.6f, 0.85f);
137.             GUI.Box( new Rect( rect.x, rect.y, rect.width * ( (float) currentIndex / lengthResolution ), rect.height ), "" );
138.             GUI.Label( rect, "Working..." );
139.  
140.             this.Repaint();
141.         }
142.     }
143. }
144.  
145.  

Open the editor window by clicking EDITORS->Mesh Spectrum.
Drag the audio source and a mesh asset into the fields, set values, and click the play button. The clip will play (you might have to toggle audio on in the editor to hear it; you might not actually have to hear it for it to work), and when finished (it takes the length of time that the clip is), the mesh will be set.

You might want to do something with the values, on line 32 I simple take the Log of the spectrum value.
You might also want to do some smoothing, I dunno.

 

You really wouldn't know until you try it on your target platforms. Even it where expensive, you don't have to grab a sample every frame. Every 10 frame, or even every second might still give you acceptable visuals. "Premature optimization is the root of all evil."

I say try it in real time, and if that doesn't meet your performance requirements, you can them look into pre-calculations.

 

Regarding the runtime solution, it's going to look like this:

Code (csharp):

1. using UnityEngine;
2. using System.Collections;
3.  
4. public class RuntimeSpectrum : MonoBehaviour {
5.  
6.     public MeshFilter targetMeshFilter;
7.     public AudioSource audioSource;
8.     public widths widthResolutionChoice = 0;
9.     public int lengthResolution = 100;
10.     public float meshSizeX = 10, meshSizeZ = 50;
11.  
12.     Mesh mesh;
13.     Vector3[] meshVerts;
14.     float readInterval, lastTime;
15.     int currentIndex, widthResolution;
16.  
17.     public enum widths { o64, o128, o256, o512, o1024, o2048, o4096, o8192 };
18.     int[] widthOptions = new int[] { 64, 128, 256, 512, 1024, 2048, 4096, 8192 };
19.  
20.     void Start () {
21.         widthResolution = widthOptions[(int) widthResolutionChoice];
22.         mesh = targetMeshFilter.mesh;
23.         if ( mesh == null ) {
24.             mesh = new Mesh();
25.             targetMeshFilter.mesh = mesh;
26.         }
27.         float sourceLength = audioSource.clip.length;
28.         readInterval = sourceLength / lengthResolution;
29.         lastTime = 0;
30.         currentIndex = 0;
31.         meshVerts = new Vector3[lengthResolution * widthResolution];
32.         Vector2[] uvs = new Vector2[lengthResolution * widthResolution];
33.         for ( int y = 0; y < lengthResolution; y++ ) {
34.             for ( int x = 0; x < widthResolution; x++ ) {
35.                 meshVerts[y * widthResolution + x] = new Vector3( ( x / ( widthResolution - 1f ) ) * meshSizeX, 0, ( y / ( lengthResolution - 1f ) ) * meshSizeZ );
36.                 uvs[y * widthResolution + x] = new Vector2( x / ( widthResolution - 1f ), y / ( lengthResolution - 1f ) );
37.             }
38.         }
39.  
40.         int[] tris = new int[( widthResolution - 1 ) * ( lengthResolution - 1 ) * 6];
41.         for ( int y = 0; y < lengthResolution - 1; y++ ) {
42.             for ( int x = 0; x < widthResolution - 1; x++ ) {
43.                 tris[( ( y * ( widthResolution - 1 ) ) + x ) * 6 + 0] = y * widthResolution + x;
44.                 tris[( ( y * ( widthResolution - 1 ) ) + x ) * 6 + 1] = ( y + 1 ) * widthResolution + x;
45.                 tris[( ( y * ( widthResolution - 1 ) ) + x ) * 6 + 2] = y * widthResolution + x + 1;
46.  
47.                 tris[( ( y * ( widthResolution - 1 ) ) + x ) * 6 + 3] = y * widthResolution + x + 1;
48.                 tris[( ( y * ( widthResolution - 1 ) ) + x ) * 6 + 4] = ( y + 1 ) * widthResolution + x;
49.                 tris[( ( y * ( widthResolution - 1 ) ) + x ) * 6 + 5] = ( y + 1 ) * widthResolution + x + 1;
50.             }
51.         }
52.         mesh.Clear();
53.         mesh.vertices = meshVerts;
54.         mesh.triangles = tris;
55.         mesh.uv = uvs;
56.         mesh.RecalculateNormals();
57.         currentIndex = 0;
58.     }
59.  
60.     void Update () {
61.         if ( Time.time - readInterval > lastTime  audioSource.isPlaying ) {
62.             float[] spectrum = new float[widthResolution];
63.             audioSource.GetSpectrumData( spectrum, 0, FFTWindow.BlackmanHarris );
64.             for ( int x = 0; x < widthResolution; x++ ) {
65.                 float f = Mathf.Max( Mathf.Log( spectrum[x] ), -10 ) / 8f + 1.2f;
66.                 meshVerts[currentIndex * widthResolution + x] = new Vector3( ( x / ( widthResolution - 1f ) ) * meshSizeX, f, ( currentIndex / ( lengthResolution - 1f ) ) * meshSizeZ );
67.             }
68.             mesh.vertices = meshVerts;
69.  
70.             lastTime = Time.time;
71.             currentIndex++;
72.         }
73.     }
74. }
75.  

Note that because you aren't running through all values before assigning the heights, you can't properly normalize (min and max in the editor solution) so Line 65 is just a guess at normalizing the values from the clip I was testing with.

 

thank you. this is a great starting point.