WebGL with streaming option - like WebPlayer

Hey,

While we worked on a WebGL port for our game, we needed an option to stream levels like in the WebPlayer.
@jonas echterhoff wrote a basic script that did that on the beta version of Unity5.
We updated it, and added the option to check download progress.

Instead of Application.CanStreamedLevelBeLoaded() and Application.GetStreamProgressForLevel()
We now have
WebGLLevelStreaming.CanStreamedLevelBeLoaded()
WebGLLevelStreaming.GetStreamProgressForLevel()

Use "firstStreamedLevelWithResources" like in the "First Streamed Level" option of the WebPlayer
http://docs.unity3d.com/Manual/class-PlayerSettingsWeb.html

So, the code for WebGLLevelStreaming.cs

Code (CSharp):

1. using UnityEngine;
2. #if UNITY_EDITOR
3. using UnityEditor;
4. using UnityEditor.Callbacks;
5. #endif
6. using System.Collections;
7. using System.Collections.Generic;
8. using System.IO;
9. using System.Diagnostics;
10. using System.Linq;
11. using System.Runtime.InteropServices;
12.  
13.  
14. public class WebGLLevelStreaming {
15.  
16. #if UNITY_EDITOR && UNITY_WEBGL
17.     private const string kOutputDataExtension = ".data";
18.     private const string kOutputFileLoaderFileName = "fileloader.js";
19.     private const string kResourcesDirName = "Resources";
20.     private const string kResourcesFileName = "unity_default_resources";
21.     private const string kResourcesExtraFileName = "unity_builtin_extra";
22.  
23.     public static string buildToolsDir
24.     {
25.         get {
26.             // Find WebGL build tools
27.             var webGLBuildTools = Path.Combine("WebGLSupport", "BuildTools");
28.             var playbackEngines = Path.Combine(EditorApplication.applicationContentsPath, "PlaybackEngines");
29.             var path = Path.Combine(playbackEngines, webGLBuildTools);
30.             if (Directory.Exists(path))
31.                 return path;
32.             else
33.                 return Path.Combine(Path.Combine(EditorApplication.applicationPath, "../../"), webGLBuildTools);
34.         }
35.     }
36.  
37.     public static string emscriptenDir
38.     {
39.         get { return buildToolsDir + "/Emscripten"; }
40.     }
41.  
42.     static string packager
43.     {
44.         get { return emscriptenDir + "/" + "tools/file_packager.py"; }
45.     }
46.  
47.     public static string pythonExecutable
48.     {
49.         get
50.         {
51.             if (Application.platform == RuntimePlatform.WindowsEditor)
52.                 return "\"" +  buildToolsDir + "/" + "Emscripten_Win/python/2.7.5.3_64bit/python.exe\"";
53.             return "python";
54.         }
55.     }
56.  
57.     public static int firstStreamedLevelWithResources = 1;
58.  
59.     // Run emscripten file packager to pack a .data file
60.     private static bool RunPackager (string filename, string stagingAreaData, IEnumerable<string> filesToShip)
61.     {
62.         var argumentsForPacker = "\"" + packager + "\"" ;
63.         argumentsForPacker += string.Format(" \"{0}\"", filename + kOutputDataExtension);
64.         argumentsForPacker += " --no-heap-copy";
65.         argumentsForPacker += " --js-output=\"" + filename + ".loader.js" + "\"";
66.  
67.         if (PlayerSettings.GetPropertyBool("dataCaching", BuildTargetGroup.WebGL))
68.             argumentsForPacker += " --use-preload-cache";
69.  
70.         argumentsForPacker += " --preload";
71.         argumentsForPacker += filesToShip.Aggregate("", (current, file) => current + (" \"" + file + "\""));
72.  
73.         var processStartInfo = new ProcessStartInfo(pythonExecutable)
74.         {
75.             Arguments = argumentsForPacker,
76.             WorkingDirectory = stagingAreaData,
77.             UseShellExecute = false,
78.         };
79.  
80.         var p = Process.Start(processStartInfo);
81.         p.WaitForExit();
82.         if (p.ExitCode == 0)
83.             return true;
84.  
85.         throw new System.Exception("Failed running " + processStartInfo.FileName + " " + processStartInfo.Arguments);
86.     }
87.    
88.     private static bool PackageData (string filename, string stagingAreaData, IEnumerable<string> _filesToShip)
89.     {
90.         var filesToShip = new HashSet<string>(_filesToShip.Select(o => Path.GetFileName(o)));
91.         filesToShip.Add(Path.Combine(kResourcesDirName, kResourcesFileName));
92.         filesToShip.Add(Path.Combine(kResourcesDirName, kResourcesExtraFileName));
93.  
94.         if (firstStreamedLevelWithResources < 0)
95.             firstStreamedLevelWithResources = 0;
96.         if (firstStreamedLevelWithResources >= Application.levelCount)
97.             firstStreamedLevelWithResources = Application.levelCount-1;
98.         var loaderJSStart = "var StreamProgressForLevelArray = [];\nStreamProgressForLevelArray[0]=1;\n";
99.         var loaderJS = "";
100.         var loaderJS0 = "";
101.  
102.         int packageIndex = 0;
103.         // Generate a data file for each streaming level
104.         while (true)
105.         {
106.             // find all the files for this level
107.             var fileNames = new List<string>();
108.             int levelIndex = packageIndex;
109.             fileNames.Add("sharedassets" + levelIndex + ".resource");
110.             fileNames.Add("sharedassets" + levelIndex + ".assets");
111.             if(packageIndex>0)
112.             {
113.                 fileNames.Add("level" + (levelIndex-1) + "");
114.             }else{
115.                 fileNames.Add(Path.Combine(kResourcesDirName, kResourcesFileName));
116.                 fileNames.Add(Path.Combine(kResourcesDirName, kResourcesExtraFileName));
117.                 fileNames.Add("mainData");
118.             }
119.  
120.             if(levelIndex==firstStreamedLevelWithResources)
121.             {
122.                 fileNames.Add("resources.assets");
123.                 fileNames.Add("resources.resource");
124.             }
125.  
126.             var files = new List<string>();
127.             foreach (var f in fileNames)
128.             {
129.                 if (filesToShip.Contains(f))
130.                 {
131.                     filesToShip.Remove(f);
132.                     files.Add(f);
133.                 }
134.             }
135.             if (files.Count == 0)
136.                 break;
137.  
138.             var packageFile = filename + "." + packageIndex;
139.             if (!RunPackager(packageFile, stagingAreaData, files))
140.                 return false;
141.  
142.             // Extract loader script for this data file, and put it's content into a JavaScript function
143.             if(packageIndex>0)
144.             {
145.                 var loaderText = File.ReadAllText(Path.Combine(stagingAreaData, packageFile + ".loader.js"));
146.                 loaderText = loaderText.Substring(loaderText.IndexOf("\n(function() {"));
147.                 loaderText = loaderText.Replace("if (Module['setStatus']) Module['setStatus']('Downloading data... (' + loaded + '/' + total + ')');",
148.                                                 "StreamProgressForLevelArray["+packageIndex+"]=event.loaded/size;");
149.                 loaderJS += "function DownloadDataForPackage" + packageIndex + "(){\n" + loaderText + "}\n";
150.                 loaderJSStart +="StreamProgressForLevelArray[" + packageIndex + "]=0;\n";
151.             }else{
152.                 loaderJS0 = File.ReadAllText(Path.Combine(stagingAreaData, filename + ".0.loader.js"));
153.                 loaderJS0 = loaderJS0.Replace("if (Module['setStatus']) Module['setStatus']('Downloading data... (' + loaded + '/' + total + ')');",
154.                                               "if (Module['setStatus']) Module['setStatus']('Downloading data... (' + event.loaded + '/' + size + ')');");
155.             }
156.             packageIndex++;
157.         }
158.  
159.         if(filesToShip.Count>0)
160.         {
161.             var packageFile = filename + "." + packageIndex;
162.             if (!RunPackager(packageFile, stagingAreaData, filesToShip))
163.                 return false;
164.             var loaderText2 = File.ReadAllText(Path.Combine(stagingAreaData, packageFile + ".loader.js"));
165.             loaderText2 = loaderText2.Substring(loaderText2.IndexOf("\n(function() {"));
166.             loaderJS += "function DownloadDataForPackage" + packageIndex + "(){\n" + loaderText2 + "}\n";
167.             packageIndex++;
168.         }
169.  
170.         loaderJS = loaderJSStart + loaderJS + loaderJS0;
171.  
172.         for (int j=0; j < packageIndex-1; j++)
173.         {
174.             // Patch loader scripts to invoke the function to start loading the next data file when it's done
175.             var line = "Module['removeRunDependency']('datafile_" + filename + "." + j + ".data');";
176.             // Call function to load next data file in a callback to avoid calling it while clearing pre-run dependencies.
177.             loaderJS = loaderJS.Replace (line, line + "\nStreamProgressForLevelArray[" + j + "]=1;\nwindow.setTimeout(DownloadDataForPackage" + (j+1) + @",1);");
178.         }
179.  
180.         // write out loader script for all data files
181.         File.WriteAllText(Path.Combine(stagingAreaData, kOutputFileLoaderFileName), loaderJS);
182.         return true;
183.     }
184.  
185.     // Generate gzip compressed versions of files
186.     private static void CompressFilesInOutputDirectory (string dir, string outputDir)
187.     {
188.         var filesToCompress = Directory.GetFiles(dir).Where(f => f.EndsWith(kOutputFileLoaderFileName) || f.EndsWith(".data"));
189.         foreach (var file in filesToCompress)
190.         {
191.             var processName = "7za";
192.             if (Application.platform == RuntimePlatform.WindowsEditor)
193.                 processName = "7z.exe";
194.             var processStartInfo = new ProcessStartInfo(EditorApplication.applicationContentsPath+"/Tools/"+processName)
195.             {
196.                 Arguments = "a -tgzip \""+Path.Combine(outputDir, Path.GetFileName(file))+"gz\" \""+file+"\"",
197.                 UseShellExecute = false,
198.                 CreateNoWindow = true
199.             };
200.  
201.             var p = Process.Start(processStartInfo);
202.             p.WaitForExit();
203.             if (p.ExitCode != 0)
204.                 throw new System.Exception("Failed running " + processStartInfo.FileName + " " + processStartInfo.Arguments);
205.         }
206.     }
207.  
208.     [PostProcessBuild]
209.     public static void OnPostprocessBuild(BuildTarget target, string pathToBuiltProject) {
210.         if (target != BuildTarget.WebGL)
211.             return;
212.         var stagingAreaData = Path.Combine(Path.Combine("Temp", "StagingArea"), "Data");
213.         var filename = Path.GetFileName(pathToBuiltProject);
214.  
215.         // Find all files to package
216.         IEnumerable<string> filesToShip = Directory.GetFiles(stagingAreaData, "*.resource");
217.         filesToShip = filesToShip.Concat(Directory.GetFiles(stagingAreaData, "*.assets"));
218.         filesToShip = filesToShip.Concat(Directory.GetFiles(stagingAreaData, "mainData"));
219.         filesToShip = filesToShip.Concat(Directory.GetFiles(stagingAreaData, "level*"));
220.  
221.         // put files into packages
222.         PackageData (filename, stagingAreaData, filesToShip);
223.  
224.         // delete old data file
225.         File.Delete (Path.Combine(pathToBuiltProject, Path.Combine("Release", filename+".data")));
226.  
227.         // copy new data files into build
228.         File.Copy(Path.Combine(stagingAreaData, kOutputFileLoaderFileName), Path.Combine(Path.Combine(pathToBuiltProject, "Release"),kOutputFileLoaderFileName), true);
229.         foreach (var f in Directory.GetFiles(stagingAreaData, "*.*.data"))
230.             File.Copy(f, Path.Combine(Path.Combine(pathToBuiltProject, "Release"),Path.GetFileName(f)), true);
231.  
232.         // compress new data files
233.         CompressFilesInOutputDirectory (Path.Combine(pathToBuiltProject, "Release"), Path.Combine(pathToBuiltProject, "Compressed"));
234.  
235.         // delete old compressed data file
236.         File.Delete (Path.Combine(pathToBuiltProject, Path.Combine("Compressed", filename+".datagz")));
237.     }
238. #endif
239.  
240. #if UNITY_WEBGL
241.     [DllImport("__Internal")]
242.     private static extern float GetStreamProgressForLevelFromWeb(int levelIndex);
243. #endif
244.  
245.     static public bool CanStreamedLevelBeLoaded (int levelIndex)
246.     {
247. #if UNITY_WEBGL && !UNITY_EDITOR
248.         if (levelIndex == 0) // file = mainData
249.             return true;
250.         return File.Exists("level"+(levelIndex-1));
251. #else
252.         return Application.CanStreamedLevelBeLoaded(levelIndex);
253. #endif  
254.     }
255.  
256.     static public float GetStreamProgressForLevel (int levelIndex)
257.     {
258.         #if UNITY_WEBGL && !UNITY_EDITOR
259.         if (levelIndex == 0) // file = mainData
260.             return 1;
261.         return GetStreamProgressForLevelFromWeb(levelIndex);
262.         #else
263.         return Application.GetStreamProgressForLevel(levelIndex);
264.         #endif  
265.     }
266. }

And code for the JS file, WebGLLevelStreamingJS.jslib

Code (JavaScript):

1. var WebGLLevelStreamingJS = {
2.     GetStreamProgressForLevelFromWeb: function(levelIndex)
3.     {
4.         return StreamProgressForLevelArray[levelIndex];
5.     }
6. };
7.  
8. mergeInto(LibraryManager.library, WebGLLevelStreamingJS);

**IMPORTANT**
Tested on Unity5.0.0f4.
Unity still working on the WebGL build process, so this script may not work on versions newer than 5.0.0f4.

Known Issues:
GetStreamProgressForLevel() return 0 if loading from cache.
Some other issues with the progress that we need to check, but weren't critical for us.

You all are welcome to improve the script

 

srry for being off-topic on your problem, De-Panther, but could you mention what is your game about? Type, genre, online features, etc?

 

Hey,

The script is very useful, thanks!
For those who is not aware where is is coming from here is the link to the presentation.

De-Panther, the presentation also contains very interesting slide #25, that says how you dealt with textures and resources. Could you please share details?

Is that possible to share build script?

 

Can any1 tell me how to make this work??
I Used the 1st script and it did split files.
But what to do with 2nd script.

When i tested not all objects in level where loaded.

 

@slmoloch I can't share those, as they are specific for the client's project.
But maybe @liortal can explain how it works - he worked on the build script.
I just remember that it's something about searching on the scene files for specific textures and replace them with URL to download.
And about removing resources and scenes - we move them to the "Editor" folder while the build proccess is running, and move them back when it finished.

@skeleton king , just put the "WebGLLevelStreamingJS.jslib" file under Assets/Plugins/WebGL/

 

We have actually created a build infrastructure (see the menu item in the screenshot) that builds the game.

This takes into account a customized build properties file that has a few pieces of data:

1. Excluded scenes - this is a collection of scenes (dragged from the project window) that will not be included in the final output. The build process calls BuildPipeline.BuildPlayer; this method accepts an array of scenes, so we can easily pass in the scenes we want to build.
2. Excluded resources - this is a collection of resources (files / folders under a Resources folder) that also will be excluded from the build. These can be stuff that is unused in the game, so there's no point of including them in the final game build. Since there's no way to "exclude" resources, we do a little trick - we just move these into a folder that is not under "Resources" and then move them back when the build is done.
3. Remove texture usage - for some textures that were too large, we actually incorporated a small trick - we don't include them in the game, but at runtime we download them from a server. This is done automatically by the build process - we have a data file that describes which textures we'd like to "patch" in this way. The build process looks for all of these textures, and removes them (replacing them with a special component that knows how to fetch them at runtime).

At this point, the build process was created for this project in our company (Moon Active), i don't think we can share it.

I have taken the concepts from the design we used for this project and i plan to create a new project (open source?) of build tools for Unity. Watch this space in the future: https://github.com/liortal53/UnityBuildTools

 

Nice, thanks for the prompt reply! I totally understand that you can not share the script, but could you point me to the direction on where to find the sample of component that downloads the texture?

 

Looking forward for updates to that repository. When do you expect to add something there?

 

The component that downloads textures is a super basic one.

It has a public URL field that is set from the editor, and in its Awake method it downloads the texture at the given URL, creates a Sprite object and assigns it to the SpriteRenderer.

 

Very Cool! I need this for my project, but I'm currently using Unity 5.0.2. Any idea if this will work there?

Also has the "GetStreamProgressForLevel() return 0 if loading from cache." error been resolved? If not does it actually cause a problem in the WebGL build for the player to move forward with loading the game or level even if it was cached?

I think this should be an officially added feature as with all the Memory Allocation issues WebGL has currently, something like this is critical for us to build and deploy WebGL games in the current state.

 

Should be fine, although we haven't tested it with that particular version yet.

 

Thanks @Kuroato
GetStreamProgressForLevel() should return 1 now if loading from cache, but I suggest you should also check it yourself.

Please reply if it work or didn't work

 

Cool Thanks! Good to know!

 

Hi De-Panther,
When i compiled your WebGLLevelStreaming.cs script from unity 5.1,i get huge python errors.

Failed running python "/Applications/Unity5.1/Unity.app/Contents/PlaybackEngines/WebGLSupport/BuildTools/Emscripten/emcc" @"/Users/apple/Downloads/GameFolder/Game/Assets/../Temp/emcc_arguments.resp"

stdout:
stderr:
warning: unresolved symbol: glFlushMappedBufferRange
warning: unresolved symbol: glGetInternalformativ
warning: unresolved symbol: tcflush
warning: unresolved symbol: pthread_create
warning: unresolved symbol: _ZN4FMOD13DSPConnection6setMixEf
warning: unresolved symbol: glUnmapBuffer
warning: unresolved symbol: glGetStringi
warning: unresolved symbol: glProgramBinary
warning: unresolved symbol: glMapBufferRange
warning: unresolved symbol: glGetProgramBinary
warning: unresolved symbol: glCopyBufferSubData
Traceback (most recent call last):
File "/Applications/Unity5.1/Unity.app/Contents/PlaybackEngines/WebGLSupport/BuildTools/Emscripten/emscripten.py", line 1675, in <module>
_main(environ=os.environ)
File "/Applications/Unity5.1/Unity.app/Contents/PlaybackEngines/WebGLSupport/BuildTools/Emscripten/emscripten.py", line 1663, in _main
temp_files.run_and_clean(lambda: main(
File "/Applications/Unity5.1/Unity.app/Contents/PlaybackEngines/WebGLSupport/BuildTools/Emscripten/tools/tempfiles.py", line 39, in run_and_clean
return func()
File "/Applications/Unity5.1/Unity.app/Contents/PlaybackEngines/WebGLSupport/BuildTools/Emscripten/emscripten.py", line 1671, in <lambda>
DEBUG_CACHE=DEBUG_CACHE,
File "/Applications/Unity5.1/Unity.app/Contents/PlaybackEngines/WebGLSupport/BuildTools/Emscripten/emscripten.py", line 1558, in main
jcache=jcache, temp_files=temp_files, DEBUG=DEBUG, DEBUG_CACHE=DEBUG_CACHE)
File "/Applications/Unity5.1/Unity.app/Contents/PlaybackEngines/WebGLSupport/BuildTools/Emscripten/emscripten.py", line 924, in emscript_fast
%s''' % (staticbump, global_initializers, mem_init)) # XXX wrong size calculation!
UnicodeDecodeError: 'ascii' codec can't decode byte 0xc2 in position 308695: ordinal not in range(128)
Traceback (most recent call last):
File "/Applications/Unity5.1/Unity.app/Contents/PlaybackEngines/WebGLSupport/BuildTools/Emscripten/emcc", line 1323, in <module>
final = shared.Building.emscripten(final, append_ext=False, extra_args=extra_args)
File "/Applications/Unity5.1/Unity.app/Contents/PlaybackEngines/WebGLSupport/BuildTools/Emscripten/tools/shared.py", line 1535, in emscripten
assert os.path.exists(filename + '.o.js'), 'Emscripten failed to generate .js'
AssertionError: Emscripten failed to generate .js

UnityEditor.HostView:OnGUI()


Can you help me on this?

 

@Gaurav Gulati I didn't test it on 5.1, so I'm not familiar with this problem.
try @jonas echterhoff 's version from the asset store
https://www.assetstore.unity3d.com/en/#!/content/38368
and if you want to add the feature of GetStreamProgressForLevel compare both of the versions for changes...

 

The asset store script is not working with Unity 5.1.2 - it's working with 5.1.1 - but with the new version I get the error:
Uncaught RangeError: Maximum call stack size exceded

@jonas echterhoff It would be cool if you could check that for me.

 

Very helpfull.Thanks!

 

Starting from 5.1.2 there is a another file that you need to add to the first scene - global-metadata.dat. Just add two lines of code:

filesToShip.Add(Path.Combine(@"Il2CppData\Metadata", "global-metadata.dat"));

and

fileNames.Add(Path.Combine(@"Il2CppData\Metadata", "global-metadata.dat"));

 

Hi, i'm having a bit of an issue, i copied paste the de-panther script and added the 2 lines above, set the "firstStreamedLevelWithResources " to 0 which is my preloader scene, but when i launch the game, only the "webGL.data" file is being downloaded.

The "webGL.0.data", "webGL.1.data" etc... files aren't downloaded so i guess i must have missed something :s. Is there anything else to add, or maybe call a function somewhere ?

I tried to use the version of @jonas echterhoff and added the 2 lines above, but i get the following errors:
- run() called, but dependencies remain, so not running
- Not implemented: Class::GetClassBitmap (mutilple times)
- Not implemented: Il2CppTypeCompare::compare

I noticed that using his version correctly download the "webGL.0.data", "webGL.1.data" files, but generate errors when loading is complete :s.

Any idea how to fix this ? thx for reading

 

Hey @AnonDreamer

1. What Unity version are you on ?
2. What is the output you get after building your project with either scripts? does the script split up the data files properly? I don't think you should have the webgl.data file at all. It should be split into multiple files eventually after running the script.
3. are you getting the errors you posted (eg: GetClassbitmap) when not using the streaming script ?

-Lior

 

Hi, I encountered similar problem as AnonDreamer in which the script works fine on 5.1.1p1, but got the following error when I built it on 5.1.2.

From browser java console:

Not implemented: Class::GetClassBitmap (1xx times)

Cannot enlarge memory arrays. Either (1) compile with -s TOTAL_MEMORY=X with X higher than the current value 536870912, (2) compile with ALLOW_MEMORY_GROWTH which adjusts the size at runtime but prevents some optimizations, or (3) set Module.TOTAL_MEMORY before the program runs.

.
.
.



Thanks & Regards,
Rick

 

@liortal
1 - I'm currently using Unity 5.1.2f1, building fast development webGL builds.
2 - When using the @jonas echterhoff version, i get webGL.0.data, webGL.1.data, webGL.2.data files, the files gets downloaded but in the javascript console, i get multiple "Not implemented: Class::GetClassBitmap" errors and "Not implemented: Il2CppTypeCompare::compare" errors, i have no error in the unity console after the build is done.

When using the de-panther version, i have one error after the build is finished : DirectoryNotFoundException: Could not find a part of the path "C:/Users/Admin/Desktop/builds_test/webGL\Release\webGL.data", but the streaming seems to works fine when i make a release build instead of a development one. (i only have one "webGL.data" file if i make a development build)
Also what does the WebGLLevelStreamingJS.jslib script do ?


3 - If i don't use either streaming scripts, i don't get any error on the output and the builds works just fine.

So it seems the de-panther script works for me but only if i don't use the development build.

@Rick Lee the "cannot enlarge memory arrays" can be solved if you either increase the "TOTAL_MEMORY" value in the index.html file or you do it by going to "player settings" in the build window, then Publishing Settings and you modify the WebGL Memory Size value.
If your browser displays an error like "cannot assign the required memory", try to make a fast build instead of a slow one. I kept getting this error in chrome (even with 256mo) unless i made "fast" builds.
For the "Not implemented: Class::GetClassBitmap ", i don't have a clue how to solve it :\

 

What line of code exactly throws the error your mentioned ? (e.g: directory not found?)

Also, i believe the script is hard-wired to use Release builds only (not sure about the original script by @jonas echterhoff)

You can see that the script is inserting the string "Release" in multiple locations. I *THINK* it shouldn't be too difficult to make some changes so it'll support development builds (unless there's something internal that prevents dev builds from using streaming).

Lastly, the WebGLLevelStreaming.jslib file is a file containing javascript code that gets included into the final JS code of your game.

You can see that the WebGLStreaming class calls into it in the GetStreamProgressForLevel method, to check up on the progress for loading a level.

 

The Version 5.1.1 was working fine. But with 5.1.2 still not working. We use the original Script from the Asset Store.
We included the 2 lines here:

Code (CSharp):

1.  
2.      filesToShip.Add(Path.Combine(kResourcesDirName, kResourcesExtraFileName));
3.      filesToShip.Add(Path.Combine(@"Il2CppData\Metadata", "global-metadata.dat"));
4.  
5.      fileNames.Add("level" + (levelIndex-1) + "");
6.      fileNames.Add(Path.Combine(@"Il2CppData\Metadata", "global-metadata.dat"));
7.  

But still get errors in Browser console and the build doesn't load. Error in Chrome is:
Not implemented: Class::FromIl2CppType

Any idea?

 

Is there any stack you are getting with this error? also is that a dev or release build? (i assume release since the script doesn't handle dev builds.. ?)

 

@Madsen i didn't make the same changes as you did, here's the de-panther file with the 2 lines of code added:

Code (CSharp):

1.  
2. private static bool PackageData (string filename, string stagingAreaData, IEnumerable<string> _filesToShip)
3.     {
4.         var filesToShip = new HashSet<string>(_filesToShip.Select(o => Path.GetFileName(o)));
5.         filesToShip.Add(Path.Combine(kResourcesDirName, kResourcesFileName));
6.         filesToShip.Add(Path.Combine(kResourcesDirName, kResourcesExtraFileName));
7.         filesToShip.Add(Path.Combine(@"Il2CppData\Metadata", "global-metadata.dat")); //Modification for 5.1.2 version
8.  
9.         if (firstStreamedLevelWithResources < 0)
10.             firstStreamedLevelWithResources = 0;
11.         if (firstStreamedLevelWithResources >= Application.levelCount)
12.             firstStreamedLevelWithResources = Application.levelCount-1;
13.         var loaderJSStart = "var StreamProgressForLevelArray = [];\nStreamProgressForLevelArray[0]=1;\n";
14.         var loaderJS = "";
15.         var loaderJS0 = "";
16.  
17.         int packageIndex = 0;
18.         // Generate a data file for each streaming level
19.         while (true)
20.         {
21.             // find all the files for this level
22.             var fileNames = new List<string>();
23.             int levelIndex = packageIndex;
24.             fileNames.Add("sharedassets" + levelIndex + ".resource");
25.             fileNames.Add("sharedassets" + levelIndex + ".assets");
26.             fileNames.Add(Path.Combine(@"Il2CppData\Metadata", "global-metadata.dat"));//Modification for 5.1.2 version
27.  
28.             if(packageIndex>0)
29.             {
30.                 fileNames.Add("level" + (levelIndex-1) + "");
31.             }else{
32.                 fileNames.Add(Path.Combine(kResourcesDirName, kResourcesFileName));
33.                 fileNames.Add(Path.Combine(kResourcesDirName, kResourcesExtraFileName));
34.                 fileNames.Add("mainData");
35.             }
36.  
37.             if(levelIndex==firstStreamedLevelWithResources)
38.             {
39.                 fileNames.Add("resources.assets");
40.                 fileNames.Add("resources.resource");
41.             }
42.  
43.             var files = new List<string>();
44.             foreach (var f in fileNames)
45.             {
46.                 if (filesToShip.Contains(f))
47.                 {
48.                     filesToShip.Remove(f);
49.                     files.Add(f);
50.                 }
51.             }
52.             if (files.Count == 0)
53.                 break;
54.  
55.             var packageFile = filename + "." + packageIndex;
56.             if (!RunPackager(packageFile, stagingAreaData, files))
57.                 return false;
58.  
59.             // Extract loader script for this data file, and put it's content into a JavaScript function
60.             if(packageIndex>0)
61.             {
62.                 var loaderText = File.ReadAllText(Path.Combine(stagingAreaData, packageFile + ".loader.js"));
63.                 loaderText = loaderText.Substring(loaderText.IndexOf("\n(function() {"));
64.                 loaderText = loaderText.Replace("if (Module['setStatus']) Module['setStatus']('Downloading data... (' + loaded + '/' + total + ')');",
65.                                                 "StreamProgressForLevelArray["+packageIndex+"]=event.loaded/size;");
66.                 loaderJS += "function DownloadDataForPackage" + packageIndex + "(){\n" + loaderText + "}\n";
67.                 loaderJSStart +="StreamProgressForLevelArray[" + packageIndex + "]=0;\n";
68.             }else{
69.                 loaderJS0 = File.ReadAllText(Path.Combine(stagingAreaData, filename + ".0.loader.js"));
70.                 loaderJS0 = loaderJS0.Replace("if (Module['setStatus']) Module['setStatus']('Downloading data... (' + loaded + '/' + total + ')');",
71.                                               "if (Module['setStatus']) Module['setStatus']('Downloading data... (' + event.loaded + '/' + size + ')');");
72.             }
73.             packageIndex++;
74.         }
75.  
76.         if(filesToShip.Count>0)
77.         {
78.             var packageFile = filename + "." + packageIndex;
79.             if (!RunPackager(packageFile, stagingAreaData, filesToShip))
80.                 return false;
81.             var loaderText2 = File.ReadAllText(Path.Combine(stagingAreaData, packageFile + ".loader.js"));
82.             loaderText2 = loaderText2.Substring(loaderText2.IndexOf("\n(function() {"));
83.             loaderJS += "function DownloadDataForPackage" + packageIndex + "(){\n" + loaderText2 + "}\n";
84.             packageIndex++;
85.         }
86.  
87.         loaderJS = loaderJSStart + loaderJS + loaderJS0;
88.  
89.         for (int j=0; j < packageIndex-1; j++)
90.         {
91.             // Patch loader scripts to invoke the function to start loading the next data file when it's done
92.             var line = "Module['removeRunDependency']('datafile_" + filename + "." + j + ".data');";
93.             // Call function to load next data file in a callback to avoid calling it while clearing pre-run dependencies.
94.             loaderJS = loaderJS.Replace (line, line + "\nStreamProgressForLevelArray[" + j + "]=1;\nwindow.setTimeout(DownloadDataForPackage" + (j+1) + @",1);");
95.         }
96.  
97.         // write out loader script for all data files
98.         File.WriteAllText(Path.Combine(stagingAreaData, kOutputFileLoaderFileName), loaderJS);
99.         return true;
100.     }

With this i don't get the "Not implemented: Class::FromIl2CppType" error in chrome as i do with the jonas version.

@liortal indeed i didn't notice at first it was coded for a release build, i just have to change the folder for each version, the directory error was because of that:

Code (CSharp):

1. File.Delete (Path.Combine(pathToBuiltProject, Path.Combine("Release", filename+".data")));

And thanks for the explanations, i wasn't sure it was necessary since the main code was in csharp and this part in js ^^;

 

Thanks AnonDreamer, you motivated me to give the improved version a try, and this one works good with 5.1.2p1 .
So better use this one and not the one from asset store.
To make it work with both development builds and release builds just change the code like it is in jonas version:

Code (CSharp):

1. var outputFolder = "Release";
2.         if (EditorUserBuildSettings.development)
3.             outputFolder = "Development";
4.      
5.         // delete old data file
6.         File.Delete (Path.Combine(pathToBuiltProject, Path.Combine(outputFolder, filename+".data")));
7.      
8.         // copy new data files into build
9.         File.Copy(Path.Combine(stagingAreaData, kOutputFileLoaderFileName), Path.Combine(Path.Combine(pathToBuiltProject, outputFolder),kOutputFileLoaderFileName), true);
10.         foreach (var f in Directory.GetFiles(stagingAreaData, "*.*.data"))
11.             File.Copy(f, Path.Combine(Path.Combine(pathToBuiltProject, outputFolder),Path.GetFileName(f)), true);
12.      
13.    
14. // compress new data files
15.      if (!EditorUserBuildSettings.development) {
16.        CompressFilesInOutputDirectory (Path.Combine (pathToBuiltProject, outputFolder), Path.Combine (pathToBuiltProject, "Compressed"));
17.      }
18.  

Thanks again - hope we don't meet again at the same topic with Unity 5.1.3

 

Good to see it works fine now

 

Hi,

I would also love to get this to work.

I tried De-panthers code (+jslib), and added AnonDreamers changes, but can not make this work on Unity 5.2 or 5.1.3. Did I miss something?

The project compiles without warnings, but when loading the game in the browser I get a bunch of errors, mostly seems like a lot of methods, types etc. can not be found.

Ex:
Unable to find type [UnityEngine.dll]UnityEngine.MonoBehaviour
Unable to find type [UnityEngine.dll]UnityEngine.Component
Unable to find type [UnityEngine.dll]UnityEngine.ScriptableObject
Unable to find type [UnityEngine.dll]UnityEngine.Vector2
+ 50 more similar warnings...

And then a couple of these warnings:
The referenced script on this Behaviour is missing!
A script behaviour has a different serialization layout when loading. (Read 32 bytes but expected 68 bytes)
Did you #ifdef UNITY_EDITOR a section of your serialized properties in any of your scripts?

Then the split up .data files are loaded from cache...but afterwards, this warning:
warning: a problem occurred in builtin C++ name demangling; build with -s DEMANGLE_SUPPORT=1 to link in libcxxabi demangling index.html:34:7
Invoking error handler due to uncaught exception: abort(133) at jsStackTrace@file:///........game.js

Thankful for any help

 

Did anyone try this script with the latest version of Unity that's out there (5.2.x) ? does it work out of the box ?

 

@liortal I tried it but it doesn't work. Sorry I couldn't give you more information. I had to downgrade it because I had limited time.

 

i haven't tried yet, i'm still on 5.1.2f1 :\

 

I tried the script in the asset store on Unity 5.2.1, but it reports about memory issues or an uncaught range error (trying different memory allocations).
It would be nice to see an official update on the asset store or the integration of this feature in the editor.

 

The asset store now says this:

Does that mean this technique is impossible to do now or that it has been builtin properly?

I'm finding I need to set huge memory sizes 1.5Gb even for data files that are only 5Mb compressed, seems odd that we need so much memory to run even small games.

 

I'd like to know that too, monark, or if anyone got this working in 5.3.x (we're in 5.3.1)? Not hitting those memory sizes in ours but initial load time is too long. Player runs great once it loads. We had level streaming working in 5.1.3 but it doesn't really make sense to go back.

One thing I noticed looking at OnPostprocessBuild where filesToShip is declared. I don't see any .resource, .asset, mainData or level files in my staging area data directory. I'd guess something with the whole process changed in 5.3. UnityLoader.js is new too I think. There used to be UnityConfig and fileloader js files. I'd assume UnityLoader handles all that now? Don't think that one is too human readable iirc.

Also, for anyone who hasn't noticed, if you're on a Mac the WebGLSupport/BuildTools moved. It used to be inside the app Contents folder. In 5.3.1 it's just next to the app in PlaybackEngines (probably the new installer). Use EditorApplication.applicationPath instead of the contents path and then strip the /Unity.app off the end of the string. It's in buildToolsDir {get} at the top of the class. Not sure where they are on Windows.

 

Thanks for the reply, Marco. I had a feeling asset bundles would be the recommendation. Not sure how practical they are for us for level streaming. They aren't quite the same as loading a scene. I guess I'll be taking a look!

 

Hi.
I'm using De-Panther script with unity 5.0.4 but I cant make it work.
My game contains 7 scenes. I put De-Panter C# script to Assets/
JS script to Assets/Plugins/WebGL
I build game as WebGL with player settings: Stripping Level (use microlib), optimized mesh data(on), enable exeptions (none), optimization level (slow), development build (off).
I'm using unity 5.0.4, my webgl build goes to xampp/htdocs/project folder and I open it via latest chrome http://localhost/project/
My realase folder contains 7 .data files (i think it is good because i have 7 scenes) and one project.js (100mb)


When i run game i see following errors in my chrome console:
errors with enable exeptions (off)

Code (CSharp):

1. (index):33 run() called, but dependencies remain, so not runningModule.printErr @ (index):33run @ detektyw.js:2892820(anonymous function) @ detektyw.js:2892931
2. (index):33 pre-main prep time: 669 ms
3.  
4. detektyw.js:2892804 Uncaught 14518992 - Exception catching is disabled, this exception cannot be caught. Compile with -s DISABLE_EXCEPTION_CATCHING=0 or DISABLE_EXCEPTION_CATCHING=2 to catch.

and errors with enable exeptions (on)

Code (CSharp):

1.  
2. (index):33 run() called, but dependencies remain, so not runningModule.printErr @ (index):33run @ detektyw.js:2892820(anonymous function) @ detektyw.js:2892931
3. (index):33 pre-main prep time: 669 ms
4.  
5. detektyw.js:4087481 Uncaught abort() at Error
6.     at jsStackTrace (http://localhost/detektyw/Release/detektyw.js:992:13)
7.     at stackTrace (http://localhost/detektyw/Release/detektyw.js:1009:22)
8.     at abort (http://localhost/detektyw/Release/detektyw.js:4087587:25)
9.     at enlargeMemory (http://localhost/detektyw/Release/detektyw.js:1028:3)
10.     at Function.Runtime.dynamicAlloc [as alloc] (http://localhost/detektyw/Release/detektyw.js:408:161)
11.     at _sbrk (http://localhost/detektyw/Release/detektyw.js:13521:28)
12.     at _GC_expand_hp_inner (http://localhost/detektyw/Release/detektyw.js:23436:11)
13.     at _GC_collect_or_expand (http://localhost/detektyw/Release/detektyw.js:18782:9)
14.     at _GC_alloc_large (http://localhost/detektyw/Release/detektyw.js:18355:12)
15.     at _GC_generic_malloc (http://localhost/detektyw/Release/detektyw.js:18987:11)
16. If this abort() is unexpected, build with -s ASSERTIONS=1 which can give more information.

Can someone explain me what I'm doing wrong? How to set my 0 scene was loaded first?
How do you switch scenes? Do you check in update if next scene is already loaded?
Do i need to edit my index or other files inside build folder to make it work?

 

you can check with "CanStreamedLevelBeLoaded" to check if you can load the level.
And you can set "firstStreamedLevelWithResources" to decide the first scenes that should download together on start

 

We tested it with Unity 5.3.4 and 5.3.5. It seems that there's no way to re-pack the levels for streaming anymore.

In the past, the engine didn't delete the "sharedassets" files in the build process. But currently the editor pack those files to it's own format, and delete them before the "PostProcessBuild" is called.

Maybe @jonas-echterhoff will have another idea. But we started to use AssetBundles for some of the stuff, although it's not ideal if we want to share Sprite Atlases between some of the scenes.

 

It seems exactly what i need for my wegbl project, but i dont know how to implement this thing in my project.
please can anyone help me with this.