diff --git a/NeutrinoJSONTest.ipynb b/NeutrinoJSONTest.ipynb index 6d9e36b..585cc03 100644 --- a/NeutrinoJSONTest.ipynb +++ b/NeutrinoJSONTest.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 157, + "execution_count": 217, "source": [ "import json\r\n", "\r\n", @@ -29,7 +29,7 @@ " return data\r\n", "\r\n", "# write the data to a JSON file\r\n", - "def save_scene(data):\r\n", + "def save_scene(data, readable):\r\n", " # create working copy of the scene data\r\n", " clean_data = data.copy()\r\n", "\r\n", @@ -38,7 +38,10 @@ " \r\n", " filename = data[\"meta\"][\"name\"][1].replace(\" \", \"\") + \".json\"\r\n", " with open(filename, \"w\") as outfile:\r\n", - " json.dump(clean_data, outfile, indent = 4)\r\n", + " if readable:\r\n", + " json.dump(clean_data, outfile, indent = 4)\r\n", + " else:\r\n", + " json.dump(clean_data, outfile)\r\n", "\r\n", "# get a new indexed object key and track it\r\n", "def new_key(index):\r\n", @@ -84,69 +87,16 @@ "metadata": {} }, { - "cell_type": "code", - "execution_count": 158, + "cell_type": "markdown", "source": [ - "# cache the scene\r\n", - "def cache_scene(data):\r\n", - " # add the cache object to the scene data\r\n", - " data[\"cache\"] = {}\r\n", - "\r\n", - " containers = [\r\n", - " data[\"graph\"][\"scene\"],\r\n", - " data[\"graph\"][\"assets\"]\r\n", - " ]\r\n", - "\r\n", - " for objects in containers:\r\n", - " # temp cache\r\n", - " hash_cache = {}\r\n", - "\r\n", - " # hash all values\r\n", - " for key, value in objects.items():\r\n", - " for key, value in value[1].items():\r\n", - " # ignore pointers (the only non-tuple object)\r\n", - " if type(value) == tuple:\r\n", - " # convert into string and hash that\r\n", - " value_hash = hash(str(value))\r\n", - "\r\n", - " # track in temp cache\r\n", - " if value_hash not in hash_cache:\r\n", - " hash_cache[value_hash] = {\"value\": value, \"count\": 1}\r\n", - " else:\r\n", - " hash_cache[value_hash][\"count\"] += 1\r\n", - "\r\n", - " # throw out all non-repeated values\r\n", - " bad_keys = []\r\n", - " for key, value in hash_cache.items():\r\n", - " if value[\"count\"] < 2:\r\n", - " bad_keys.append(key)\r\n", - " for key in bad_keys:\r\n", - " del hash_cache[key]\r\n", - "\r\n", - " # create hash objects for each repeated value\r\n", - " for key, value in hash_cache.items():\r\n", - " cache_pointer = cache_key(data)\r\n", - " data[\"cache\"][cache_pointer] = value[\"value\"]\r\n", - " hash_cache[key][\"pointer\"] = cache_pointer\r\n", - "\r\n", - " # replace all instances of cached values in the graph with corresponding cache pointers\r\n", - " for object_key, object_value in objects.items():\r\n", - " for value_key, value_value in object_value[1].items():\r\n", - " # ignore pointers (the only non-tuple object)\r\n", - " if type(value_value) == tuple:\r\n", - " # convert into string and hash that\r\n", - " value_hash = hash(str(value_value))\r\n", - "\r\n", - " # if this value is cached, replace it with its cache pointer\r\n", - " if value_hash in hash_cache:\r\n", - " objects[object_key][1][value_key] = \"#\" + hash_cache[value_hash][\"pointer\"]" + "### Implement SPORC for storage/memory optimization\r\n", + "(Single-Pointer Objective Cache)" ], - "outputs": [], "metadata": {} }, { "cell_type": "code", - "execution_count": 159, + "execution_count": 218, "source": [ "# recursively cache a single typeval tuple object\r\n", "def cache_typeval(cache, typeval):\r\n", @@ -232,22 +182,14 @@ " serial_cache[key] = value[\"value\"]\r\n", "\r\n", " # add that cache to the neutrino scene data\r\n", - " data[\"cache\"] = serial_cache\r\n", - "\r\n", - " # DEBUG\r\n", - " with open(\"hash_cache.json\", \"w\") as outfile:\r\n", - " json.dump(hash_cache, outfile, indent = 4)\r\n", - " with open(\"key_cache.json\", \"w\") as outfile:\r\n", - " json.dump(key_cache, outfile, indent = 4)\r\n", - " with open(\"serial_cache.json\", \"w\") as outfile:\r\n", - " json.dump(serial_cache, outfile, indent = 4)" + " data[\"cache\"] = serial_cache" ], "outputs": [], "metadata": {} }, { "cell_type": "code", - "execution_count": 160, + "execution_count": 219, "source": [ "# just returns a random string\r\n", "import random\r\n", @@ -260,19 +202,17 @@ "\r\n", "# populate assets\r\n", "asset_names = []\r\n", - "for i in range(3):\r\n", + "for i in range(10):\r\n", " name = random_string(8)\r\n", " add_asset(test_scene, name, \"Assets/TestAsset.obj\")\r\n", " asset_names.append(name)\r\n", "\r\n", "# populate objects in scene\r\n", - "for i in range(5):\r\n", + "for i in range(50):\r\n", " spawn_object(test_scene, random_string(8), random.choice(asset_names))\r\n", "\r\n", - "# cache the scene\r\n", "cache_scene(test_scene)\r\n", - "\r\n", - "save_scene(test_scene)" + "save_scene(test_scene, False)" ], "outputs": [], "metadata": {} diff --git a/NeutrinoTestScene.json b/NeutrinoTestScene.json index 8bdf12f..ec593ff 100644 --- a/NeutrinoTestScene.json +++ b/NeutrinoTestScene.json @@ -1,137 +1 @@ -{ - "meta": { - "name": [ - "name", - "Neutrino Test Scene" - ], - "scale": [ - "float", - 1.0 - ], - "asset_path": [ - "path", - "./" - ] - }, - "graph": { - "scene": { - "0x4": [ - "object", - { - "name": [ - "name", - "MSWVFKJ1" - ], - "asset": "*0x2", - "trans": "#0x4" - } - ], - "0x5": [ - "object", - { - "name": [ - "name", - "BKZH76FF" - ], - "asset": "*0x1", - "trans": "#0x4" - } - ], - "0x6": [ - "object", - { - "name": [ - "name", - "RHIO8EEY" - ], - "asset": "*0x3", - "trans": "#0x4" - } - ], - "0x7": [ - "object", - { - "name": [ - "name", - "CMLYZE24" - ], - "asset": "*0x1", - "trans": "#0x4" - } - ], - "0x8": [ - "object", - { - "name": [ - "name", - "1MMYHH3R" - ], - "asset": "*0x3", - "trans": "#0x4" - } - ] - }, - "assets": { - "0x1": [ - "asset", - { - "name": [ - "name", - "MN9XAQYF" - ], - "file": "#0xa" - } - ], - "0x2": [ - "asset", - { - "name": [ - "name", - "L722FBDH" - ], - "file": "#0xa" - } - ], - "0x3": [ - "asset", - { - "name": [ - "name", - "BN05I9TO" - ], - "file": "#0xa" - } - ] - } - }, - "cache": { - "0x2": [ - "vec3", - [ - 0.0, - 0.0, - 0.0 - ] - ], - "0x3": [ - "vec3", - [ - 1.0, - 1.0, - 1.0 - ] - ], - "0x4": [ - "trans", - { - "position": "#0x2", - "rotation": "#0x2", - "scale": "#0x3" - } - ], - "0xa": [ - "path", - "Assets/TestAsset.obj" - ] - } -} \ No newline at end of file +{"meta": {"name": ["name", "Neutrino Test Scene"], "scale": ["float", 1.0], "asset_path": ["path", "./"]}, "graph": {"scene": {"0xb": ["object", {"name": ["name", "K8ZWNOQD"], "asset": "*0x7", "trans": "#0x4"}], "0xc": ["object", {"name": ["name", "US0DHKXM"], "asset": "*0x2", "trans": "#0x4"}], "0xd": ["object", {"name": ["name", "DZ53QE47"], "asset": "*0x7", "trans": "#0x4"}], "0xe": ["object", {"name": ["name", "L0MLPJ07"], "asset": "*0x9", "trans": "#0x4"}], "0xf": ["object", {"name": ["name", "RHAQ5Y6Y"], "asset": "*0x3", "trans": "#0x4"}], "0x10": ["object", {"name": ["name", "70EO9E5L"], "asset": "*0x9", "trans": "#0x4"}], "0x11": ["object", {"name": ["name", "V7QSXNME"], "asset": "*0x8", "trans": "#0x4"}], "0x12": ["object", {"name": ["name", "F60CAXPY"], "asset": "*0xa", "trans": "#0x4"}], "0x13": ["object", {"name": ["name", "FBOCRYEU"], "asset": "*0x5", "trans": "#0x4"}], "0x14": ["object", {"name": ["name", "BU7FDZ8X"], "asset": "*0x9", "trans": "#0x4"}], "0x15": ["object", {"name": ["name", "6J5HMTF0"], "asset": "*0x6", "trans": "#0x4"}], "0x16": ["object", {"name": ["name", "GWNR7BVP"], "asset": "*0xa", "trans": "#0x4"}], "0x17": ["object", {"name": ["name", "3L6PXJMP"], "asset": "*0x3", "trans": "#0x4"}], "0x18": ["object", {"name": ["name", "GFQ1ND0L"], "asset": "*0x5", "trans": "#0x4"}], "0x19": ["object", {"name": ["name", "TJOVYREC"], "asset": "*0x6", "trans": "#0x4"}], "0x1a": ["object", {"name": ["name", "8APBDYNQ"], "asset": "*0x2", "trans": "#0x4"}], "0x1b": ["object", {"name": ["name", "QEBU36A0"], "asset": "*0x3", "trans": "#0x4"}], "0x1c": ["object", {"name": ["name", "E2DONFRV"], "asset": "*0x8", "trans": "#0x4"}], "0x1d": ["object", {"name": ["name", "K266KMAB"], "asset": "*0x6", "trans": "#0x4"}], "0x1e": ["object", {"name": ["name", "YDW0I1J5"], "asset": "*0xa", "trans": "#0x4"}], "0x1f": ["object", {"name": ["name", "0CKXOWCH"], "asset": "*0x8", "trans": "#0x4"}], "0x20": ["object", {"name": ["name", "5ETNQA9W"], "asset": "*0x6", "trans": "#0x4"}], "0x21": ["object", {"name": ["name", "9OE70GMX"], "asset": "*0x1", "trans": "#0x4"}], "0x22": ["object", {"name": ["name", "R90HUM0L"], "asset": "*0x1", "trans": "#0x4"}], "0x23": ["object", {"name": ["name", "MZY8N9IZ"], "asset": "*0x6", "trans": "#0x4"}], "0x24": ["object", {"name": ["name", "NLFWVF6B"], "asset": "*0x1", "trans": "#0x4"}], "0x25": ["object", {"name": ["name", "6BA2464B"], "asset": "*0x6", "trans": "#0x4"}], "0x26": ["object", {"name": ["name", "7LNCT9LH"], "asset": "*0x3", "trans": "#0x4"}], "0x27": ["object", {"name": ["name", "A1R1LT89"], "asset": "*0xa", "trans": "#0x4"}], "0x28": ["object", {"name": ["name", "89ZWAJ9G"], "asset": "*0x1", "trans": "#0x4"}], "0x29": ["object", {"name": ["name", "IRB764Q6"], "asset": "*0x4", "trans": "#0x4"}], "0x2a": ["object", {"name": ["name", "PBR36UOB"], "asset": "*0x6", "trans": "#0x4"}], "0x2b": ["object", {"name": ["name", "I8FT701J"], "asset": "*0x7", "trans": "#0x4"}], "0x2c": ["object", {"name": ["name", "1I637ECH"], "asset": "*0x2", "trans": "#0x4"}], "0x2d": ["object", {"name": ["name", "A6UDB97O"], "asset": "*0x8", "trans": "#0x4"}], "0x2e": ["object", {"name": ["name", "0M9RLFDZ"], "asset": "*0x6", "trans": "#0x4"}], "0x2f": ["object", {"name": ["name", "IXTLUU20"], "asset": "*0x7", "trans": "#0x4"}], "0x30": ["object", {"name": ["name", "D3SL2R4M"], "asset": "*0x9", "trans": "#0x4"}], "0x31": ["object", {"name": ["name", "CIIS6Y6K"], "asset": "*0x8", "trans": "#0x4"}], "0x32": ["object", {"name": ["name", "M1XFZQJZ"], "asset": "*0xa", "trans": "#0x4"}], "0x33": ["object", {"name": ["name", "W699N3Y5"], "asset": "*0xa", "trans": "#0x4"}], "0x34": ["object", {"name": ["name", "MK4XICU6"], "asset": "*0x7", "trans": "#0x4"}], "0x35": ["object", {"name": ["name", "JH5PLU6Q"], "asset": "*0x7", "trans": "#0x4"}], "0x36": ["object", {"name": ["name", "IIUCHG2G"], "asset": "*0x4", "trans": "#0x4"}], "0x37": ["object", {"name": ["name", "Q4GXK1K4"], "asset": "*0x2", "trans": "#0x4"}], "0x38": ["object", {"name": ["name", "S623ETE1"], "asset": "*0x1", "trans": "#0x4"}], "0x39": ["object", {"name": ["name", "2065J1LS"], "asset": "*0x5", "trans": "#0x4"}], "0x3a": ["object", {"name": ["name", "ZTNA2RLO"], "asset": "*0xa", "trans": "#0x4"}], "0x3b": ["object", {"name": ["name", "SDC5B3QK"], "asset": "*0x3", "trans": "#0x4"}], "0x3c": ["object", {"name": ["name", "EGIL6IC4"], "asset": "*0x7", "trans": "#0x4"}]}, "assets": {"0x1": ["asset", {"name": ["name", "C2FK95T4"], "file": "#0x37"}], "0x2": ["asset", {"name": ["name", "L4UBCTM0"], "file": "#0x37"}], "0x3": ["asset", {"name": ["name", "JLV3H133"], "file": "#0x37"}], "0x4": ["asset", {"name": ["name", "KW5P72ZQ"], "file": "#0x37"}], "0x5": ["asset", {"name": ["name", "VMPY9TWY"], "file": "#0x37"}], "0x6": ["asset", {"name": ["name", "5UI4DUPF"], "file": "#0x37"}], "0x7": ["asset", {"name": ["name", "F9HV6WMM"], "file": "#0x37"}], "0x8": ["asset", {"name": ["name", "GUH5P1MT"], "file": "#0x37"}], "0x9": ["asset", {"name": ["name", "CMJT9GHT"], "file": "#0x37"}], "0xa": ["asset", {"name": ["name", "L0QXBOVT"], "file": "#0x37"}]}}, "cache": {"0x2": ["vec3", [0.0, 0.0, 0.0]], "0x3": ["vec3", [1.0, 1.0, 1.0]], "0x4": ["trans", {"position": "#0x2", "rotation": "#0x2", "scale": "#0x3"}], "0x37": ["path", "Assets/TestAsset.obj"]}} \ No newline at end of file