From 6249803b0f67b61ac6727066c01c8aff93a39555 Mon Sep 17 00:00:00 2001 From: Skye Terran Date: Tue, 24 Aug 2021 00:20:08 -0700 Subject: [PATCH] Implemented SPORC cache --- NeutrinoJSONTest.ipynb | 137 ++++++++++++++++++++++++++++++++++------- NeutrinoTestScene.json | 80 ++++++++++++------------ 2 files changed, 158 insertions(+), 59 deletions(-) diff --git a/NeutrinoJSONTest.ipynb b/NeutrinoJSONTest.ipynb index d684d22..6d9e36b 100644 --- a/NeutrinoJSONTest.ipynb +++ b/NeutrinoJSONTest.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 7, + "execution_count": 157, "source": [ "import json\r\n", "\r\n", @@ -20,8 +20,8 @@ " \"assets\": {}\r\n", " },\r\n", " \"internal\": {\r\n", - " \"max_object_key\": 0,\r\n", - " \"max_cache_key\": 0\r\n", + " \"max_object_key\": {\"index\": 0},\r\n", + " \"max_cache_key\": {\"index\": 0}\r\n", " }\r\n", " }\r\n", "\r\n", @@ -41,22 +41,12 @@ " json.dump(clean_data, outfile, indent = 4)\r\n", "\r\n", "# get a new indexed object key and track it\r\n", - "def object_key(data):\r\n", + "def new_key(index):\r\n", " # get the indexed key\r\n", - " key = hex(data[\"internal\"][\"max_object_key\"] + 1)\r\n", + " key = hex(index[\"index\"] + 1)\r\n", "\r\n", " # index the max key\r\n", - " data[\"internal\"][\"max_object_key\"] += 1\r\n", - "\r\n", - " return key\r\n", - "\r\n", - "# get a new indexed cache key and track it\r\n", - "def cache_key(data):\r\n", - " # get the indexed key\r\n", - " key = hex(data[\"internal\"][\"max_cache_key\"] + 1)\r\n", - "\r\n", - " # index the max key\r\n", - " data[\"internal\"][\"max_cache_key\"] += 1\r\n", + " index[\"index\"] += 1\r\n", "\r\n", " return key\r\n", "\r\n", @@ -68,14 +58,18 @@ " }\r\n", " \r\n", " # add the asset to the graph\r\n", - " data[\"graph\"][\"assets\"][object_key(data)] = (\"asset\", asset_data)\r\n", + " data[\"graph\"][\"assets\"][new_key(data[\"internal\"][\"max_object_key\"])] = (\"asset\", asset_data)\r\n", "\r\n", "# add an object to the scene\r\n", "def spawn_object(data, name, asset):\r\n", " object_data = {\r\n", " \"name\": (\"name\", name),\r\n", " \"asset\": \"\",\r\n", - " \"trans\": (\"trans\", [[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], [1.0, 1.0, 1.0]])\r\n", + " \"trans\": (\"trans\", {\r\n", + " \"position\": (\"vec3\", [0.0, 0.0, 0.0]),\r\n", + " \"rotation\": (\"vec3\", [0.0, 0.0, 0.0]),\r\n", + " \"scale\": (\"vec3\", [1.0, 1.0, 1.0])\r\n", + " })\r\n", " }\r\n", "\r\n", " # get an asset key by the provided name\r\n", @@ -84,14 +78,14 @@ " object_data[\"asset\"] = f\"*{key}\"\r\n", "\r\n", " # add the object to the scene\r\n", - " data[\"graph\"][\"scene\"][object_key(data)] = (\"object\", object_data)" + " data[\"graph\"][\"scene\"][new_key(data[\"internal\"][\"max_object_key\"])] = (\"object\", object_data)" ], "outputs": [], "metadata": {} }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 158, "source": [ "# cache the scene\r\n", "def cache_scene(data):\r\n", @@ -152,7 +146,108 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 159, + "source": [ + "# recursively cache a single typeval tuple object\r\n", + "def cache_typeval(cache, typeval):\r\n", + " # ignore if not typeval\r\n", + " if type(typeval) == tuple:\r\n", + " for key, value in typeval[1].items():\r\n", + " # refuse to cache pointers (that's just... that would just be a nightmare)\r\n", + " if type(value) == str:\r\n", + " is_pointer = (\"*\" in value)\r\n", + " else:\r\n", + " is_pointer = False\r\n", + " if not is_pointer:\r\n", + " # cache member objects if it's a dictionary object\r\n", + " if type(value[1]) == dict:\r\n", + " cache_typeval(cache, value)\r\n", + "\r\n", + " value_hash = hash(str(value))\r\n", + "\r\n", + " # track in cache\r\n", + " if value_hash not in cache[\"objects\"]:\r\n", + " cache_pointer = new_key(cache[\"key_index\"])\r\n", + " cache[\"objects\"][value_hash] = {\"key\": cache_pointer, \"value\": value, \"count\": 1}\r\n", + " else:\r\n", + " cache_pointer = cache[\"objects\"][value_hash][\"key\"]\r\n", + " cache[\"objects\"][value_hash][\"count\"] += 1\r\n", + "\r\n", + " # replace real value with hash\r\n", + " typeval[1][key] = \"#\" + cache_pointer\r\n", + "\r\n", + "# if there's only one instance of a certain value, convert it back to the original value and destroy the cached version\r\n", + "def uncache_typeval(cache, typeval):\r\n", + " for key, value in typeval[1].items():\r\n", + " # refuse to cache pointers (that's just... that would just be a nightmare)\r\n", + " if type(value) == str:\r\n", + " is_pointer = (\"*\" in value)\r\n", + " else:\r\n", + " is_pointer = False\r\n", + " if not is_pointer:\r\n", + " # cache member objects if it's a dictionary object\r\n", + " if type(value[1]) == dict:\r\n", + " uncache_typeval(cache, value)\r\n", + "\r\n", + " value_hash = hash(str(value))\r\n", + "\r\n", + " # check if it occurs only once\r\n", + " cache_key = value.replace(\"#\", \"\")\r\n", + " if cache[cache_key][\"count\"] <= 1:\r\n", + " # replace the cache pointer in the scene data with its original value\r\n", + " typeval[1][key] = cache[cache_key][\"value\"]\r\n", + "\r\n", + " # delete this object from the cache\r\n", + " del cache[cache_key]\r\n", + "\r\n", + "# cache the scene\r\n", + "def cache_scene(data):\r\n", + " # add the cache object to the scene data\r\n", + " data[\"cache\"] = {}\r\n", + "\r\n", + " containers = [\r\n", + " data[\"graph\"][\"scene\"],\r\n", + " data[\"graph\"][\"assets\"]\r\n", + " ]\r\n", + "\r\n", + " # build a cache of value hashes and pointers\r\n", + " hash_cache = {\"key_index\": {\"index\": 0}, \"objects\": {}}\r\n", + " for objects in containers:\r\n", + " for key, value in objects.items():\r\n", + " cache_typeval(hash_cache, value)\r\n", + "\r\n", + " # create a cache hashed with pointer keys instead of value hashes\r\n", + " key_cache = {}\r\n", + " for key, value in hash_cache[\"objects\"].items():\r\n", + " key_cache[value[\"key\"]] = {\"value\": value[\"value\"], \"count\": value[\"count\"]}\r\n", + "\r\n", + " # prune the cache to only redirect repeat values\r\n", + " for objects in containers:\r\n", + " for key, value in objects.items():\r\n", + " uncache_typeval(key_cache, value)\r\n", + "\r\n", + " # create a serialized cache usable by neutrino\r\n", + " serial_cache = {}\r\n", + " for key, value in key_cache.items():\r\n", + " serial_cache[key] = value[\"value\"]\r\n", + "\r\n", + " # add that cache to the neutrino scene data\r\n", + " data[\"cache\"] = serial_cache\r\n", + "\r\n", + " # DEBUG\r\n", + " with open(\"hash_cache.json\", \"w\") as outfile:\r\n", + " json.dump(hash_cache, outfile, indent = 4)\r\n", + " with open(\"key_cache.json\", \"w\") as outfile:\r\n", + " json.dump(key_cache, outfile, indent = 4)\r\n", + " with open(\"serial_cache.json\", \"w\") as outfile:\r\n", + " json.dump(serial_cache, outfile, indent = 4)" + ], + "outputs": [], + "metadata": {} + }, + { + "cell_type": "code", + "execution_count": 160, "source": [ "# just returns a random string\r\n", "import random\r\n", diff --git a/NeutrinoTestScene.json b/NeutrinoTestScene.json index d85de6d..8bdf12f 100644 --- a/NeutrinoTestScene.json +++ b/NeutrinoTestScene.json @@ -20,10 +20,10 @@ { "name": [ "name", - "NO94Z37B" + "MSWVFKJ1" ], - "asset": "*0x1", - "trans": "#0x1" + "asset": "*0x2", + "trans": "#0x4" } ], "0x5": [ @@ -31,10 +31,10 @@ { "name": [ "name", - "NC302YU1" + "BKZH76FF" ], - "asset": "*0x2", - "trans": "#0x1" + "asset": "*0x1", + "trans": "#0x4" } ], "0x6": [ @@ -42,10 +42,10 @@ { "name": [ "name", - "GW5625SS" + "RHIO8EEY" ], "asset": "*0x3", - "trans": "#0x1" + "trans": "#0x4" } ], "0x7": [ @@ -53,10 +53,10 @@ { "name": [ "name", - "THUUN5IN" + "CMLYZE24" ], - "asset": "*0x3", - "trans": "#0x1" + "asset": "*0x1", + "trans": "#0x4" } ], "0x8": [ @@ -64,10 +64,10 @@ { "name": [ "name", - "JT5FVYOW" + "1MMYHH3R" ], - "asset": "*0x2", - "trans": "#0x1" + "asset": "*0x3", + "trans": "#0x4" } ] }, @@ -77,9 +77,9 @@ { "name": [ "name", - "3QDJ22P1" + "MN9XAQYF" ], - "file": "#0x2" + "file": "#0xa" } ], "0x2": [ @@ -87,9 +87,9 @@ { "name": [ "name", - "49ITSLZI" + "L722FBDH" ], - "file": "#0x2" + "file": "#0xa" } ], "0x3": [ @@ -97,35 +97,39 @@ { "name": [ "name", - "O6YSH0DS" + "BN05I9TO" ], - "file": "#0x2" + "file": "#0xa" } ] } }, "cache": { - "0x1": [ - "trans", + "0x2": [ + "vec3", [ - [ - 0.0, - 0.0, - 0.0 - ], - [ - 0.0, - 0.0, - 0.0 - ], - [ - 1.0, - 1.0, - 1.0 - ] + 0.0, + 0.0, + 0.0 ] ], - "0x2": [ + "0x3": [ + "vec3", + [ + 1.0, + 1.0, + 1.0 + ] + ], + "0x4": [ + "trans", + { + "position": "#0x2", + "rotation": "#0x2", + "scale": "#0x3" + } + ], + "0xa": [ "path", "Assets/TestAsset.obj" ]