robotcritter/Neutrino
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Implemented SPORC cache

Browse Source
This commit is contained in:
Skye Terran 2021-08-24 00:20:08 -07:00
parent 60b94a426f
commit 6249803b0f
2 changed files with 158 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

137
NeutrinoJSONTest.ipynb
View File

@ -2,7 +2,7 @@
"cells": [ "cells": [
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 7, "execution_count": 157,
"source": [ "source": [
"import json\r\n", "import json\r\n",
"\r\n", "\r\n",
@ -20,8 +20,8 @@
" \"assets\": {}\r\n", " \"assets\": {}\r\n",
" },\r\n", " },\r\n",
" \"internal\": {\r\n", " \"internal\": {\r\n",
" \"max_object_key\": 0,\r\n", " \"max_object_key\": {\"index\": 0},\r\n",
" \"max_cache_key\": 0\r\n", " \"max_cache_key\": {\"index\": 0}\r\n",
" }\r\n", " }\r\n",
" }\r\n", " }\r\n",
"\r\n", "\r\n",
@ -41,22 +41,12 @@
" json.dump(clean_data, outfile, indent = 4)\r\n", " json.dump(clean_data, outfile, indent = 4)\r\n",
"\r\n", "\r\n",
"# get a new indexed object key and track it\r\n", "# get a new indexed object key and track it\r\n",
"def object_key(data):\r\n", "def new_key(index):\r\n",
" # get the indexed key\r\n", " # get the indexed key\r\n",
" key = hex(data[\"internal\"][\"max_object_key\"] + 1)\r\n", " key = hex(index[\"index\"] + 1)\r\n",
"\r\n", "\r\n",
" # index the max key\r\n", " # index the max key\r\n",
" data[\"internal\"][\"max_object_key\"] += 1\r\n", " index[\"index\"] += 1\r\n",
"\r\n",
" return key\r\n",
"\r\n",
"# get a new indexed cache key and track it\r\n",
"def cache_key(data):\r\n",
" # get the indexed key\r\n",
" key = hex(data[\"internal\"][\"max_cache_key\"] + 1)\r\n",
"\r\n",
" # index the max key\r\n",
" data[\"internal\"][\"max_cache_key\"] += 1\r\n",
"\r\n", "\r\n",
" return key\r\n", " return key\r\n",
"\r\n", "\r\n",
@ -68,14 +58,18 @@
" }\r\n", " }\r\n",
" \r\n", " \r\n",
" # add the asset to the graph\r\n", " # add the asset to the graph\r\n",
" data[\"graph\"][\"assets\"][object_key(data)] = (\"asset\", asset_data)\r\n", " data[\"graph\"][\"assets\"][new_key(data[\"internal\"][\"max_object_key\"])] = (\"asset\", asset_data)\r\n",
"\r\n", "\r\n",
"# add an object to the scene\r\n", "# add an object to the scene\r\n",
"def spawn_object(data, name, asset):\r\n", "def spawn_object(data, name, asset):\r\n",
" object_data = {\r\n", " object_data = {\r\n",
" \"name\": (\"name\", name),\r\n", " \"name\": (\"name\", name),\r\n",
" \"asset\": \"\",\r\n", " \"asset\": \"\",\r\n",
" \"trans\": (\"trans\", [[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], [1.0, 1.0, 1.0]])\r\n", " \"trans\": (\"trans\", {\r\n",
" \"position\": (\"vec3\", [0.0, 0.0, 0.0]),\r\n",
" \"rotation\": (\"vec3\", [0.0, 0.0, 0.0]),\r\n",
" \"scale\": (\"vec3\", [1.0, 1.0, 1.0])\r\n",
" })\r\n",
" }\r\n", " }\r\n",
"\r\n", "\r\n",
" # get an asset key by the provided name\r\n", " # get an asset key by the provided name\r\n",
@ -84,14 +78,14 @@
" object_data[\"asset\"] = f\"*{key}\"\r\n", " object_data[\"asset\"] = f\"*{key}\"\r\n",
"\r\n", "\r\n",
" # add the object to the scene\r\n", " # add the object to the scene\r\n",
" data[\"graph\"][\"scene\"][object_key(data)] = (\"object\", object_data)" " data[\"graph\"][\"scene\"][new_key(data[\"internal\"][\"max_object_key\"])] = (\"object\", object_data)"
], ],
"outputs": [], "outputs": [],
"metadata": {} "metadata": {}
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 8, "execution_count": 158,
"source": [ "source": [
"# cache the scene\r\n", "# cache the scene\r\n",
"def cache_scene(data):\r\n", "def cache_scene(data):\r\n",
@ -152,7 +146,108 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 9, "execution_count": 159,
"source": [
"# recursively cache a single typeval tuple object\r\n",
"def cache_typeval(cache, typeval):\r\n",
" # ignore if not typeval\r\n",
" if type(typeval) == tuple:\r\n",
" for key, value in typeval[1].items():\r\n",
" # refuse to cache pointers (that's just... that would just be a nightmare)\r\n",
" if type(value) == str:\r\n",
" is_pointer = (\"*\" in value)\r\n",
" else:\r\n",
" is_pointer = False\r\n",
" if not is_pointer:\r\n",
" # cache member objects if it's a dictionary object\r\n",
" if type(value[1]) == dict:\r\n",
" cache_typeval(cache, value)\r\n",
"\r\n",
" value_hash = hash(str(value))\r\n",
"\r\n",
" # track in cache\r\n",
" if value_hash not in cache[\"objects\"]:\r\n",
" cache_pointer = new_key(cache[\"key_index\"])\r\n",
" cache[\"objects\"][value_hash] = {\"key\": cache_pointer, \"value\": value, \"count\": 1}\r\n",
" else:\r\n",
" cache_pointer = cache[\"objects\"][value_hash][\"key\"]\r\n",
" cache[\"objects\"][value_hash][\"count\"] += 1\r\n",
"\r\n",
" # replace real value with hash\r\n",
" typeval[1][key] = \"#\" + cache_pointer\r\n",
"\r\n",
"# if there's only one instance of a certain value, convert it back to the original value and destroy the cached version\r\n",
"def uncache_typeval(cache, typeval):\r\n",
" for key, value in typeval[1].items():\r\n",
" # refuse to cache pointers (that's just... that would just be a nightmare)\r\n",
" if type(value) == str:\r\n",
" is_pointer = (\"*\" in value)\r\n",
" else:\r\n",
" is_pointer = False\r\n",
" if not is_pointer:\r\n",
" # cache member objects if it's a dictionary object\r\n",
" if type(value[1]) == dict:\r\n",
" uncache_typeval(cache, value)\r\n",
"\r\n",
" value_hash = hash(str(value))\r\n",
"\r\n",
" # check if it occurs only once\r\n",
" cache_key = value.replace(\"#\", \"\")\r\n",
" if cache[cache_key][\"count\"] <= 1:\r\n",
" # replace the cache pointer in the scene data with its original value\r\n",
" typeval[1][key] = cache[cache_key][\"value\"]\r\n",
"\r\n",
" # delete this object from the cache\r\n",
" del cache[cache_key]\r\n",
"\r\n",
"# cache the scene\r\n",
"def cache_scene(data):\r\n",
" # add the cache object to the scene data\r\n",
" data[\"cache\"] = {}\r\n",
"\r\n",
" containers = [\r\n",
" data[\"graph\"][\"scene\"],\r\n",
" data[\"graph\"][\"assets\"]\r\n",
" ]\r\n",
"\r\n",
" # build a cache of value hashes and pointers\r\n",
" hash_cache = {\"key_index\": {\"index\": 0}, \"objects\": {}}\r\n",
" for objects in containers:\r\n",
" for key, value in objects.items():\r\n",
" cache_typeval(hash_cache, value)\r\n",
"\r\n",
" # create a cache hashed with pointer keys instead of value hashes\r\n",
" key_cache = {}\r\n",
" for key, value in hash_cache[\"objects\"].items():\r\n",
" key_cache[value[\"key\"]] = {\"value\": value[\"value\"], \"count\": value[\"count\"]}\r\n",
"\r\n",
" # prune the cache to only redirect repeat values\r\n",
" for objects in containers:\r\n",
" for key, value in objects.items():\r\n",
" uncache_typeval(key_cache, value)\r\n",
"\r\n",
" # create a serialized cache usable by neutrino\r\n",
" serial_cache = {}\r\n",
" for key, value in key_cache.items():\r\n",
" serial_cache[key] = value[\"value\"]\r\n",
"\r\n",
" # add that cache to the neutrino scene data\r\n",
" data[\"cache\"] = serial_cache\r\n",
"\r\n",
" # DEBUG\r\n",
" with open(\"hash_cache.json\", \"w\") as outfile:\r\n",
" json.dump(hash_cache, outfile, indent = 4)\r\n",
" with open(\"key_cache.json\", \"w\") as outfile:\r\n",
" json.dump(key_cache, outfile, indent = 4)\r\n",
" with open(\"serial_cache.json\", \"w\") as outfile:\r\n",
" json.dump(serial_cache, outfile, indent = 4)"
],
"outputs": [],
"metadata": {}
},
{
"cell_type": "code",
"execution_count": 160,
"source": [ "source": [
"# just returns a random string\r\n", "# just returns a random string\r\n",
"import random\r\n", "import random\r\n",

64
NeutrinoTestScene.json
View File

@ -20,10 +20,10 @@
{ {
"name": [ "name": [
"name", "name",
"NO94Z37B" "MSWVFKJ1"
], ],
"asset": "*0x1", "asset": "*0x2",
"trans": "#0x1" "trans": "#0x4"
} }
], ],
"0x5": [ "0x5": [
@ -31,10 +31,10 @@
{ {
"name": [ "name": [
"name", "name",
"NC302YU1" "BKZH76FF"
], ],
"asset": "*0x2", "asset": "*0x1",
"trans": "#0x1" "trans": "#0x4"
} }
], ],
"0x6": [ "0x6": [
@ -42,10 +42,10 @@
{ {
"name": [ "name": [
"name", "name",
"GW5625SS" "RHIO8EEY"
], ],
"asset": "*0x3", "asset": "*0x3",
"trans": "#0x1" "trans": "#0x4"
} }
], ],
"0x7": [ "0x7": [
@ -53,10 +53,10 @@
{ {
"name": [ "name": [
"name", "name",
"THUUN5IN" "CMLYZE24"
], ],
"asset": "*0x3", "asset": "*0x1",
"trans": "#0x1" "trans": "#0x4"
} }
], ],
"0x8": [ "0x8": [
@ -64,10 +64,10 @@
{ {
"name": [ "name": [
"name", "name",
"JT5FVYOW" "1MMYHH3R"
], ],
"asset": "*0x2", "asset": "*0x3",
"trans": "#0x1" "trans": "#0x4"
} }
] ]
}, },
@ -77,9 +77,9 @@
{ {
"name": [ "name": [
"name", "name",
"3QDJ22P1" "MN9XAQYF"
], ],
"file": "#0x2" "file": "#0xa"
} }
], ],
"0x2": [ "0x2": [
@ -87,9 +87,9 @@
{ {
"name": [ "name": [
"name", "name",
"49ITSLZI" "L722FBDH"
], ],
"file": "#0x2" "file": "#0xa"
} }
], ],
"0x3": [ "0x3": [
@ -97,35 +97,39 @@
{ {
"name": [ "name": [
"name", "name",
"O6YSH0DS" "BN05I9TO"
], ],
"file": "#0x2" "file": "#0xa"
} }
] ]
} }
}, },
"cache": { "cache": {
"0x1": [ "0x2": [
"trans", "vec3",
[
[
0.0,
0.0,
0.0
],
[ [
0.0, 0.0,
0.0, 0.0,
0.0 0.0
]
], ],
"0x3": [
"vec3",
[ [
1.0, 1.0,
1.0, 1.0,
1.0 1.0
] ]
]
], ],
"0x2": [ "0x4": [
"trans",
{
"position": "#0x2",
"rotation": "#0x2",
"scale": "#0x3"
}
],
"0xa": [
"path", "path",
"Assets/TestAsset.obj" "Assets/TestAsset.obj"
] ]