Implemented SPORC cache
This commit is contained in:
parent
60b94a426f
commit
6249803b0f
|
|
@ -2,7 +2,7 @@
|
|||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 7,
|
||||
"execution_count": 157,
|
||||
"source": [
|
||||
"import json\r\n",
|
||||
"\r\n",
|
||||
|
|
@ -20,8 +20,8 @@
|
|||
" \"assets\": {}\r\n",
|
||||
" },\r\n",
|
||||
" \"internal\": {\r\n",
|
||||
" \"max_object_key\": 0,\r\n",
|
||||
" \"max_cache_key\": 0\r\n",
|
||||
" \"max_object_key\": {\"index\": 0},\r\n",
|
||||
" \"max_cache_key\": {\"index\": 0}\r\n",
|
||||
" }\r\n",
|
||||
" }\r\n",
|
||||
"\r\n",
|
||||
|
|
@ -41,22 +41,12 @@
|
|||
" json.dump(clean_data, outfile, indent = 4)\r\n",
|
||||
"\r\n",
|
||||
"# get a new indexed object key and track it\r\n",
|
||||
"def object_key(data):\r\n",
|
||||
"def new_key(index):\r\n",
|
||||
" # get the indexed key\r\n",
|
||||
" key = hex(data[\"internal\"][\"max_object_key\"] + 1)\r\n",
|
||||
" key = hex(index[\"index\"] + 1)\r\n",
|
||||
"\r\n",
|
||||
" # index the max key\r\n",
|
||||
" data[\"internal\"][\"max_object_key\"] += 1\r\n",
|
||||
"\r\n",
|
||||
" return key\r\n",
|
||||
"\r\n",
|
||||
"# get a new indexed cache key and track it\r\n",
|
||||
"def cache_key(data):\r\n",
|
||||
" # get the indexed key\r\n",
|
||||
" key = hex(data[\"internal\"][\"max_cache_key\"] + 1)\r\n",
|
||||
"\r\n",
|
||||
" # index the max key\r\n",
|
||||
" data[\"internal\"][\"max_cache_key\"] += 1\r\n",
|
||||
" index[\"index\"] += 1\r\n",
|
||||
"\r\n",
|
||||
" return key\r\n",
|
||||
"\r\n",
|
||||
|
|
@ -68,14 +58,18 @@
|
|||
" }\r\n",
|
||||
" \r\n",
|
||||
" # add the asset to the graph\r\n",
|
||||
" data[\"graph\"][\"assets\"][object_key(data)] = (\"asset\", asset_data)\r\n",
|
||||
" data[\"graph\"][\"assets\"][new_key(data[\"internal\"][\"max_object_key\"])] = (\"asset\", asset_data)\r\n",
|
||||
"\r\n",
|
||||
"# add an object to the scene\r\n",
|
||||
"def spawn_object(data, name, asset):\r\n",
|
||||
" object_data = {\r\n",
|
||||
" \"name\": (\"name\", name),\r\n",
|
||||
" \"asset\": \"\",\r\n",
|
||||
" \"trans\": (\"trans\", [[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], [1.0, 1.0, 1.0]])\r\n",
|
||||
" \"trans\": (\"trans\", {\r\n",
|
||||
" \"position\": (\"vec3\", [0.0, 0.0, 0.0]),\r\n",
|
||||
" \"rotation\": (\"vec3\", [0.0, 0.0, 0.0]),\r\n",
|
||||
" \"scale\": (\"vec3\", [1.0, 1.0, 1.0])\r\n",
|
||||
" })\r\n",
|
||||
" }\r\n",
|
||||
"\r\n",
|
||||
" # get an asset key by the provided name\r\n",
|
||||
|
|
@ -84,14 +78,14 @@
|
|||
" object_data[\"asset\"] = f\"*{key}\"\r\n",
|
||||
"\r\n",
|
||||
" # add the object to the scene\r\n",
|
||||
" data[\"graph\"][\"scene\"][object_key(data)] = (\"object\", object_data)"
|
||||
" data[\"graph\"][\"scene\"][new_key(data[\"internal\"][\"max_object_key\"])] = (\"object\", object_data)"
|
||||
],
|
||||
"outputs": [],
|
||||
"metadata": {}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 8,
|
||||
"execution_count": 158,
|
||||
"source": [
|
||||
"# cache the scene\r\n",
|
||||
"def cache_scene(data):\r\n",
|
||||
|
|
@ -152,7 +146,108 @@
|
|||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 9,
|
||||
"execution_count": 159,
|
||||
"source": [
|
||||
"# recursively cache a single typeval tuple object\r\n",
|
||||
"def cache_typeval(cache, typeval):\r\n",
|
||||
" # ignore if not typeval\r\n",
|
||||
" if type(typeval) == tuple:\r\n",
|
||||
" for key, value in typeval[1].items():\r\n",
|
||||
" # refuse to cache pointers (that's just... that would just be a nightmare)\r\n",
|
||||
" if type(value) == str:\r\n",
|
||||
" is_pointer = (\"*\" in value)\r\n",
|
||||
" else:\r\n",
|
||||
" is_pointer = False\r\n",
|
||||
" if not is_pointer:\r\n",
|
||||
" # cache member objects if it's a dictionary object\r\n",
|
||||
" if type(value[1]) == dict:\r\n",
|
||||
" cache_typeval(cache, value)\r\n",
|
||||
"\r\n",
|
||||
" value_hash = hash(str(value))\r\n",
|
||||
"\r\n",
|
||||
" # track in cache\r\n",
|
||||
" if value_hash not in cache[\"objects\"]:\r\n",
|
||||
" cache_pointer = new_key(cache[\"key_index\"])\r\n",
|
||||
" cache[\"objects\"][value_hash] = {\"key\": cache_pointer, \"value\": value, \"count\": 1}\r\n",
|
||||
" else:\r\n",
|
||||
" cache_pointer = cache[\"objects\"][value_hash][\"key\"]\r\n",
|
||||
" cache[\"objects\"][value_hash][\"count\"] += 1\r\n",
|
||||
"\r\n",
|
||||
" # replace real value with hash\r\n",
|
||||
" typeval[1][key] = \"#\" + cache_pointer\r\n",
|
||||
"\r\n",
|
||||
"# if there's only one instance of a certain value, convert it back to the original value and destroy the cached version\r\n",
|
||||
"def uncache_typeval(cache, typeval):\r\n",
|
||||
" for key, value in typeval[1].items():\r\n",
|
||||
" # refuse to cache pointers (that's just... that would just be a nightmare)\r\n",
|
||||
" if type(value) == str:\r\n",
|
||||
" is_pointer = (\"*\" in value)\r\n",
|
||||
" else:\r\n",
|
||||
" is_pointer = False\r\n",
|
||||
" if not is_pointer:\r\n",
|
||||
" # cache member objects if it's a dictionary object\r\n",
|
||||
" if type(value[1]) == dict:\r\n",
|
||||
" uncache_typeval(cache, value)\r\n",
|
||||
"\r\n",
|
||||
" value_hash = hash(str(value))\r\n",
|
||||
"\r\n",
|
||||
" # check if it occurs only once\r\n",
|
||||
" cache_key = value.replace(\"#\", \"\")\r\n",
|
||||
" if cache[cache_key][\"count\"] <= 1:\r\n",
|
||||
" # replace the cache pointer in the scene data with its original value\r\n",
|
||||
" typeval[1][key] = cache[cache_key][\"value\"]\r\n",
|
||||
"\r\n",
|
||||
" # delete this object from the cache\r\n",
|
||||
" del cache[cache_key]\r\n",
|
||||
"\r\n",
|
||||
"# cache the scene\r\n",
|
||||
"def cache_scene(data):\r\n",
|
||||
" # add the cache object to the scene data\r\n",
|
||||
" data[\"cache\"] = {}\r\n",
|
||||
"\r\n",
|
||||
" containers = [\r\n",
|
||||
" data[\"graph\"][\"scene\"],\r\n",
|
||||
" data[\"graph\"][\"assets\"]\r\n",
|
||||
" ]\r\n",
|
||||
"\r\n",
|
||||
" # build a cache of value hashes and pointers\r\n",
|
||||
" hash_cache = {\"key_index\": {\"index\": 0}, \"objects\": {}}\r\n",
|
||||
" for objects in containers:\r\n",
|
||||
" for key, value in objects.items():\r\n",
|
||||
" cache_typeval(hash_cache, value)\r\n",
|
||||
"\r\n",
|
||||
" # create a cache hashed with pointer keys instead of value hashes\r\n",
|
||||
" key_cache = {}\r\n",
|
||||
" for key, value in hash_cache[\"objects\"].items():\r\n",
|
||||
" key_cache[value[\"key\"]] = {\"value\": value[\"value\"], \"count\": value[\"count\"]}\r\n",
|
||||
"\r\n",
|
||||
" # prune the cache to only redirect repeat values\r\n",
|
||||
" for objects in containers:\r\n",
|
||||
" for key, value in objects.items():\r\n",
|
||||
" uncache_typeval(key_cache, value)\r\n",
|
||||
"\r\n",
|
||||
" # create a serialized cache usable by neutrino\r\n",
|
||||
" serial_cache = {}\r\n",
|
||||
" for key, value in key_cache.items():\r\n",
|
||||
" serial_cache[key] = value[\"value\"]\r\n",
|
||||
"\r\n",
|
||||
" # add that cache to the neutrino scene data\r\n",
|
||||
" data[\"cache\"] = serial_cache\r\n",
|
||||
"\r\n",
|
||||
" # DEBUG\r\n",
|
||||
" with open(\"hash_cache.json\", \"w\") as outfile:\r\n",
|
||||
" json.dump(hash_cache, outfile, indent = 4)\r\n",
|
||||
" with open(\"key_cache.json\", \"w\") as outfile:\r\n",
|
||||
" json.dump(key_cache, outfile, indent = 4)\r\n",
|
||||
" with open(\"serial_cache.json\", \"w\") as outfile:\r\n",
|
||||
" json.dump(serial_cache, outfile, indent = 4)"
|
||||
],
|
||||
"outputs": [],
|
||||
"metadata": {}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 160,
|
||||
"source": [
|
||||
"# just returns a random string\r\n",
|
||||
"import random\r\n",
|
||||
|
|
|
|||
|
|
@ -20,10 +20,10 @@
|
|||
{
|
||||
"name": [
|
||||
"name",
|
||||
"NO94Z37B"
|
||||
"MSWVFKJ1"
|
||||
],
|
||||
"asset": "*0x1",
|
||||
"trans": "#0x1"
|
||||
"asset": "*0x2",
|
||||
"trans": "#0x4"
|
||||
}
|
||||
],
|
||||
"0x5": [
|
||||
|
|
@ -31,10 +31,10 @@
|
|||
{
|
||||
"name": [
|
||||
"name",
|
||||
"NC302YU1"
|
||||
"BKZH76FF"
|
||||
],
|
||||
"asset": "*0x2",
|
||||
"trans": "#0x1"
|
||||
"asset": "*0x1",
|
||||
"trans": "#0x4"
|
||||
}
|
||||
],
|
||||
"0x6": [
|
||||
|
|
@ -42,10 +42,10 @@
|
|||
{
|
||||
"name": [
|
||||
"name",
|
||||
"GW5625SS"
|
||||
"RHIO8EEY"
|
||||
],
|
||||
"asset": "*0x3",
|
||||
"trans": "#0x1"
|
||||
"trans": "#0x4"
|
||||
}
|
||||
],
|
||||
"0x7": [
|
||||
|
|
@ -53,10 +53,10 @@
|
|||
{
|
||||
"name": [
|
||||
"name",
|
||||
"THUUN5IN"
|
||||
"CMLYZE24"
|
||||
],
|
||||
"asset": "*0x3",
|
||||
"trans": "#0x1"
|
||||
"asset": "*0x1",
|
||||
"trans": "#0x4"
|
||||
}
|
||||
],
|
||||
"0x8": [
|
||||
|
|
@ -64,10 +64,10 @@
|
|||
{
|
||||
"name": [
|
||||
"name",
|
||||
"JT5FVYOW"
|
||||
"1MMYHH3R"
|
||||
],
|
||||
"asset": "*0x2",
|
||||
"trans": "#0x1"
|
||||
"asset": "*0x3",
|
||||
"trans": "#0x4"
|
||||
}
|
||||
]
|
||||
},
|
||||
|
|
@ -77,9 +77,9 @@
|
|||
{
|
||||
"name": [
|
||||
"name",
|
||||
"3QDJ22P1"
|
||||
"MN9XAQYF"
|
||||
],
|
||||
"file": "#0x2"
|
||||
"file": "#0xa"
|
||||
}
|
||||
],
|
||||
"0x2": [
|
||||
|
|
@ -87,9 +87,9 @@
|
|||
{
|
||||
"name": [
|
||||
"name",
|
||||
"49ITSLZI"
|
||||
"L722FBDH"
|
||||
],
|
||||
"file": "#0x2"
|
||||
"file": "#0xa"
|
||||
}
|
||||
],
|
||||
"0x3": [
|
||||
|
|
@ -97,35 +97,39 @@
|
|||
{
|
||||
"name": [
|
||||
"name",
|
||||
"O6YSH0DS"
|
||||
"BN05I9TO"
|
||||
],
|
||||
"file": "#0x2"
|
||||
"file": "#0xa"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"cache": {
|
||||
"0x1": [
|
||||
"trans",
|
||||
"0x2": [
|
||||
"vec3",
|
||||
[
|
||||
[
|
||||
0.0,
|
||||
0.0,
|
||||
0.0
|
||||
],
|
||||
[
|
||||
0.0,
|
||||
0.0,
|
||||
0.0
|
||||
],
|
||||
[
|
||||
1.0,
|
||||
1.0,
|
||||
1.0
|
||||
]
|
||||
0.0,
|
||||
0.0,
|
||||
0.0
|
||||
]
|
||||
],
|
||||
"0x2": [
|
||||
"0x3": [
|
||||
"vec3",
|
||||
[
|
||||
1.0,
|
||||
1.0,
|
||||
1.0
|
||||
]
|
||||
],
|
||||
"0x4": [
|
||||
"trans",
|
||||
{
|
||||
"position": "#0x2",
|
||||
"rotation": "#0x2",
|
||||
"scale": "#0x3"
|
||||
}
|
||||
],
|
||||
"0xa": [
|
||||
"path",
|
||||
"Assets/TestAsset.obj"
|
||||
]
|
||||
|
|
|
|||
Loading…
Reference in New Issue