Scene descriptor: custom shader uniforms, GPU wind, keyframe animation, organic instancing

- Auto-declare custom shader uniforms in GLSL (fixes undeclared-identifier
  compile failures for tint/deep/shallow-style fragment shaders)
- GPU wind shader for instanced meshes (per-instance phase attribute), no
  CPU per-instance animation
- Keyframe + pose-keyframe animation via AnimationMixer/AnimationClip
- Organic instancing variation: seeded scatter plus tilt and color jitter
- AUTHORING_GUIDE.md for manual LLM handoff
This commit is contained in:
Tanish Manku committed 2026-06-28 01:10:41 -07:00
1 parent a6620b63e8
commit 35d6abe7cb
10 files changed
+691 -19

No files matched your search

@@ -878,6 +878,9 @@ class Editor {
// Advance any in-flight entrance animations.
this.entranceAnimator.tick(delta_time);
// Advance keyframe-animation mixers (experimental scene descriptor).
this.activeScene.tickAnimations(delta_time);
this.activeScene.shader_objects.forEach((shader) => {
// Guard: custom (LLM-authored) shader materials may not carry a
// `time` uniform — only tick the ones that do.
@@ -38,6 +38,22 @@ void main() {
}
`;
// Instancing-aware default vertex shader: same varyings, but places each
// vertex through the per-instance `instanceMatrix` so a custom material on
// an InstancedMesh keeps every instance in its scattered position.
const DEFAULT_VERTEX_SHADER_INSTANCED = `
attribute mat4 instanceMatrix;
varying vec2 vUv;
varying vec3 vNormal;
varying vec3 vPosition;
void main() {
vUv = uv;
vNormal = normalize(normalMatrix * normal);
vPosition = position;
gl_Position = projectionMatrix * modelViewMatrix * instanceMatrix * vec4(position, 1.0);
}
`;
// Prepended to every custom fragment shader so the injected uniforms and
// the default vertex varyings are always in scope.
const SHADER_PREAMBLE = `
@@ -48,6 +64,40 @@ varying vec3 vNormal;
varying vec3 vPosition;
`;
// Built-in GPU wind for instanced fields (grass/foliage). Height-weighted
// sway so each instance pivots at its base, with a per-instance phase so
// the field doesn't move in unison — all on the GPU, one draw call.
const WIND_VERTEX_SHADER = `
attribute mat4 instanceMatrix;
attribute float instancePhase;
uniform float time;
uniform float windStrength;
uniform float windSpeed;
uniform float windTurbulence;
varying float vPhase;
void main() {
vec3 pos = position;
float h = max(pos.y, 0.0);
float w = sin(time * windSpeed + instancePhase)
+ windTurbulence * sin(time * windSpeed * 2.3 + instancePhase * 1.7);
float sway = w * windStrength * h;
pos.x += sway;
pos.z += sway * 0.35;
vPhase = instancePhase;
gl_Position = projectionMatrix * modelViewMatrix * instanceMatrix * vec4(pos, 1.0);
}
`;
const WIND_FRAGMENT_SHADER = `
uniform vec3 baseColor;
varying float vPhase;
void main() {
// Subtle per-instance shade variation (free organic colour).
float v = 0.85 + 0.15 * sin(vPhase * 3.1);
gl_FragColor = vec4(baseColor * v, 1.0);
}
`;
// Capabilities Scene needs from outside its own state. Editor wires
// these in inline at construction (Phase 2 idiom — same shape as
// SceneUtilsDeps, SaveManagerDeps, etc.). Scene does NOT import the
@@ -90,6 +140,10 @@ class Scene {
shader_objects: Water[] = [];
video_planes: HTMLVideoElement[] = [];
// Per-object animation mixers (experimental scene descriptor), keyed by
// object uuid. Ticked each frame from the editor render loop.
animationMixers: Map<string, THREE.AnimationMixer> = new Map();
skybox: string;
// loading indicator manager
@@ -372,11 +426,33 @@ class Scene {
): Promise<THREE.InstancedMesh> {
const instances: InstanceLike[] = spec?.instances ?? [];
const geometry = this.buildBaseGeometry(spec?.base);
const material = new THREE.MeshPhongMaterial({
color: new THREE.Color(color),
side: THREE.DoubleSide,
});
material.shininess = 0.0;
const wind = spec?.wind;
// Wind = a GPU vertex shader driven by the shared `time` uniform +
// per-instance phase (one draw call, no per-frame CPU). Otherwise a
// flat gray-box Phong material.
let material: THREE.Material;
if (wind) {
material = new THREE.ShaderMaterial({
uniforms: {
time: { value: 0 },
windStrength: { value: wind.strength ?? 0.15 },
windSpeed: { value: wind.speed ?? 1.5 },
windTurbulence: { value: wind.turbulence ?? 0 },
baseColor: { value: new THREE.Color(color) },
},
vertexShader: WIND_VERTEX_SHADER,
fragmentShader: WIND_FRAGMENT_SHADER,
side: THREE.DoubleSide,
});
} else {
const phong = new THREE.MeshPhongMaterial({
color: new THREE.Color(color),
side: THREE.DoubleSide,
});
phong.shininess = 0.0;
material = phong;
}
const count = Math.max(1, instances.length);
const mesh = new THREE.InstancedMesh(geometry, material, count);
@@ -386,7 +462,8 @@ class Scene {
const q = new THREE.Quaternion();
const e = new THREE.Euler();
const s = new THREE.Vector3(1, 1, 1);
const hasColors = instances.some((i) => !!i.color);
// Per-instance color only applies to the flat (non-wind) material.
const hasColors = !wind && instances.some((i) => !!i.color);
if (hasColors) {
mesh.instanceColor = new THREE.InstancedBufferAttribute(
new Float32Array(count * 3),
@@ -408,6 +485,20 @@ class Scene {
});
mesh.instanceMatrix.needsUpdate = true;
if (mesh.instanceColor) mesh.instanceColor.needsUpdate = true;
// Wind: per-instance phase attribute + register for the per-frame
// `time` tick so the GPU sway animates.
if (wind) {
const phases = new Float32Array(count);
for (let i = 0; i < count; i++) phases[i] = (i * 2.399963) % 6.28318;
geometry.setAttribute(
"instancePhase",
new THREE.InstancedBufferAttribute(phases, 1),
);
// eslint-disable-next-line @typescript-eslint/no-explicit-any
this.shader_objects.push(mesh as any);
}
mesh.castShadow = true;
mesh.receiveShadow = true;
mesh.name = "Instances";
@@ -436,14 +527,18 @@ class Scene {
applyShaderMaterial(object_uuid: string, spec: any): void {
const object = this.get_object_by_uuid(object_uuid);
if (!object || !spec?.fragmentShader) return;
const material = this.buildShaderMaterial(spec);
let registered = false;
object.traverse((c: THREE.Object3D) => {
if (!(c instanceof THREE.Mesh)) return;
c.material = material;
// InstancedMesh needs an instancing-aware vertex shader so each
// instance keeps its scattered position.
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const instanced = (c as any).isInstancedMesh === true;
c.material = this.buildShaderMaterial(spec, instanced);
// Register a single representative mesh for the time tick.
if (spec.animated && !registered) {
this.shader_objects.push(c);
// eslint-disable-next-line @typescript-eslint/no-explicit-any
this.shader_objects.push(c as any);
registered = true;
}
});
@@ -451,14 +546,28 @@ class Scene {
}
// eslint-disable-next-line @typescript-eslint/no-explicit-any
private buildShaderMaterial(spec: any): THREE.ShaderMaterial {
private buildShaderMaterial(spec: any, instanced = false): THREE.ShaderMaterial {
const uniforms: Record<string, THREE.IUniform> = {
time: { value: 0 },
resolution: { value: new THREE.Vector2(1, 1) },
};
// Build the value objects AND the GLSL declarations for custom
// uniforms. ShaderMaterial does NOT auto-declare uniforms in the shader
// source, so without these any custom uniform the author references
// (e.g. `tint`) is an undeclared identifier and the shader fails to
// compile. We infer the GLSL type from the value's arity.
const raw = spec.uniforms ?? {};
const customDecls: string[] = [];
for (const key of Object.keys(raw)) {
const v = raw[key];
const glslType = Array.isArray(v)
? v.length === 2
? "vec2"
: v.length === 3
? "vec3"
: "vec4"
: "float";
customDecls.push(`uniform ${glslType} ${key};`);
uniforms[key] = {
value: Array.isArray(v)
? v.length === 2
@@ -469,10 +578,16 @@ class Scene {
: v,
};
}
const customUniformGlsl = customDecls.length
? customDecls.join("\n") + "\n"
: "";
return new THREE.ShaderMaterial({
uniforms,
vertexShader: spec.vertexShader || DEFAULT_VERTEX_SHADER,
fragmentShader: SHADER_PREAMBLE + spec.fragmentShader,
vertexShader:
spec.vertexShader ||
(instanced ? DEFAULT_VERTEX_SHADER_INSTANCED : DEFAULT_VERTEX_SHADER),
fragmentShader: SHADER_PREAMBLE + customUniformGlsl + spec.fragmentShader,
transparent: !!spec.transparent,
side: spec.doubleSide ? THREE.DoubleSide : THREE.FrontSide,
});
@@ -506,6 +621,181 @@ class Scene {
}
}
// Attach (or replace) a keyframe animation on an object. Builds a
// THREE.AnimationClip from the spec's tracks (or plays a baked GLTF clip
// by name), drives it with an AnimationMixer, and registers the mixer for
// the per-frame tick. The spec is stashed on userData for round-trip.
// eslint-disable-next-line @typescript-eslint/no-explicit-any
attachAnimation(object_uuid: string, spec: any): void {
const object = this.get_object_by_uuid(object_uuid);
if (!object || !spec) return;
// Replace any existing mixer on this object.
const existing = this.animationMixers.get(object_uuid);
if (existing) {
existing.stopAllAction();
this.animationMixers.delete(object_uuid);
}
const mixer = new THREE.AnimationMixer(object);
let clip: THREE.AnimationClip | null = null;
// Baked GLTF clip by name (e.g. a mixamo "Walk") if present.
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const baked = (object as any).animations as THREE.AnimationClip[] | undefined;
if (spec.clip && baked && baked.length > 0) {
clip = THREE.AnimationClip.findByName(baked, spec.clip) ?? null;
}
if (!clip) clip = this.buildAnimationClip(spec);
if (!clip) return;
const action = mixer.clipAction(clip);
action.loop =
spec.loop === "once"
? THREE.LoopOnce
: spec.loop === "pingpong"
? THREE.LoopPingPong
: THREE.LoopRepeat;
if (spec.loop === "once") action.clampWhenFinished = true;
mixer.timeScale = spec.timeScale ?? 1;
if (spec.autoplay !== false) action.play();
this.animationMixers.set(object_uuid, mixer);
object.userData["animationSpec"] = spec;
}
// Advance all mixers. Called from the editor render loop each frame.
tickAnimations(delta: number): void {
this.animationMixers.forEach((mixer) => mixer.update(delta));
}
// Stop and drop all mixers (scene reset / reload).
clearAnimations(): void {
this.animationMixers.forEach((mixer) => mixer.stopAllAction());
this.animationMixers.clear();
}
// eslint-disable-next-line @typescript-eslint/no-explicit-any
private buildAnimationClip(spec: any): THREE.AnimationClip | null {
const tracks: THREE.KeyframeTrack[] = [];
const q = new THREE.Quaternion();
const e = new THREE.Euler();
let maxTime = 0;
// eslint-disable-next-line @typescript-eslint/no-explicit-any
for (const t of (spec.tracks ?? []) as any[]) {
const times: number[] = t.times ?? [];
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const values: any[] = t.values ?? [];
if (times.length === 0 || values.length === 0) continue;
maxTime = Math.max(maxTime, ...times);
// Track node: "" → the object itself (root), else a child by name
// (e.g. a mixamo bone). PropertyBinding resolves names in the subtree.
const node = t.target === "bone" && t.bone ? t.bone : "";
if (t.property === "position" || t.property === "scale") {
const flat: number[] = [];
for (const v of values) flat.push(v.x ?? 0, v.y ?? 0, v.z ?? 0);
const track = new THREE.VectorKeyframeTrack(
`${node}.${t.property}`,
times,
flat,
);
this.applyTrackInterpolation(track, t.interpolation);
tracks.push(track);
} else if (t.property === "rotationDeg") {
const flat: number[] = [];
for (const v of values) {
e.set(
THREE.MathUtils.degToRad(v.x ?? 0),
THREE.MathUtils.degToRad(v.y ?? 0),
THREE.MathUtils.degToRad(v.z ?? 0),
);
q.setFromEuler(e);
flat.push(q.x, q.y, q.z, q.w);
}
// Quaternion tracks interpolate spherically by default.
tracks.push(
new THREE.QuaternionKeyframeTrack(`${node}.quaternion`, times, flat),
);
}
}
// Pose-keyframes: a timeline of whole-body poses → one
// QuaternionKeyframeTrack per bone (+ a root position track). Each
// bone only gets keys at the poses that set it, so limbs interpolate
// on their own cadence. This is the character-friendly authoring path.
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const poses = spec.poses as any[] | undefined;
if (poses && poses.length > 0) {
const sorted = [...poses].sort(
(a, b) => (a.timeSec ?? 0) - (b.timeSec ?? 0),
);
const boneTimes: Record<string, number[]> = {};
const boneQuats: Record<string, number[]> = {};
const rootTimes: number[] = [];
const rootVals: number[] = [];
for (const pk of sorted) {
const t = pk.timeSec ?? 0;
maxTime = Math.max(maxTime, t);
if (pk.rootPosition) {
rootTimes.push(t);
rootVals.push(
pk.rootPosition.x ?? 0,
pk.rootPosition.y ?? 0,
pk.rootPosition.z ?? 0,
);
}
const bones = pk.bones ?? {};
for (const boneName of Object.keys(bones)) {
const rot = bones[boneName]?.rotationDeg ?? bones[boneName];
if (!rot) continue;
e.set(
THREE.MathUtils.degToRad(rot.x ?? 0),
THREE.MathUtils.degToRad(rot.y ?? 0),
THREE.MathUtils.degToRad(rot.z ?? 0),
);
q.setFromEuler(e);
if (!boneTimes[boneName]) {
boneTimes[boneName] = [];
boneQuats[boneName] = [];
}
boneTimes[boneName].push(t);
boneQuats[boneName].push(q.x, q.y, q.z, q.w);
}
}
for (const boneName of Object.keys(boneTimes)) {
tracks.push(
new THREE.QuaternionKeyframeTrack(
`${boneName}.quaternion`,
boneTimes[boneName],
boneQuats[boneName],
),
);
}
if (rootTimes.length > 0) {
tracks.push(new THREE.VectorKeyframeTrack(`.position`, rootTimes, rootVals));
}
}
if (tracks.length === 0) return null;
const duration = spec.durationSec ?? (maxTime > 0 ? maxTime : -1);
return new THREE.AnimationClip(spec.name ?? "clip", duration, tracks);
}
private applyTrackInterpolation(
track: THREE.KeyframeTrack,
interpolation: string | undefined,
): void {
if (interpolation === "smooth") track.setInterpolation(THREE.InterpolateSmooth);
else if (interpolation === "discrete")
track.setInterpolation(THREE.InterpolateDiscrete);
// default: linear (THREE.InterpolateLinear)
}
get_objects_name(uuid: string): string {
const object = this.get_object_by_uuid(uuid);
if (object) {
@@ -46,6 +46,9 @@ export interface ObjectJSON {
instancing?: any;
// eslint-disable-next-line @typescript-eslint/no-explicit-any
shaderMaterial?: any;
// Keyframe animation spec (experimental scene descriptor).
// eslint-disable-next-line @typescript-eslint/no-explicit-any
animation?: any;
// eslint-disable-next-line @typescript-eslint/no-explicit-any
user_data: Record<string, any>;
}
@@ -75,6 +78,8 @@ export class StoryTellerProxy3DObject {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
shaderMaterial?: any;
// eslint-disable-next-line @typescript-eslint/no-explicit-any
animation?: any;
// eslint-disable-next-line @typescript-eslint/no-explicit-any
userData: Record<string, any>;
constructor(version: number, media_file_token: string) {
@@ -166,6 +171,9 @@ export class StoryTellerProxy3DObject {
if (this.shaderMaterial) {
json.shaderMaterial = this.shaderMaterial;
}
if (this.animation) {
json.animation = this.animation;
}
return json;
}
}
@@ -74,6 +74,9 @@ export class StoryTellerProxyScene {
if (child.userData["shaderMaterialSpec"]) {
proxyObject3D.shaderMaterial = child.userData["shaderMaterialSpec"];
}
if (child.userData["animationSpec"]) {
proxyObject3D.animation = child.userData["animationSpec"];
}
const json_data = proxyObject3D.toJSON();
return json_data;
@@ -150,6 +153,10 @@ export class StoryTellerProxyScene {
while (this.scene.scene.children.length > 0) {
this.scene.scene.remove(this.scene.scene.children[0]);
}
// Drop any animation mixers + per-frame shader registrations from the
// previous scene before rebuilding (they're re-added as objects load).
this.scene.clearAnimations();
this.scene.shader_objects.length = 0;
// Warm Scene's URL cache up front so per-asset loadObject() calls
// never hit the network for token→URL resolution. One batch call
@@ -334,6 +341,10 @@ export class StoryTellerProxyScene {
if (json_object.shaderMaterial) {
this.scene.applyShaderMaterial(obj.uuid, json_object.shaderMaterial);
}
// Re-attach keyframe animation (drives transform/bones each frame).
if (json_object.animation) {
this.scene.attachAnimation(obj.uuid, json_object.animation);
}
}
this.scene._createGrid();
@@ -0,0 +1,251 @@
# Scene Descriptor — LLM Authoring Guide
Paste this whole document to an LLM as context, then give it the current scene
descriptor (JSON) plus an instruction ("add a forest", "make the grass sway",
"raise the left arm"). The model returns **only** the updated descriptor JSON.
You are editing a 3D scene as a single JSON document.
- Coordinates: **meters**, **Y-up**, right-handed.
- Rotations: **degrees** (Euler XYZ).
- **Gray-box** world: flat colors only, no textures.
- The scene is rebuilt from what you return: keep what should stay, change what
should change, drop what should go.
---
## Quick reference
```
descriptor { descriptorVersion:2, units, coordinateSystem, environment{skybox?}, camera?(read-only), entities[] }
entity { id, name, kind, color?, visible?, transform, + kind fields, + material?, + animation? }
transform { position:{x,y,z}, rotationDeg:{x,y,z}, scale:{x,y,z} }
kind "primitive" + shape: "Box"|"Sphere"|"Cone"|"Cylinder"|"Donut"|"Water"|"PointLight"
kind "mesh" + geometry:{ positions:[x,y,z,...] } // non-indexed triangle soup, local space
kind "instances" + instancing:{ base, scatter? | instances?, wind? }
kind "character" + pose:{ rootBone, rootPosition?, bones:{ [name]:{rotationDeg} } } // EDIT ONLY
kind "model"|"image"|"light"|"point" // EDIT ONLY
instancing.base { shape } | { positions:[...] } | { plane:{width,height} }
instancing.scatter { count, area:{x,z}, seed?, yJitterDeg?, scaleRange?[min,max], tiltDeg?, colorJitter? }
instancing.instances[ { position, rotationDeg, scale, color? } ] // explicit (instead of scatter)
instancing.wind { strength?, speed?, turbulence? } // GPU sway for ALL instances
material (shader) { type:"shader", fragmentShader, vertexShader?, uniforms?, animated?, transparent?, doubleSide? }
animation { poses?:[ {timeSec, rootPosition?, bones:{[name]:{rotationDeg}}} ], tracks?:[...], loop?, timeScale?, autoplay?, durationSec?, clip? }
poses[] ← USE THIS for characters (dance/walk): a timeline of whole-body poses, same `bones` shape as `pose`
```
## Apply semantics (how your JSON becomes the scene)
- **Edit** an existing object: keep its `id` **and its `source` field verbatim**
(lossless passthrough — never modify it). Change `transform`/`color`/`visible`/
`pose`/`material`/`animation`.
- **Create**: use `primitive`, `mesh`, or `instances`; omit `id` and `source`.
- **Delete**: omit the entity from `entities` (reconciled by id; missing ⇒ removed).
- New `character`/`model` are **skipped** — you can't summon a rigged/asset object
from text. Edit existing ones only.
- Applying is **one undoable step**; the whole change reverts together.
---
## primitive
```jsonc
{ "kind": "primitive", "shape": "Box", "color": "#6b6b6b", "transform": {...} }
```
`shape` ∈ `Box`, `Sphere`, `Cone`, `Cylinder`, `Donut`, `Water`, `PointLight`.
Each is ≈1m (radius 0.5); use `scale` to resize.
## mesh (raw vertex data)
```jsonc
{ "kind": "mesh", "color": "#aa8844", "transform": {...},
"geometry": { "positions": [x,y,z, x,y,z, ...] } }
```
`positions` is a **non-indexed triangle soup** in local space: every **9 numbers**
(3 vertices) = one triangle. Normals auto-computed. Keep counts modest.
## instances (forests, grass — one draw call)
One `instances` entity becomes **one draw call** for the whole field, however many
copies. Always use this for many copies — never emit hundreds of entities.
```jsonc
{ "kind": "instances", "color": "#2f7d32", "transform": { /* field origin */ },
"instancing": {
"base": { "shape": "Cone" }, // OR { "positions":[...blade verts...] } OR { "plane":{ "width":1,"height":1 } }
"scatter": { // procedural placement (preferred) — OR explicit "instances":[...]
"count": 350,
"area": { "x": 60, "z": 60 }, // spread centered on the field origin
"seed": 11, // deterministic — same seed = same layout
"yJitterDeg": 360, // random yaw
"scaleRange": [1.2, 3.4], // random uniform scale
"tiltDeg": 8, // random lean — organic
"colorJitter": 0.18 // 0..1 per-instance shade off `color`
}
} }
```
For **no two alike**, set `tiltDeg`, `colorJitter`, a wide `scaleRange`, and
`yJitterDeg:360`. Grass = a few-vertex blade in `base.positions` with a high
`count`; trees = `base.shape:"Cone"` with a smaller `count`.
**Wind — animate the WHOLE field on the GPU.** Add `instancing.wind` to make every
instance sway: a vertex shader on a shared `time` uniform with a per-instance phase
— one draw call, no per-frame CPU, scales to tens of thousands. Height-weighted,
so each instance pivots at its base.
```jsonc
"wind": { "strength": 0.25, "speed": 1.5, "turbulence": 0.4 }
```
> ⚠️ To animate instances, use `instancing.wind` — **do NOT** put a per-object
> `animation` on an instanced field. A per-object `animation` moves the entire
> field as one rigid block, not the individual blades.
## material (custom shader — any kind)
Give an entity's mesh(es) a custom GLSL material. Works on any kind, including
`instances` (the engine supplies instancing-aware vertex placement automatically).
```jsonc
"material": {
"type": "shader",
"animated": true, // ticks the injected `time` uniform
"doubleSide": true,
"uniforms": { "tint": [0.2, 0.8, 0.4] }, // number→float, [a,b]→vec2, [a,b,c]→vec3, [a,b,c,d]→vec4
"fragmentShader": "void main(){ float g = 0.5 + 0.5*sin(time*2.0 + vUv.x*12.0); gl_FragColor = vec4(vUv.x, g, 1.0 - vUv.y, 1.0); }"
}
```
**Shader contract — already declared for you, just use them:**
- Uniforms: `float time`, `vec2 resolution`, plus anything in your `uniforms`.
- Varyings: `vec2 vUv`, `vec3 vNormal` (view space), `vec3 vPosition` (local).
- Write only `fragmentShader` (a full `void main(){ ... gl_FragColor = ...; }`).
A `vertexShader` is optional and rarely needed.
## animation (keyframes — per object or bones)
Make an entity move over time (Three.js AnimationMixer). Times are **seconds**;
each track's `values` are one per keyframe, matching `times`.
```jsonc
"animation": {
"loop": "repeat", // "repeat" | "once" | "pingpong" (default repeat)
"timeScale": 1, // playback speed (default 1)
"autoplay": true, // (default true)
"durationSec": 2, // optional; inferred from times if omitted
"tracks": [
{ "property": "position", "times": [0,1,2], "values": [ {"x":0,"y":1,"z":0}, {"x":0,"y":3,"z":0}, {"x":0,"y":1,"z":0} ] },
{ "property": "rotationDeg", "times": [0,2], "values": [ {"x":0,"y":0,"z":0}, {"x":0,"y":360,"z":0} ] }
// property ∈ "position" | "rotationDeg" (degrees) | "scale"
// optional "interpolation": "linear" (default) | "smooth" | "discrete"
]
}
```
- A track overrides that channel of `transform` while playing; omitted channels stay static.
- Play a **baked clip** the asset ships with: `"animation": { "clip": "Walk", "loop": "repeat" }` (falls back to `tracks`/`poses` if absent).
- For many copies, this is per-OBJECT — for swaying grass use `instancing.wind` instead.
### Character motion — pose-keyframes (USE THIS for characters)
Don't hand-author per-bone tracks for a character. Author a **timeline of
whole-body poses** — the same sparse `bones: { name: { rotationDeg } }` shape as
the static `pose`, plus a `timeSec`. The engine interpolates between them. Block a
few key poses; a bone you omit at a keyframe just interpolates between the
keyframes that *do* set it, so each limb can move on its own cadence.
```jsonc
"animation": {
"loop": "repeat",
"poses": [
{ "timeSec": 0.0, "bones": { "mixamorigLeftArm": {"rotationDeg":{"x":0,"y":0,"z":-70}}, "mixamorigRightArm": {"rotationDeg":{"x":0,"y":0,"z":70}} } },
{ "timeSec": 0.5, "bones": { "mixamorigLeftArm": {"rotationDeg":{"x":0,"y":0,"z":-30}}, "mixamorigRightArm": {"rotationDeg":{"x":0,"y":0,"z":30}} } },
{ "timeSec": 1.0, "bones": { "mixamorigLeftArm": {"rotationDeg":{"x":0,"y":0,"z":-70}}, "mixamorigRightArm": {"rotationDeg":{"x":0,"y":0,"z":70}} } }
]
}
```
- **Dance / wave / idle:** a short loop of 2–4 key poses with `loop:"repeat"`.
- **Walk / run in place:** alternate two contact poses (legs + counter-swinging arms), `loop:"repeat"`.
- **Walk along a path (road / through the forest):** add `rootPosition` to keyframes
to move the hips, AND repeat the leg/arm contact poses along the journey, in ONE
clip. Use `loop:"once"` for a one-way trip (e.g. waypoints weaving between trees),
or `"repeat"` to loop the route. Example skeleton:
```jsonc
"animation": { "loop": "once", "durationSec": 8, "poses": [
{ "timeSec": 0, "rootPosition": {"x":-20,"y":0,"z":0}, "bones": { /* contact pose A */ } },
{ "timeSec": 0.5, "bones": { /* contact pose B */ } },
{ "timeSec": 1.0, "rootPosition": {"x":-17,"y":0,"z":3}, "bones": { /* pose A */ } },
/* …repeat the A/B leg cycle every ~0.5s while stepping rootPosition along the path… */
{ "timeSec": 8, "rootPosition": {"x":20,"y":0,"z":-5}, "bones": { /* pose A */ } }
] }
```
(Add `mixamorigHips` rotation in the poses to make the character face its travel direction.)
## character pose (mixamo) — edit only
```jsonc
{ "kind": "character", "id": "keep-this", "source": {/* keep verbatim */}, "transform": {...},
"pose": {
"rootBone": "mixamorigHips",
"rootPosition": { "x": 0, "y": 1.0, "z": 0 }, // optional hip translation
"bones": {
"mixamorigLeftArm": { "rotationDeg": { "x": 0, "y": 0, "z": -60 } },
"mixamorigRightArm": { "rotationDeg": { "x": 0, "y": 0, "z": 60 } },
"mixamorigSpine": { "rotationDeg": { "x": 8, "y": 0, "z": 0 } }
}
} }
```
`bones` is sparse — list only what you change (local rotation, degrees). Omitted
bones keep their current pose. The **same `bones` shape is used for static `pose`
and for `animation.poses`** — pose it once, or key it over time.
**The character must already be in the scene.** You'll see it as `kind:"character"`
with its current rotations in `pose.bones` — that exported pose (≈ a T-pose at rest)
is your **starting point**; adjust from there.
**Mixamo posable bones (and what moves them from the T-pose rest):**
- **Spine/head:** `mixamorigSpine`, `mixamorigSpine1`, `mixamorigSpine2` (bend/twist torso), `mixamorigNeck`, `mixamorigHead`.
- **Arms** (rest = arms out along ±X): `mixamorigLeftShoulder`, `mixamorigLeftArm` (**z** raises/lowers the arm), `mixamorigLeftForeArm` (bends the elbow), `mixamorigLeftHand`; mirror with `Right`.
- **Legs:** `mixamorigLeftUpLeg` (**x** swings the thigh fwd/back), `mixamorigLeftLeg` (**x** bends the knee), `mixamorigLeftFoot`; mirror with `Right`.
- **Root:** `mixamorigHips` (whole-body lean/turn) + `rootPosition` for translation.
Axis signs differ per rig, and **left/right are mirrored** (flip the sign on the
swinging axis). Start from the exported current rotation, nudge, and verify.
---
## Full example (forest + windy grass + animated shader)
```json
{
"descriptorVersion": 2, "units": "meters", "coordinateSystem": "y-up-right-handed", "environment": {},
"entities": [
{ "id": "ground", "name": "Ground", "kind": "primitive", "shape": "Box", "color": "#5b6b4a",
"transform": { "position": {"x":0,"y":-0.5,"z":0}, "rotationDeg": {"x":0,"y":0,"z":0}, "scale": {"x":80,"y":0.2,"z":80} } },
{ "id": "forest", "name": "Forest", "kind": "instances", "color": "#2f7d32",
"transform": { "position": {"x":0,"y":1,"z":0}, "rotationDeg": {"x":0,"y":0,"z":0}, "scale": {"x":1,"y":1,"z":1} },
"instancing": { "base": { "shape": "Cone" },
"scatter": { "count": 350, "area": {"x":60,"z":60}, "seed": 11, "yJitterDeg": 360, "scaleRange": [1.2,3.4], "tiltDeg": 8, "colorJitter": 0.18 } } },
{ "id": "grass", "name": "Grass", "kind": "instances", "color": "#5a8f3a",
"transform": { "position": {"x":0,"y":0,"z":0}, "rotationDeg": {"x":0,"y":0,"z":0}, "scale": {"x":1,"y":1,"z":1} },
"instancing": { "base": { "positions": [-0.05,0,0, 0.05,0,0, 0.05,0.5,0, -0.05,0,0, 0.05,0.5,0, -0.05,0.5,0] },
"scatter": { "count": 4000, "area": {"x":50,"z":50}, "seed": 3, "scaleRange": [0.6,1.6] },
"wind": { "strength": 0.25, "speed": 1.6, "turbulence": 0.4 } } },
{ "id": "orb", "name": "Shader Orb", "kind": "primitive", "shape": "Sphere", "color": "#ffffff",
"transform": { "position": {"x":0,"y":4,"z":0}, "rotationDeg": {"x":0,"y":0,"z":0}, "scale": {"x":2,"y":2,"z":2} },
"material": { "type": "shader", "animated": true, "doubleSide": true,
"fragmentShader": "void main(){ float p=0.5+0.5*sin(time*3.0); float f=pow(1.0-abs(vNormal.z),2.0); gl_FragColor=vec4(f+p*0.3, f, 1.0, 1.0); }" },
"animation": { "loop": "repeat", "tracks": [ { "property": "position", "times": [0,2,4], "values": [ {"x":0,"y":4,"z":0}, {"x":0,"y":5,"z":0}, {"x":0,"y":4,"z":0} ] } ] } }
]
}
```
## Checklist before returning
- Output is **valid JSON only** — the whole descriptor, no prose, no trailing commas.
- Kept `id` and `source` for every existing object you intend to preserve.
- Rotations in **degrees**; positions in **meters**, Y-up; colors flat hex.
- Used `instances` + `scatter` for many copies; `instancing.wind` (not `animation`) to animate a field.
- New objects are only `primitive` / `mesh` / `instances` (no new `character`/`model`).
@@ -176,6 +176,11 @@ function updateObjectInPlace(
if (entity.material?.fragmentShader) {
scene.applyShaderMaterial(obj.uuid, entity.material);
}
// Keyframe animation — drives transform/bones each frame.
if (entity.animation) {
scene.attachAnimation(obj.uuid, entity.animation);
}
}
// Instantiate a primitive locally (no network) and stamp the userData the
@@ -242,7 +247,10 @@ async function createInstances(
scene: Scene,
entity: DescriptorEntity,
): Promise<THREE.Object3D | undefined> {
const resolved = resolveInstancing(entity.instancing);
const resolved = resolveInstancing(
entity.instancing,
entity.color || GRAY_BOX_COLOR,
);
if (!resolved?.base) return undefined;
const created = await scene.instantiateInstancedMesh(
@@ -106,6 +106,9 @@ function objectJsonToEntity(
if (obj.shaderMaterial) {
entity.material = obj.shaderMaterial;
}
if (obj.animation) {
entity.animation = obj.animation;
}
return entity;
}
@@ -17,8 +17,12 @@ export type {
DescriptorInstancing,
InstancingBase,
InstancingScatter,
InstancingWind,
InstanceTransform,
ShaderMaterialSpec,
AnimationSpec,
AnimationTrack,
PoseKeyframe,
BonePose,
Vec3,
} from "./scene_descriptor";
@@ -3,6 +3,7 @@
// expand it deterministically here so apply just consumes a flat instance
// list. Same seed → same layout (reproducible across export/apply/undo).
import * as THREE from "three";
import {
DescriptorInstancing,
InstanceTransform,
@@ -10,38 +11,69 @@ import {
} from "./scene_descriptor";
// Return a spec with concrete `instances`. Explicit instances win; a
// scatter spec is expanded; otherwise the spec is returned as-is.
// scatter spec is expanded; otherwise the spec is returned as-is. The base
// color (the entity's color) seeds per-instance color jitter.
export function resolveInstancing(
spec: DescriptorInstancing | undefined,
baseColor?: string,
): DescriptorInstancing | undefined {
if (!spec) return spec;
if (spec.instances && spec.instances.length > 0) return spec;
if (spec.scatter) {
return { base: spec.base, instances: scatterInstances(spec.scatter) };
return {
base: spec.base,
instances: scatterInstances(spec.scatter, baseColor),
wind: spec.wind,
};
}
return spec;
}
function scatterInstances(s: InstancingScatter): InstanceTransform[] {
function scatterInstances(
s: InstancingScatter,
baseColor?: string,
): InstanceTransform[] {
const rng = mulberry32(s.seed ?? 1);
const yaw = s.yJitterDeg ?? 360;
const tilt = s.tiltDeg ?? 0;
const jitter = s.colorJitter ?? 0;
const [sMin, sMax] = s.scaleRange ?? [1, 1];
const count = Math.max(0, Math.floor(s.count));
// Per-instance color is derived from the base color in HSL so the field
// reads as one species with natural shade-to-shade variation.
const base = jitter > 0 && baseColor ? new THREE.Color(baseColor) : null;
const hsl = { h: 0, s: 0, l: 0 };
if (base) base.getHSL(hsl);
const out: InstanceTransform[] = [];
for (let i = 0; i < count; i++) {
const x = (rng() - 0.5) * s.area.x;
const z = (rng() - 0.5) * s.area.z;
const ry = (rng() - 0.5) * yaw;
const tx = (rng() - 0.5) * tilt;
const tz = (rng() - 0.5) * tilt;
const sc = sMin + rng() * (sMax - sMin);
out.push({
const inst: InstanceTransform = {
position: { x: round(x), y: 0, z: round(z) },
rotationDeg: { x: 0, y: round(ry), z: 0 },
rotationDeg: { x: round(tx), y: round(ry), z: round(tz) },
scale: { x: round(sc), y: round(sc), z: round(sc) },
});
};
if (base) {
const h = (hsl.h + (rng() - 0.5) * jitter * 0.1 + 1) % 1;
const l = clamp01(hsl.l + (rng() - 0.5) * jitter);
inst.color = "#" + new THREE.Color().setHSL(h, hsl.s, l).getHexString();
}
out.push(inst);
}
return out;
}
function clamp01(n: number): number {
return n < 0 ? 0 : n > 1 ? 1 : n;
}
// Small, fast, seedable PRNG (mulberry32) — deterministic per seed.
function mulberry32(seed: number): () => number {
let a = seed >>> 0;
@@ -81,6 +81,10 @@ export interface DescriptorEntity {
// The LLM-authored "custom mat" path. See ShaderMaterialSpec.
material?: ShaderMaterialSpec;
// Keyframe animation that plays on this entity (transform and, for
// characters, bones). See AnimationSpec.
animation?: AnimationSpec;
// Flat hex color, e.g. "#9a9a9a". Absent ⇒ gray-box default.
color?: string;
visible?: boolean;
@@ -135,12 +139,25 @@ export interface InstanceTransform {
// Procedural placement — far cheaper for an LLM to emit than N explicit
// instances. Deterministic given `seed`. Expanded to instances on apply.
// The jitter knobs give an organic look (no two instances alike).
export interface InstancingScatter {
count: number;
area: { x: number; z: number }; // spread, centered on the field origin
seed?: number;
yJitterDeg?: number; // random yaw range, default 360
scaleRange?: [number, number]; // uniform scale range, default [1,1]
tiltDeg?: number; // random lean on X/Z, default 0 (e.g. 8 for trees)
colorJitter?: number; // 0..1 per-instance HSL variance off the base color
}
// GPU wind animation for an instanced field — a vertex shader driven by a
// shared `time` uniform + per-instance phase. All instances sway together
// (phase-shifted), one draw call, no per-frame CPU. Height-weighted, so
// each instance pivots at its base (grass/foliage).
export interface InstancingWind {
strength?: number; // sway distance at the tip, default 0.15
speed?: number; // oscillation speed, default 1.5
turbulence?: number; // adds a second frequency for choppier motion, default 0
}
export interface DescriptorInstancing {
@@ -148,6 +165,8 @@ export interface DescriptorInstancing {
// Provide explicit instances OR a scatter spec (scatter wins if both).
instances?: InstanceTransform[];
scatter?: InstancingScatter;
// Optional GPU wind animation applied to every instance.
wind?: InstancingWind;
}
// ── Custom shader material (LLM-authored) ───────────────────────────────
@@ -166,6 +185,49 @@ export interface ShaderMaterialSpec {
animated?: boolean; // tick the injected `time` uniform each frame
}
// ── Keyframe animation ──────────────────────────────────────────────────
// One animated channel. Times are seconds; values are one per keyframe.
// - property "position" / "scale": values are Vec3 (world/local units)
// - property "rotationDeg": values are Vec3 Euler degrees
// target "object" (default) animates the entity itself; target "bone"
// animates a named mixamo bone (characters only).
export interface AnimationTrack {
target?: "object" | "bone";
bone?: string;
property: "position" | "rotationDeg" | "scale";
times: number[];
values: Vec3[];
interpolation?: "linear" | "smooth" | "discrete";
}
// A whole-body keyframe: the character's pose at one moment. Same sparse
// shape as a static `pose` (only the bones you set), plus a time. This is
// the easy way to author character motion — block a few key poses and the
// engine interpolates between them. Bones omitted at a given keyframe just
// don't get a key there (they interpolate between the keys that do set
// them), so each limb can move on its own cadence.
export interface PoseKeyframe {
timeSec: number;
rootPosition?: Vec3; // hip translation at this keyframe (for locomotion)
bones: Record<string, BonePose>;
}
// A keyframe animation on one entity, driven by a THREE.AnimationMixer.
// Author it ONE of these ways:
// - poses: a timeline of whole-body poses (best for characters: dance, walk)
// - tracks: per-channel curves (object transform or individual bones)
// - clip: play a baked GLTF clip by name (e.g. a mixamo "Walk")
export interface AnimationSpec {
clip?: string;
durationSec?: number; // inferred from key times if omitted
loop?: "repeat" | "once" | "pingpong"; // default repeat
timeScale?: number; // default 1
autoplay?: boolean; // default true
poses?: PoseKeyframe[];
tracks?: AnimationTrack[];
}
// One bone's editable local transform. Rotation only for now — that's
// what FK posing manipulates; bone translation/scale are rarely posed.
export interface BonePose {