diff --git a/frontend/libs/components/pagescene/src/lib/engine/editor.ts b/frontend/libs/components/pagescene/src/lib/engine/editor.ts index 57f851da17..ee236d14ff 100644 --- a/frontend/libs/components/pagescene/src/lib/engine/editor.ts +++ b/frontend/libs/components/pagescene/src/lib/engine/editor.ts @@ -878,6 +878,9 @@ class Editor { // Advance any in-flight entrance animations. this.entranceAnimator.tick(delta_time); + // Advance keyframe-animation mixers (experimental scene descriptor). + this.activeScene.tickAnimations(delta_time); + this.activeScene.shader_objects.forEach((shader) => { // Guard: custom (LLM-authored) shader materials may not carry a // `time` uniform — only tick the ones that do. diff --git a/frontend/libs/components/pagescene/src/lib/engine/scene.ts b/frontend/libs/components/pagescene/src/lib/engine/scene.ts index dd8335fb44..450f14c1be 100644 --- a/frontend/libs/components/pagescene/src/lib/engine/scene.ts +++ b/frontend/libs/components/pagescene/src/lib/engine/scene.ts @@ -38,6 +38,22 @@ void main() { } `; +// Instancing-aware default vertex shader: same varyings, but places each +// vertex through the per-instance `instanceMatrix` so a custom material on +// an InstancedMesh keeps every instance in its scattered position. +const DEFAULT_VERTEX_SHADER_INSTANCED = ` +attribute mat4 instanceMatrix; +varying vec2 vUv; +varying vec3 vNormal; +varying vec3 vPosition; +void main() { + vUv = uv; + vNormal = normalize(normalMatrix * normal); + vPosition = position; + gl_Position = projectionMatrix * modelViewMatrix * instanceMatrix * vec4(position, 1.0); +} +`; + // Prepended to every custom fragment shader so the injected uniforms and // the default vertex varyings are always in scope. const SHADER_PREAMBLE = ` @@ -48,6 +64,40 @@ varying vec3 vNormal; varying vec3 vPosition; `; +// Built-in GPU wind for instanced fields (grass/foliage). Height-weighted +// sway so each instance pivots at its base, with a per-instance phase so +// the field doesn't move in unison — all on the GPU, one draw call. +const WIND_VERTEX_SHADER = ` +attribute mat4 instanceMatrix; +attribute float instancePhase; +uniform float time; +uniform float windStrength; +uniform float windSpeed; +uniform float windTurbulence; +varying float vPhase; +void main() { + vec3 pos = position; + float h = max(pos.y, 0.0); + float w = sin(time * windSpeed + instancePhase) + + windTurbulence * sin(time * windSpeed * 2.3 + instancePhase * 1.7); + float sway = w * windStrength * h; + pos.x += sway; + pos.z += sway * 0.35; + vPhase = instancePhase; + gl_Position = projectionMatrix * modelViewMatrix * instanceMatrix * vec4(pos, 1.0); +} +`; + +const WIND_FRAGMENT_SHADER = ` +uniform vec3 baseColor; +varying float vPhase; +void main() { + // Subtle per-instance shade variation (free organic colour). + float v = 0.85 + 0.15 * sin(vPhase * 3.1); + gl_FragColor = vec4(baseColor * v, 1.0); +} +`; + // Capabilities Scene needs from outside its own state. Editor wires // these in inline at construction (Phase 2 idiom — same shape as // SceneUtilsDeps, SaveManagerDeps, etc.). Scene does NOT import the @@ -90,6 +140,10 @@ class Scene { shader_objects: Water[] = []; video_planes: HTMLVideoElement[] = []; + // Per-object animation mixers (experimental scene descriptor), keyed by + // object uuid. Ticked each frame from the editor render loop. + animationMixers: Map = new Map(); + skybox: string; // loading indicator manager @@ -372,11 +426,33 @@ class Scene { ): Promise { const instances: InstanceLike[] = spec?.instances ?? []; const geometry = this.buildBaseGeometry(spec?.base); - const material = new THREE.MeshPhongMaterial({ - color: new THREE.Color(color), - side: THREE.DoubleSide, - }); - material.shininess = 0.0; + const wind = spec?.wind; + + // Wind = a GPU vertex shader driven by the shared `time` uniform + + // per-instance phase (one draw call, no per-frame CPU). Otherwise a + // flat gray-box Phong material. + let material: THREE.Material; + if (wind) { + material = new THREE.ShaderMaterial({ + uniforms: { + time: { value: 0 }, + windStrength: { value: wind.strength ?? 0.15 }, + windSpeed: { value: wind.speed ?? 1.5 }, + windTurbulence: { value: wind.turbulence ?? 0 }, + baseColor: { value: new THREE.Color(color) }, + }, + vertexShader: WIND_VERTEX_SHADER, + fragmentShader: WIND_FRAGMENT_SHADER, + side: THREE.DoubleSide, + }); + } else { + const phong = new THREE.MeshPhongMaterial({ + color: new THREE.Color(color), + side: THREE.DoubleSide, + }); + phong.shininess = 0.0; + material = phong; + } const count = Math.max(1, instances.length); const mesh = new THREE.InstancedMesh(geometry, material, count); @@ -386,7 +462,8 @@ class Scene { const q = new THREE.Quaternion(); const e = new THREE.Euler(); const s = new THREE.Vector3(1, 1, 1); - const hasColors = instances.some((i) => !!i.color); + // Per-instance color only applies to the flat (non-wind) material. + const hasColors = !wind && instances.some((i) => !!i.color); if (hasColors) { mesh.instanceColor = new THREE.InstancedBufferAttribute( new Float32Array(count * 3), @@ -408,6 +485,20 @@ class Scene { }); mesh.instanceMatrix.needsUpdate = true; if (mesh.instanceColor) mesh.instanceColor.needsUpdate = true; + + // Wind: per-instance phase attribute + register for the per-frame + // `time` tick so the GPU sway animates. + if (wind) { + const phases = new Float32Array(count); + for (let i = 0; i < count; i++) phases[i] = (i * 2.399963) % 6.28318; + geometry.setAttribute( + "instancePhase", + new THREE.InstancedBufferAttribute(phases, 1), + ); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + this.shader_objects.push(mesh as any); + } + mesh.castShadow = true; mesh.receiveShadow = true; mesh.name = "Instances"; @@ -436,14 +527,18 @@ class Scene { applyShaderMaterial(object_uuid: string, spec: any): void { const object = this.get_object_by_uuid(object_uuid); if (!object || !spec?.fragmentShader) return; - const material = this.buildShaderMaterial(spec); let registered = false; object.traverse((c: THREE.Object3D) => { if (!(c instanceof THREE.Mesh)) return; - c.material = material; + // InstancedMesh needs an instancing-aware vertex shader so each + // instance keeps its scattered position. + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const instanced = (c as any).isInstancedMesh === true; + c.material = this.buildShaderMaterial(spec, instanced); // Register a single representative mesh for the time tick. if (spec.animated && !registered) { - this.shader_objects.push(c); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + this.shader_objects.push(c as any); registered = true; } }); @@ -451,14 +546,28 @@ class Scene { } // eslint-disable-next-line @typescript-eslint/no-explicit-any - private buildShaderMaterial(spec: any): THREE.ShaderMaterial { + private buildShaderMaterial(spec: any, instanced = false): THREE.ShaderMaterial { const uniforms: Record = { time: { value: 0 }, resolution: { value: new THREE.Vector2(1, 1) }, }; + // Build the value objects AND the GLSL declarations for custom + // uniforms. ShaderMaterial does NOT auto-declare uniforms in the shader + // source, so without these any custom uniform the author references + // (e.g. `tint`) is an undeclared identifier and the shader fails to + // compile. We infer the GLSL type from the value's arity. const raw = spec.uniforms ?? {}; + const customDecls: string[] = []; for (const key of Object.keys(raw)) { const v = raw[key]; + const glslType = Array.isArray(v) + ? v.length === 2 + ? "vec2" + : v.length === 3 + ? "vec3" + : "vec4" + : "float"; + customDecls.push(`uniform ${glslType} ${key};`); uniforms[key] = { value: Array.isArray(v) ? v.length === 2 @@ -469,10 +578,16 @@ class Scene { : v, }; } + const customUniformGlsl = customDecls.length + ? customDecls.join("\n") + "\n" + : ""; + return new THREE.ShaderMaterial({ uniforms, - vertexShader: spec.vertexShader || DEFAULT_VERTEX_SHADER, - fragmentShader: SHADER_PREAMBLE + spec.fragmentShader, + vertexShader: + spec.vertexShader || + (instanced ? DEFAULT_VERTEX_SHADER_INSTANCED : DEFAULT_VERTEX_SHADER), + fragmentShader: SHADER_PREAMBLE + customUniformGlsl + spec.fragmentShader, transparent: !!spec.transparent, side: spec.doubleSide ? THREE.DoubleSide : THREE.FrontSide, }); @@ -506,6 +621,181 @@ class Scene { } } + // Attach (or replace) a keyframe animation on an object. Builds a + // THREE.AnimationClip from the spec's tracks (or plays a baked GLTF clip + // by name), drives it with an AnimationMixer, and registers the mixer for + // the per-frame tick. The spec is stashed on userData for round-trip. + // eslint-disable-next-line @typescript-eslint/no-explicit-any + attachAnimation(object_uuid: string, spec: any): void { + const object = this.get_object_by_uuid(object_uuid); + if (!object || !spec) return; + + // Replace any existing mixer on this object. + const existing = this.animationMixers.get(object_uuid); + if (existing) { + existing.stopAllAction(); + this.animationMixers.delete(object_uuid); + } + + const mixer = new THREE.AnimationMixer(object); + let clip: THREE.AnimationClip | null = null; + + // Baked GLTF clip by name (e.g. a mixamo "Walk") if present. + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const baked = (object as any).animations as THREE.AnimationClip[] | undefined; + if (spec.clip && baked && baked.length > 0) { + clip = THREE.AnimationClip.findByName(baked, spec.clip) ?? null; + } + if (!clip) clip = this.buildAnimationClip(spec); + if (!clip) return; + + const action = mixer.clipAction(clip); + action.loop = + spec.loop === "once" + ? THREE.LoopOnce + : spec.loop === "pingpong" + ? THREE.LoopPingPong + : THREE.LoopRepeat; + if (spec.loop === "once") action.clampWhenFinished = true; + mixer.timeScale = spec.timeScale ?? 1; + if (spec.autoplay !== false) action.play(); + + this.animationMixers.set(object_uuid, mixer); + object.userData["animationSpec"] = spec; + } + + // Advance all mixers. Called from the editor render loop each frame. + tickAnimations(delta: number): void { + this.animationMixers.forEach((mixer) => mixer.update(delta)); + } + + // Stop and drop all mixers (scene reset / reload). + clearAnimations(): void { + this.animationMixers.forEach((mixer) => mixer.stopAllAction()); + this.animationMixers.clear(); + } + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + private buildAnimationClip(spec: any): THREE.AnimationClip | null { + const tracks: THREE.KeyframeTrack[] = []; + const q = new THREE.Quaternion(); + const e = new THREE.Euler(); + let maxTime = 0; + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + for (const t of (spec.tracks ?? []) as any[]) { + const times: number[] = t.times ?? []; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const values: any[] = t.values ?? []; + if (times.length === 0 || values.length === 0) continue; + maxTime = Math.max(maxTime, ...times); + + // Track node: "" → the object itself (root), else a child by name + // (e.g. a mixamo bone). PropertyBinding resolves names in the subtree. + const node = t.target === "bone" && t.bone ? t.bone : ""; + + if (t.property === "position" || t.property === "scale") { + const flat: number[] = []; + for (const v of values) flat.push(v.x ?? 0, v.y ?? 0, v.z ?? 0); + const track = new THREE.VectorKeyframeTrack( + `${node}.${t.property}`, + times, + flat, + ); + this.applyTrackInterpolation(track, t.interpolation); + tracks.push(track); + } else if (t.property === "rotationDeg") { + const flat: number[] = []; + for (const v of values) { + e.set( + THREE.MathUtils.degToRad(v.x ?? 0), + THREE.MathUtils.degToRad(v.y ?? 0), + THREE.MathUtils.degToRad(v.z ?? 0), + ); + q.setFromEuler(e); + flat.push(q.x, q.y, q.z, q.w); + } + // Quaternion tracks interpolate spherically by default. + tracks.push( + new THREE.QuaternionKeyframeTrack(`${node}.quaternion`, times, flat), + ); + } + } + + // Pose-keyframes: a timeline of whole-body poses → one + // QuaternionKeyframeTrack per bone (+ a root position track). Each + // bone only gets keys at the poses that set it, so limbs interpolate + // on their own cadence. This is the character-friendly authoring path. + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const poses = spec.poses as any[] | undefined; + if (poses && poses.length > 0) { + const sorted = [...poses].sort( + (a, b) => (a.timeSec ?? 0) - (b.timeSec ?? 0), + ); + const boneTimes: Record = {}; + const boneQuats: Record = {}; + const rootTimes: number[] = []; + const rootVals: number[] = []; + + for (const pk of sorted) { + const t = pk.timeSec ?? 0; + maxTime = Math.max(maxTime, t); + if (pk.rootPosition) { + rootTimes.push(t); + rootVals.push( + pk.rootPosition.x ?? 0, + pk.rootPosition.y ?? 0, + pk.rootPosition.z ?? 0, + ); + } + const bones = pk.bones ?? {}; + for (const boneName of Object.keys(bones)) { + const rot = bones[boneName]?.rotationDeg ?? bones[boneName]; + if (!rot) continue; + e.set( + THREE.MathUtils.degToRad(rot.x ?? 0), + THREE.MathUtils.degToRad(rot.y ?? 0), + THREE.MathUtils.degToRad(rot.z ?? 0), + ); + q.setFromEuler(e); + if (!boneTimes[boneName]) { + boneTimes[boneName] = []; + boneQuats[boneName] = []; + } + boneTimes[boneName].push(t); + boneQuats[boneName].push(q.x, q.y, q.z, q.w); + } + } + + for (const boneName of Object.keys(boneTimes)) { + tracks.push( + new THREE.QuaternionKeyframeTrack( + `${boneName}.quaternion`, + boneTimes[boneName], + boneQuats[boneName], + ), + ); + } + if (rootTimes.length > 0) { + tracks.push(new THREE.VectorKeyframeTrack(`.position`, rootTimes, rootVals)); + } + } + + if (tracks.length === 0) return null; + const duration = spec.durationSec ?? (maxTime > 0 ? maxTime : -1); + return new THREE.AnimationClip(spec.name ?? "clip", duration, tracks); + } + + private applyTrackInterpolation( + track: THREE.KeyframeTrack, + interpolation: string | undefined, + ): void { + if (interpolation === "smooth") track.setInterpolation(THREE.InterpolateSmooth); + else if (interpolation === "discrete") + track.setInterpolation(THREE.InterpolateDiscrete); + // default: linear (THREE.InterpolateLinear) + } + get_objects_name(uuid: string): string { const object = this.get_object_by_uuid(uuid); if (object) { diff --git a/frontend/libs/components/pagescene/src/lib/proxy/storyteller_proxy_3d_object.ts b/frontend/libs/components/pagescene/src/lib/proxy/storyteller_proxy_3d_object.ts index d396925c3f..7240d70406 100644 --- a/frontend/libs/components/pagescene/src/lib/proxy/storyteller_proxy_3d_object.ts +++ b/frontend/libs/components/pagescene/src/lib/proxy/storyteller_proxy_3d_object.ts @@ -46,6 +46,9 @@ export interface ObjectJSON { instancing?: any; // eslint-disable-next-line @typescript-eslint/no-explicit-any shaderMaterial?: any; + // Keyframe animation spec (experimental scene descriptor). + // eslint-disable-next-line @typescript-eslint/no-explicit-any + animation?: any; // eslint-disable-next-line @typescript-eslint/no-explicit-any user_data: Record; } @@ -75,6 +78,8 @@ export class StoryTellerProxy3DObject { // eslint-disable-next-line @typescript-eslint/no-explicit-any shaderMaterial?: any; // eslint-disable-next-line @typescript-eslint/no-explicit-any + animation?: any; + // eslint-disable-next-line @typescript-eslint/no-explicit-any userData: Record; constructor(version: number, media_file_token: string) { @@ -166,6 +171,9 @@ export class StoryTellerProxy3DObject { if (this.shaderMaterial) { json.shaderMaterial = this.shaderMaterial; } + if (this.animation) { + json.animation = this.animation; + } return json; } } diff --git a/frontend/libs/components/pagescene/src/lib/proxy/storyteller_proxy_scene.ts b/frontend/libs/components/pagescene/src/lib/proxy/storyteller_proxy_scene.ts index dd2ad86f51..21de663f9e 100644 --- a/frontend/libs/components/pagescene/src/lib/proxy/storyteller_proxy_scene.ts +++ b/frontend/libs/components/pagescene/src/lib/proxy/storyteller_proxy_scene.ts @@ -74,6 +74,9 @@ export class StoryTellerProxyScene { if (child.userData["shaderMaterialSpec"]) { proxyObject3D.shaderMaterial = child.userData["shaderMaterialSpec"]; } + if (child.userData["animationSpec"]) { + proxyObject3D.animation = child.userData["animationSpec"]; + } const json_data = proxyObject3D.toJSON(); return json_data; @@ -150,6 +153,10 @@ export class StoryTellerProxyScene { while (this.scene.scene.children.length > 0) { this.scene.scene.remove(this.scene.scene.children[0]); } + // Drop any animation mixers + per-frame shader registrations from the + // previous scene before rebuilding (they're re-added as objects load). + this.scene.clearAnimations(); + this.scene.shader_objects.length = 0; // Warm Scene's URL cache up front so per-asset loadObject() calls // never hit the network for token→URL resolution. One batch call @@ -334,6 +341,10 @@ export class StoryTellerProxyScene { if (json_object.shaderMaterial) { this.scene.applyShaderMaterial(obj.uuid, json_object.shaderMaterial); } + // Re-attach keyframe animation (drives transform/bones each frame). + if (json_object.animation) { + this.scene.attachAnimation(obj.uuid, json_object.animation); + } } this.scene._createGrid(); diff --git a/frontend/libs/components/pagescene/src/lib/scene_descriptor/AUTHORING_GUIDE.md b/frontend/libs/components/pagescene/src/lib/scene_descriptor/AUTHORING_GUIDE.md new file mode 100644 index 0000000000..12f7337158 --- /dev/null +++ b/frontend/libs/components/pagescene/src/lib/scene_descriptor/AUTHORING_GUIDE.md @@ -0,0 +1,251 @@ +# Scene Descriptor — LLM Authoring Guide + +Paste this whole document to an LLM as context, then give it the current scene +descriptor (JSON) plus an instruction ("add a forest", "make the grass sway", +"raise the left arm"). The model returns **only** the updated descriptor JSON. + +You are editing a 3D scene as a single JSON document. +- Coordinates: **meters**, **Y-up**, right-handed. +- Rotations: **degrees** (Euler XYZ). +- **Gray-box** world: flat colors only, no textures. +- The scene is rebuilt from what you return: keep what should stay, change what + should change, drop what should go. + +--- + +## Quick reference + +``` +descriptor { descriptorVersion:2, units, coordinateSystem, environment{skybox?}, camera?(read-only), entities[] } +entity { id, name, kind, color?, visible?, transform, + kind fields, + material?, + animation? } +transform { position:{x,y,z}, rotationDeg:{x,y,z}, scale:{x,y,z} } + +kind "primitive" + shape: "Box"|"Sphere"|"Cone"|"Cylinder"|"Donut"|"Water"|"PointLight" +kind "mesh" + geometry:{ positions:[x,y,z,...] } // non-indexed triangle soup, local space +kind "instances" + instancing:{ base, scatter? | instances?, wind? } +kind "character" + pose:{ rootBone, rootPosition?, bones:{ [name]:{rotationDeg} } } // EDIT ONLY +kind "model"|"image"|"light"|"point" // EDIT ONLY + +instancing.base { shape } | { positions:[...] } | { plane:{width,height} } +instancing.scatter { count, area:{x,z}, seed?, yJitterDeg?, scaleRange?[min,max], tiltDeg?, colorJitter? } +instancing.instances[ { position, rotationDeg, scale, color? } ] // explicit (instead of scatter) +instancing.wind { strength?, speed?, turbulence? } // GPU sway for ALL instances + +material (shader) { type:"shader", fragmentShader, vertexShader?, uniforms?, animated?, transparent?, doubleSide? } +animation { poses?:[ {timeSec, rootPosition?, bones:{[name]:{rotationDeg}}} ], tracks?:[...], loop?, timeScale?, autoplay?, durationSec?, clip? } + poses[] ← USE THIS for characters (dance/walk): a timeline of whole-body poses, same `bones` shape as `pose` +``` + +## Apply semantics (how your JSON becomes the scene) + +- **Edit** an existing object: keep its `id` **and its `source` field verbatim** + (lossless passthrough — never modify it). Change `transform`/`color`/`visible`/ + `pose`/`material`/`animation`. +- **Create**: use `primitive`, `mesh`, or `instances`; omit `id` and `source`. +- **Delete**: omit the entity from `entities` (reconciled by id; missing ⇒ removed). +- New `character`/`model` are **skipped** — you can't summon a rigged/asset object + from text. Edit existing ones only. +- Applying is **one undoable step**; the whole change reverts together. + +--- + +## primitive + +```jsonc +{ "kind": "primitive", "shape": "Box", "color": "#6b6b6b", "transform": {...} } +``` +`shape` ∈ `Box`, `Sphere`, `Cone`, `Cylinder`, `Donut`, `Water`, `PointLight`. +Each is ≈1m (radius 0.5); use `scale` to resize. + +## mesh (raw vertex data) + +```jsonc +{ "kind": "mesh", "color": "#aa8844", "transform": {...}, + "geometry": { "positions": [x,y,z, x,y,z, ...] } } +``` +`positions` is a **non-indexed triangle soup** in local space: every **9 numbers** +(3 vertices) = one triangle. Normals auto-computed. Keep counts modest. + +## instances (forests, grass — one draw call) + +One `instances` entity becomes **one draw call** for the whole field, however many +copies. Always use this for many copies — never emit hundreds of entities. + +```jsonc +{ "kind": "instances", "color": "#2f7d32", "transform": { /* field origin */ }, + "instancing": { + "base": { "shape": "Cone" }, // OR { "positions":[...blade verts...] } OR { "plane":{ "width":1,"height":1 } } + "scatter": { // procedural placement (preferred) — OR explicit "instances":[...] + "count": 350, + "area": { "x": 60, "z": 60 }, // spread centered on the field origin + "seed": 11, // deterministic — same seed = same layout + "yJitterDeg": 360, // random yaw + "scaleRange": [1.2, 3.4], // random uniform scale + "tiltDeg": 8, // random lean — organic + "colorJitter": 0.18 // 0..1 per-instance shade off `color` + } + } } +``` +For **no two alike**, set `tiltDeg`, `colorJitter`, a wide `scaleRange`, and +`yJitterDeg:360`. Grass = a few-vertex blade in `base.positions` with a high +`count`; trees = `base.shape:"Cone"` with a smaller `count`. + +**Wind — animate the WHOLE field on the GPU.** Add `instancing.wind` to make every +instance sway: a vertex shader on a shared `time` uniform with a per-instance phase +— one draw call, no per-frame CPU, scales to tens of thousands. Height-weighted, +so each instance pivots at its base. +```jsonc +"wind": { "strength": 0.25, "speed": 1.5, "turbulence": 0.4 } +``` +> ⚠️ To animate instances, use `instancing.wind` — **do NOT** put a per-object +> `animation` on an instanced field. A per-object `animation` moves the entire +> field as one rigid block, not the individual blades. + +## material (custom shader — any kind) + +Give an entity's mesh(es) a custom GLSL material. Works on any kind, including +`instances` (the engine supplies instancing-aware vertex placement automatically). + +```jsonc +"material": { + "type": "shader", + "animated": true, // ticks the injected `time` uniform + "doubleSide": true, + "uniforms": { "tint": [0.2, 0.8, 0.4] }, // number→float, [a,b]→vec2, [a,b,c]→vec3, [a,b,c,d]→vec4 + "fragmentShader": "void main(){ float g = 0.5 + 0.5*sin(time*2.0 + vUv.x*12.0); gl_FragColor = vec4(vUv.x, g, 1.0 - vUv.y, 1.0); }" +} +``` +**Shader contract — already declared for you, just use them:** +- Uniforms: `float time`, `vec2 resolution`, plus anything in your `uniforms`. +- Varyings: `vec2 vUv`, `vec3 vNormal` (view space), `vec3 vPosition` (local). +- Write only `fragmentShader` (a full `void main(){ ... gl_FragColor = ...; }`). + A `vertexShader` is optional and rarely needed. + +## animation (keyframes — per object or bones) + +Make an entity move over time (Three.js AnimationMixer). Times are **seconds**; +each track's `values` are one per keyframe, matching `times`. + +```jsonc +"animation": { + "loop": "repeat", // "repeat" | "once" | "pingpong" (default repeat) + "timeScale": 1, // playback speed (default 1) + "autoplay": true, // (default true) + "durationSec": 2, // optional; inferred from times if omitted + "tracks": [ + { "property": "position", "times": [0,1,2], "values": [ {"x":0,"y":1,"z":0}, {"x":0,"y":3,"z":0}, {"x":0,"y":1,"z":0} ] }, + { "property": "rotationDeg", "times": [0,2], "values": [ {"x":0,"y":0,"z":0}, {"x":0,"y":360,"z":0} ] } + // property ∈ "position" | "rotationDeg" (degrees) | "scale" + // optional "interpolation": "linear" (default) | "smooth" | "discrete" + ] +} +``` +- A track overrides that channel of `transform` while playing; omitted channels stay static. +- Play a **baked clip** the asset ships with: `"animation": { "clip": "Walk", "loop": "repeat" }` (falls back to `tracks`/`poses` if absent). +- For many copies, this is per-OBJECT — for swaying grass use `instancing.wind` instead. + +### Character motion — pose-keyframes (USE THIS for characters) + +Don't hand-author per-bone tracks for a character. Author a **timeline of +whole-body poses** — the same sparse `bones: { name: { rotationDeg } }` shape as +the static `pose`, plus a `timeSec`. The engine interpolates between them. Block a +few key poses; a bone you omit at a keyframe just interpolates between the +keyframes that *do* set it, so each limb can move on its own cadence. + +```jsonc +"animation": { + "loop": "repeat", + "poses": [ + { "timeSec": 0.0, "bones": { "mixamorigLeftArm": {"rotationDeg":{"x":0,"y":0,"z":-70}}, "mixamorigRightArm": {"rotationDeg":{"x":0,"y":0,"z":70}} } }, + { "timeSec": 0.5, "bones": { "mixamorigLeftArm": {"rotationDeg":{"x":0,"y":0,"z":-30}}, "mixamorigRightArm": {"rotationDeg":{"x":0,"y":0,"z":30}} } }, + { "timeSec": 1.0, "bones": { "mixamorigLeftArm": {"rotationDeg":{"x":0,"y":0,"z":-70}}, "mixamorigRightArm": {"rotationDeg":{"x":0,"y":0,"z":70}} } } + ] +} +``` + +- **Dance / wave / idle:** a short loop of 2–4 key poses with `loop:"repeat"`. +- **Walk / run in place:** alternate two contact poses (legs + counter-swinging arms), `loop:"repeat"`. +- **Walk along a path (road / through the forest):** add `rootPosition` to keyframes + to move the hips, AND repeat the leg/arm contact poses along the journey, in ONE + clip. Use `loop:"once"` for a one-way trip (e.g. waypoints weaving between trees), + or `"repeat"` to loop the route. Example skeleton: + ```jsonc + "animation": { "loop": "once", "durationSec": 8, "poses": [ + { "timeSec": 0, "rootPosition": {"x":-20,"y":0,"z":0}, "bones": { /* contact pose A */ } }, + { "timeSec": 0.5, "bones": { /* contact pose B */ } }, + { "timeSec": 1.0, "rootPosition": {"x":-17,"y":0,"z":3}, "bones": { /* pose A */ } }, + /* …repeat the A/B leg cycle every ~0.5s while stepping rootPosition along the path… */ + { "timeSec": 8, "rootPosition": {"x":20,"y":0,"z":-5}, "bones": { /* pose A */ } } + ] } + ``` + (Add `mixamorigHips` rotation in the poses to make the character face its travel direction.) + +## character pose (mixamo) — edit only + +```jsonc +{ "kind": "character", "id": "keep-this", "source": {/* keep verbatim */}, "transform": {...}, + "pose": { + "rootBone": "mixamorigHips", + "rootPosition": { "x": 0, "y": 1.0, "z": 0 }, // optional hip translation + "bones": { + "mixamorigLeftArm": { "rotationDeg": { "x": 0, "y": 0, "z": -60 } }, + "mixamorigRightArm": { "rotationDeg": { "x": 0, "y": 0, "z": 60 } }, + "mixamorigSpine": { "rotationDeg": { "x": 8, "y": 0, "z": 0 } } + } + } } +``` +`bones` is sparse — list only what you change (local rotation, degrees). Omitted +bones keep their current pose. The **same `bones` shape is used for static `pose` +and for `animation.poses`** — pose it once, or key it over time. + +**The character must already be in the scene.** You'll see it as `kind:"character"` +with its current rotations in `pose.bones` — that exported pose (≈ a T-pose at rest) +is your **starting point**; adjust from there. + +**Mixamo posable bones (and what moves them from the T-pose rest):** +- **Spine/head:** `mixamorigSpine`, `mixamorigSpine1`, `mixamorigSpine2` (bend/twist torso), `mixamorigNeck`, `mixamorigHead`. +- **Arms** (rest = arms out along ±X): `mixamorigLeftShoulder`, `mixamorigLeftArm` (**z** raises/lowers the arm), `mixamorigLeftForeArm` (bends the elbow), `mixamorigLeftHand`; mirror with `Right`. +- **Legs:** `mixamorigLeftUpLeg` (**x** swings the thigh fwd/back), `mixamorigLeftLeg` (**x** bends the knee), `mixamorigLeftFoot`; mirror with `Right`. +- **Root:** `mixamorigHips` (whole-body lean/turn) + `rootPosition` for translation. + +Axis signs differ per rig, and **left/right are mirrored** (flip the sign on the +swinging axis). Start from the exported current rotation, nudge, and verify. + +--- + +## Full example (forest + windy grass + animated shader) + +```json +{ + "descriptorVersion": 2, "units": "meters", "coordinateSystem": "y-up-right-handed", "environment": {}, + "entities": [ + { "id": "ground", "name": "Ground", "kind": "primitive", "shape": "Box", "color": "#5b6b4a", + "transform": { "position": {"x":0,"y":-0.5,"z":0}, "rotationDeg": {"x":0,"y":0,"z":0}, "scale": {"x":80,"y":0.2,"z":80} } }, + + { "id": "forest", "name": "Forest", "kind": "instances", "color": "#2f7d32", + "transform": { "position": {"x":0,"y":1,"z":0}, "rotationDeg": {"x":0,"y":0,"z":0}, "scale": {"x":1,"y":1,"z":1} }, + "instancing": { "base": { "shape": "Cone" }, + "scatter": { "count": 350, "area": {"x":60,"z":60}, "seed": 11, "yJitterDeg": 360, "scaleRange": [1.2,3.4], "tiltDeg": 8, "colorJitter": 0.18 } } }, + + { "id": "grass", "name": "Grass", "kind": "instances", "color": "#5a8f3a", + "transform": { "position": {"x":0,"y":0,"z":0}, "rotationDeg": {"x":0,"y":0,"z":0}, "scale": {"x":1,"y":1,"z":1} }, + "instancing": { "base": { "positions": [-0.05,0,0, 0.05,0,0, 0.05,0.5,0, -0.05,0,0, 0.05,0.5,0, -0.05,0.5,0] }, + "scatter": { "count": 4000, "area": {"x":50,"z":50}, "seed": 3, "scaleRange": [0.6,1.6] }, + "wind": { "strength": 0.25, "speed": 1.6, "turbulence": 0.4 } } }, + + { "id": "orb", "name": "Shader Orb", "kind": "primitive", "shape": "Sphere", "color": "#ffffff", + "transform": { "position": {"x":0,"y":4,"z":0}, "rotationDeg": {"x":0,"y":0,"z":0}, "scale": {"x":2,"y":2,"z":2} }, + "material": { "type": "shader", "animated": true, "doubleSide": true, + "fragmentShader": "void main(){ float p=0.5+0.5*sin(time*3.0); float f=pow(1.0-abs(vNormal.z),2.0); gl_FragColor=vec4(f+p*0.3, f, 1.0, 1.0); }" }, + "animation": { "loop": "repeat", "tracks": [ { "property": "position", "times": [0,2,4], "values": [ {"x":0,"y":4,"z":0}, {"x":0,"y":5,"z":0}, {"x":0,"y":4,"z":0} ] } ] } } + ] +} +``` + +## Checklist before returning + +- Output is **valid JSON only** — the whole descriptor, no prose, no trailing commas. +- Kept `id` and `source` for every existing object you intend to preserve. +- Rotations in **degrees**; positions in **meters**, Y-up; colors flat hex. +- Used `instances` + `scatter` for many copies; `instancing.wind` (not `animation`) to animate a field. +- New objects are only `primitive` / `mesh` / `instances` (no new `character`/`model`). diff --git a/frontend/libs/components/pagescene/src/lib/scene_descriptor/apply_descriptor.ts b/frontend/libs/components/pagescene/src/lib/scene_descriptor/apply_descriptor.ts index 358f049a72..25dd5b59c1 100644 --- a/frontend/libs/components/pagescene/src/lib/scene_descriptor/apply_descriptor.ts +++ b/frontend/libs/components/pagescene/src/lib/scene_descriptor/apply_descriptor.ts @@ -176,6 +176,11 @@ function updateObjectInPlace( if (entity.material?.fragmentShader) { scene.applyShaderMaterial(obj.uuid, entity.material); } + + // Keyframe animation — drives transform/bones each frame. + if (entity.animation) { + scene.attachAnimation(obj.uuid, entity.animation); + } } // Instantiate a primitive locally (no network) and stamp the userData the @@ -242,7 +247,10 @@ async function createInstances( scene: Scene, entity: DescriptorEntity, ): Promise { - const resolved = resolveInstancing(entity.instancing); + const resolved = resolveInstancing( + entity.instancing, + entity.color || GRAY_BOX_COLOR, + ); if (!resolved?.base) return undefined; const created = await scene.instantiateInstancedMesh( diff --git a/frontend/libs/components/pagescene/src/lib/scene_descriptor/export_descriptor.ts b/frontend/libs/components/pagescene/src/lib/scene_descriptor/export_descriptor.ts index 9785fa0c79..a976ad38b0 100644 --- a/frontend/libs/components/pagescene/src/lib/scene_descriptor/export_descriptor.ts +++ b/frontend/libs/components/pagescene/src/lib/scene_descriptor/export_descriptor.ts @@ -106,6 +106,9 @@ function objectJsonToEntity( if (obj.shaderMaterial) { entity.material = obj.shaderMaterial; } + if (obj.animation) { + entity.animation = obj.animation; + } return entity; } diff --git a/frontend/libs/components/pagescene/src/lib/scene_descriptor/index.ts b/frontend/libs/components/pagescene/src/lib/scene_descriptor/index.ts index fdd2d42896..a7f3a43ec8 100644 --- a/frontend/libs/components/pagescene/src/lib/scene_descriptor/index.ts +++ b/frontend/libs/components/pagescene/src/lib/scene_descriptor/index.ts @@ -17,8 +17,12 @@ export type { DescriptorInstancing, InstancingBase, InstancingScatter, + InstancingWind, InstanceTransform, ShaderMaterialSpec, + AnimationSpec, + AnimationTrack, + PoseKeyframe, BonePose, Vec3, } from "./scene_descriptor"; diff --git a/frontend/libs/components/pagescene/src/lib/scene_descriptor/instancing_codec.ts b/frontend/libs/components/pagescene/src/lib/scene_descriptor/instancing_codec.ts index 04db219d66..784080d99d 100644 --- a/frontend/libs/components/pagescene/src/lib/scene_descriptor/instancing_codec.ts +++ b/frontend/libs/components/pagescene/src/lib/scene_descriptor/instancing_codec.ts @@ -3,6 +3,7 @@ // expand it deterministically here so apply just consumes a flat instance // list. Same seed → same layout (reproducible across export/apply/undo). +import * as THREE from "three"; import { DescriptorInstancing, InstanceTransform, @@ -10,38 +11,69 @@ import { } from "./scene_descriptor"; // Return a spec with concrete `instances`. Explicit instances win; a -// scatter spec is expanded; otherwise the spec is returned as-is. +// scatter spec is expanded; otherwise the spec is returned as-is. The base +// color (the entity's color) seeds per-instance color jitter. export function resolveInstancing( spec: DescriptorInstancing | undefined, + baseColor?: string, ): DescriptorInstancing | undefined { if (!spec) return spec; if (spec.instances && spec.instances.length > 0) return spec; if (spec.scatter) { - return { base: spec.base, instances: scatterInstances(spec.scatter) }; + return { + base: spec.base, + instances: scatterInstances(spec.scatter, baseColor), + wind: spec.wind, + }; } return spec; } -function scatterInstances(s: InstancingScatter): InstanceTransform[] { +function scatterInstances( + s: InstancingScatter, + baseColor?: string, +): InstanceTransform[] { const rng = mulberry32(s.seed ?? 1); const yaw = s.yJitterDeg ?? 360; + const tilt = s.tiltDeg ?? 0; + const jitter = s.colorJitter ?? 0; const [sMin, sMax] = s.scaleRange ?? [1, 1]; const count = Math.max(0, Math.floor(s.count)); + + // Per-instance color is derived from the base color in HSL so the field + // reads as one species with natural shade-to-shade variation. + const base = jitter > 0 && baseColor ? new THREE.Color(baseColor) : null; + const hsl = { h: 0, s: 0, l: 0 }; + if (base) base.getHSL(hsl); + const out: InstanceTransform[] = []; for (let i = 0; i < count; i++) { const x = (rng() - 0.5) * s.area.x; const z = (rng() - 0.5) * s.area.z; const ry = (rng() - 0.5) * yaw; + const tx = (rng() - 0.5) * tilt; + const tz = (rng() - 0.5) * tilt; const sc = sMin + rng() * (sMax - sMin); - out.push({ + + const inst: InstanceTransform = { position: { x: round(x), y: 0, z: round(z) }, - rotationDeg: { x: 0, y: round(ry), z: 0 }, + rotationDeg: { x: round(tx), y: round(ry), z: round(tz) }, scale: { x: round(sc), y: round(sc), z: round(sc) }, - }); + }; + if (base) { + const h = (hsl.h + (rng() - 0.5) * jitter * 0.1 + 1) % 1; + const l = clamp01(hsl.l + (rng() - 0.5) * jitter); + inst.color = "#" + new THREE.Color().setHSL(h, hsl.s, l).getHexString(); + } + out.push(inst); } return out; } +function clamp01(n: number): number { + return n < 0 ? 0 : n > 1 ? 1 : n; +} + // Small, fast, seedable PRNG (mulberry32) — deterministic per seed. function mulberry32(seed: number): () => number { let a = seed >>> 0; diff --git a/frontend/libs/components/pagescene/src/lib/scene_descriptor/scene_descriptor.ts b/frontend/libs/components/pagescene/src/lib/scene_descriptor/scene_descriptor.ts index 4475417c9a..aa6c8dd6b5 100644 --- a/frontend/libs/components/pagescene/src/lib/scene_descriptor/scene_descriptor.ts +++ b/frontend/libs/components/pagescene/src/lib/scene_descriptor/scene_descriptor.ts @@ -81,6 +81,10 @@ export interface DescriptorEntity { // The LLM-authored "custom mat" path. See ShaderMaterialSpec. material?: ShaderMaterialSpec; + // Keyframe animation that plays on this entity (transform and, for + // characters, bones). See AnimationSpec. + animation?: AnimationSpec; + // Flat hex color, e.g. "#9a9a9a". Absent ⇒ gray-box default. color?: string; visible?: boolean; @@ -135,12 +139,25 @@ export interface InstanceTransform { // Procedural placement — far cheaper for an LLM to emit than N explicit // instances. Deterministic given `seed`. Expanded to instances on apply. +// The jitter knobs give an organic look (no two instances alike). export interface InstancingScatter { count: number; area: { x: number; z: number }; // spread, centered on the field origin seed?: number; yJitterDeg?: number; // random yaw range, default 360 scaleRange?: [number, number]; // uniform scale range, default [1,1] + tiltDeg?: number; // random lean on X/Z, default 0 (e.g. 8 for trees) + colorJitter?: number; // 0..1 per-instance HSL variance off the base color +} + +// GPU wind animation for an instanced field — a vertex shader driven by a +// shared `time` uniform + per-instance phase. All instances sway together +// (phase-shifted), one draw call, no per-frame CPU. Height-weighted, so +// each instance pivots at its base (grass/foliage). +export interface InstancingWind { + strength?: number; // sway distance at the tip, default 0.15 + speed?: number; // oscillation speed, default 1.5 + turbulence?: number; // adds a second frequency for choppier motion, default 0 } export interface DescriptorInstancing { @@ -148,6 +165,8 @@ export interface DescriptorInstancing { // Provide explicit instances OR a scatter spec (scatter wins if both). instances?: InstanceTransform[]; scatter?: InstancingScatter; + // Optional GPU wind animation applied to every instance. + wind?: InstancingWind; } // ── Custom shader material (LLM-authored) ─────────────────────────────── @@ -166,6 +185,49 @@ export interface ShaderMaterialSpec { animated?: boolean; // tick the injected `time` uniform each frame } +// ── Keyframe animation ────────────────────────────────────────────────── + +// One animated channel. Times are seconds; values are one per keyframe. +// - property "position" / "scale": values are Vec3 (world/local units) +// - property "rotationDeg": values are Vec3 Euler degrees +// target "object" (default) animates the entity itself; target "bone" +// animates a named mixamo bone (characters only). +export interface AnimationTrack { + target?: "object" | "bone"; + bone?: string; + property: "position" | "rotationDeg" | "scale"; + times: number[]; + values: Vec3[]; + interpolation?: "linear" | "smooth" | "discrete"; +} + +// A whole-body keyframe: the character's pose at one moment. Same sparse +// shape as a static `pose` (only the bones you set), plus a time. This is +// the easy way to author character motion — block a few key poses and the +// engine interpolates between them. Bones omitted at a given keyframe just +// don't get a key there (they interpolate between the keys that do set +// them), so each limb can move on its own cadence. +export interface PoseKeyframe { + timeSec: number; + rootPosition?: Vec3; // hip translation at this keyframe (for locomotion) + bones: Record; +} + +// A keyframe animation on one entity, driven by a THREE.AnimationMixer. +// Author it ONE of these ways: +// - poses: a timeline of whole-body poses (best for characters: dance, walk) +// - tracks: per-channel curves (object transform or individual bones) +// - clip: play a baked GLTF clip by name (e.g. a mixamo "Walk") +export interface AnimationSpec { + clip?: string; + durationSec?: number; // inferred from key times if omitted + loop?: "repeat" | "once" | "pingpong"; // default repeat + timeScale?: number; // default 1 + autoplay?: boolean; // default true + poses?: PoseKeyframe[]; + tracks?: AnimationTrack[]; +} + // One bone's editable local transform. Rotation only for now — that's // what FK posing manipulates; bone translation/scale are rarely posed. export interface BonePose {