Pulse Effects — architecture & native spec

frame pipeline

CAMERA (native capture, NV12/BGRA)
  ↓ zero-copy → GPU texture (CVPixelBuffer / HardwareBuffer)
TRACKING TAP (downscaled 256px copy, async, own thread)
  ├── MediaPipe Face Landmarker → landmarks, head pose, blendshapes
  └── MediaPipe Image Segmenter → person matte (R8 texture)
  ↓ (results smoothed with one-euro filter, never blocks render)
EFFECT STACK (single GPU command buffer, layer ordered)
  1 background      : blur / image / video / gradient / colour / animated
  2 background_fx   : particles behind subject (matte-masked)
  3 person          : chroma key, beauty, colour grade / LUT
  4 face            : landmark-anchored meshes + sprites
  5 foreground      : particles, flares, frames, overlays
  ↓
PROCESSED TEXTURE
  ├── local preview (Flutter Texture widget)
  ├── recorder (stories, posts, profile video)
  └── WebRTC external video source → existing live track → viewers

package:pulse_effects/pulse_effects.dart

final engine = await PulseEffects.instance.initialize(
  const EngineConfig(
    quality: QualityTier.auto,      // low | medium | high | ultra | auto
    targetFps: 30,
    trackFaces: 2,
    segmentation: SegmentationMode.auto,
  ),
);

// Catalogue + install (remote pack hosted by us)
final catalogue = await engine.catalogue.fetch(page: 0);
final effect    = await engine.catalogue.install('cyber_mask_01');

// Composable stack — nothing is exclusive except slots that must be
final stack = engine.stack;
await stack.add('bg_nightclub');
await stack.add('bty_studio');
await stack.add('msk_sunglasses');
await stack.add('fx_hearts');
await stack.add('flt_cinematic');
stack.setParam('bty_studio', 'skinSmoothing', 0.45);

// Output taps
engine.previewTextureId;                 // Flutter Texture widget
final track = await engine.videoTrack;   // MediaStreamTrack for flutter_webrtc
await engine.startRecording(path);

engine.stats.listen((s) => debugPrint('${s.fps}fps ${s.gpuMs}ms ${s.thermal}'));

channel contract

MethodChannel  'pulse/effects'          — control plane
  initialize(Map config) -> {textureId:int, caps:Map}
  installEffect(String manifestJson, String bundlePath) -> bool
  addEffect(String id) / removeEffect(String id) / clearStack()
  setParam(String id, String key, double value)
  setQuality(String tier)
  setMirror(bool)
  startRecording(String path) / stopRecording() -> String
  attachToWebRtc(String trackId) -> bool
  dispose()

EventChannel   'pulse/effects/stats'    — fps, gpuMs, droppedFrames, thermalState
EventChannel   'pulse/effects/tracking' — faceCount, landmarks (opt-in, throttled)

All heavy payloads (assets) move over the filesystem, never the channel.

ios/Classes/PulseEffectsEngine.swift

protocol PulseEffectPass {
  func encode(_ enc: MTLComputeCommandEncoder,
              input: MTLTexture, output: MTLTexture,
              frame: PulseFrameContext)
}

final class PulseEffectsEngine: NSObject, AVCaptureVideoDataOutputSampleBufferDelegate {
  private let device = MTLCreateSystemDefaultDevice()!
  private lazy var queue = device.makeCommandQueue()!
  private var textureCache: CVMetalTextureCache!
  private var passes: [PulseEffectPass] = []          // built from manifests
  private let tracker = FaceLandmarkerRunner()        // MediaPipe Tasks, .liveStream
  private let segmenter = ImageSegmenterRunner()

  func captureOutput(_ o: AVCaptureOutput, didOutput sb: CMSampleBuffer, from c: AVCaptureConnection) {
    guard let pb = CMSampleBufferGetImageBuffer(sb) else { return }
    tracker.detectAsync(pb, timestamp: ts)            // non-blocking
    segmenter.segmentAsync(pb, timestamp: ts)

    let input = makeTexture(from: pb)                 // zero-copy via CVMetalTextureCache
    let cb = queue.makeCommandBuffer()!
    var src = input
    for pass in passes { pass.encode(enc, input: src, output: pong, frame: ctx); swap(&src, &pong) }
    cb.addCompletedHandler { _ in
      self.flutterTexture.publish(src)                // FlutterTexture
      self.webrtcSource.push(pixelBuffer: self.outputPB, timestamp: ts)  // RTCVideoSource
    }
    cb.commit()
  }
}

android/src/main/kotlin/PulseEffectsEngine.kt

class PulseEffectsEngine(context: Context, private val textureRegistry: TextureRegistry) {
  private val glThread = HandlerThread("pulse-gl").apply { start() }
  private val eglCore = EglCore()                       // shared context with WebRTC
  private val passes = mutableListOf<EffectPass>()      // built from manifests
  private val faceLandmarker = FaceLandmarker.createFromOptions(context, opts)
  private val segmenter = ImageSegmenter.createFromOptions(context, segOpts)

  fun onFrame(image: ImageProxy) {                      // CameraX, OUTPUT_IMAGE_FORMAT_YUV_420_888
    val oesTex = surfaceTexture.attach()                // camera → OES texture, no CPU copy
    faceLandmarker.detectAsync(mpImage, ts)
    segmenter.segmentAsync(mpImage, ts)

    var src = oesTex
    passes.forEach { src = it.render(src, ctx) }        // FBO ping-pong

    flutterSurface.publish(src)
    videoSource.capturerObserver.onFrameCaptured(       // org.webrtc.VideoFrame
      VideoFrame(TextureBufferImpl(w, h, RGB, src, matrix, glHandler, yuvConverter, null), 0, tsNs)
    )
  }
}

host-side wiring

iOS:      RTCVideoSource + RTCVideoCapturer → capturer(_:didCapture: RTCVideoFrame(buffer:
          RTCCVPixelBuffer(pixelBuffer: processedPB)))
Android:  PeerConnectionFactory.createVideoSource(false) → capturerObserver.onFrameCaptured(
          VideoFrame(TextureBufferImpl(processedTexId, ...)))
Flutter:  final track = await PulseEffects.instance.videoTrack;
          await peerConnection.addTrack(track, localStream);

Because the processed texture IS the track source, viewers see the effects.
Nothing here replaces the existing signalling, SFU or room logic — the only
change on the streaming side is swapping the camera source for the engine source.

effects API

GET  /api/effects?cursor=&category=&q=      → { items: EffectManifest[], cursor }
GET  /api/effects/:id                       → EffectManifest
GET  /api/effects/:id/bundle                → .pfx zip (assets + manifest.json + sha256)
POST /api/effects            (admin)        → create draft
PUT  /api/effects/:id        (admin)        → new version
POST /api/effects/:id/publish|disable
DELETE /api/effects/:id

Device cache: install → verify sha256 → unzip to /effects/<id>/<version>/
             → validate manifest against schema v1 → register
             update = download new version, atomic swap, delete old
Creator effects: assets + template-constrained manifests only. No scripts,
no shaders from users — every render path is one of our vetted passes.