Sky-Kim commited on
Commit
29982f2
·
0 Parent(s):

Initial commit

Browse files
.gitattributes ADDED
@@ -0,0 +1,36 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ *.7z filter=lfs diff=lfs merge=lfs -text
2
+ *.arrow filter=lfs diff=lfs merge=lfs -text
3
+ *.bin filter=lfs diff=lfs merge=lfs -text
4
+ *.bz2 filter=lfs diff=lfs merge=lfs -text
5
+ *.ckpt filter=lfs diff=lfs merge=lfs -text
6
+ *.ftz filter=lfs diff=lfs merge=lfs -text
7
+ *.gz filter=lfs diff=lfs merge=lfs -text
8
+ *.h5 filter=lfs diff=lfs merge=lfs -text
9
+ *.joblib filter=lfs diff=lfs merge=lfs -text
10
+ *.lfs.* filter=lfs diff=lfs merge=lfs -text
11
+ *.mlmodel filter=lfs diff=lfs merge=lfs -text
12
+ *.model filter=lfs diff=lfs merge=lfs -text
13
+ *.msgpack filter=lfs diff=lfs merge=lfs -text
14
+ *.npy filter=lfs diff=lfs merge=lfs -text
15
+ *.npz filter=lfs diff=lfs merge=lfs -text
16
+ *.onnx filter=lfs diff=lfs merge=lfs -text
17
+ *.ot filter=lfs diff=lfs merge=lfs -text
18
+ *.parquet filter=lfs diff=lfs merge=lfs -text
19
+ *.pb filter=lfs diff=lfs merge=lfs -text
20
+ *.pickle filter=lfs diff=lfs merge=lfs -text
21
+ *.pkl filter=lfs diff=lfs merge=lfs -text
22
+ *.pt filter=lfs diff=lfs merge=lfs -text
23
+ *.pth filter=lfs diff=lfs merge=lfs -text
24
+ *.rar filter=lfs diff=lfs merge=lfs -text
25
+ *.safetensors filter=lfs diff=lfs merge=lfs -text
26
+ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
27
+ *.tar.* filter=lfs diff=lfs merge=lfs -text
28
+ *.tar filter=lfs diff=lfs merge=lfs -text
29
+ *.tflite filter=lfs diff=lfs merge=lfs -text
30
+ *.tgz filter=lfs diff=lfs merge=lfs -text
31
+ *.wasm filter=lfs diff=lfs merge=lfs -text
32
+ *.xz filter=lfs diff=lfs merge=lfs -text
33
+ *.zip filter=lfs diff=lfs merge=lfs -text
34
+ *.zst filter=lfs diff=lfs merge=lfs -text
35
+ *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ *.sentis filter=lfs diff=lfs merge=lfs -text
CHANGELOG.md ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ # Changelog
2
+
3
+ ## [0.1.0] - 2026-10-22
4
+ ### Added
5
+ - Initial package: `CedTiny` Unity Inference Engine (Sentis) audio tagger wrapper (in-graph log-mel frontend, 527 AudioSet classes + embedding, resampling, sliding-window streaming, sync + async readback); FP32 model weights and AudioSet labels under `Models~`.
CHANGELOG.md.meta ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ fileFormatVersion: 2
2
+ guid: ab3bd0bc4f724aac98d27c51e484d44d
3
+ TextScriptImporter:
4
+ externalObjects: {}
5
+ userData:
6
+ assetBundleName:
7
+ assetBundleVariant:
LICENSE.md ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # sentis-ced-tiny License
2
+
3
+ This package's wrapper code (`Runtime/`) is licensed under the Apache License 2.0 (below).
4
+ The bundled model weights (`Models~/ced_tiny_fp32.sentis`) are derived from CED by Heinrich Dinkel et al. (https://github.com/RicherMans/CED, https://huggingface.co/mispeech/ced-tiny)
5
+ and remain under the Apache License 2.0. Modifications to the weights file: log-mel frontend prepended as a fixed-basis
6
+ convolution, outputs exposed as sigmoid scores + clip embedding; converted to the Unity Sentis format.
7
+ `Models~/audioset_labels.txt` lists the AudioSet ontology class names (Google, CC BY 4.0).
8
+
9
+ Copyright (c) 2026 Sky Kim
10
+
11
+ Licensed under the Apache License, Version 2.0 (the "License");
12
+ you may not use this file except in compliance with the License.
13
+ You may obtain a copy of the License at
14
+
15
+ http://www.apache.org/licenses/LICENSE-2.0
16
+
17
+ Unless required by applicable law or agreed to in writing, software
18
+ distributed under the License is distributed on an "AS IS" BASIS,
19
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
20
+ See the License for the specific language governing permissions and
21
+ limitations under the License.
LICENSE.md.meta ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ fileFormatVersion: 2
2
+ guid: 780a3fc47c444291b8b8aad36e9cc74f
3
+ TextScriptImporter:
4
+ externalObjects: {}
5
+ userData:
6
+ assetBundleName:
7
+ assetBundleVariant:
Models~/audioset_labels.txt ADDED
@@ -0,0 +1,527 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ Speech
2
+ Male speech, man speaking
3
+ Female speech, woman speaking
4
+ Child speech, kid speaking
5
+ Conversation
6
+ Narration, monologue
7
+ Babbling
8
+ Speech synthesizer
9
+ Shout
10
+ Bellow
11
+ Whoop
12
+ Yell
13
+ Battle cry
14
+ Children shouting
15
+ Screaming
16
+ Whispering
17
+ Laughter
18
+ Baby laughter
19
+ Giggle
20
+ Snicker
21
+ Belly laugh
22
+ Chuckle, chortle
23
+ Crying, sobbing
24
+ Baby cry, infant cry
25
+ Whimper
26
+ Wail, moan
27
+ Sigh
28
+ Singing
29
+ Choir
30
+ Yodeling
31
+ Chant
32
+ Mantra
33
+ Male singing
34
+ Female singing
35
+ Child singing
36
+ Synthetic singing
37
+ Rapping
38
+ Humming
39
+ Groan
40
+ Grunt
41
+ Whistling
42
+ Breathing
43
+ Wheeze
44
+ Snoring
45
+ Gasp
46
+ Pant
47
+ Snort
48
+ Cough
49
+ Throat clearing
50
+ Sneeze
51
+ Sniff
52
+ Run
53
+ Shuffle
54
+ Walk, footsteps
55
+ Chewing, mastication
56
+ Biting
57
+ Gargling
58
+ Stomach rumble
59
+ Burping, eructation
60
+ Hiccup
61
+ Fart
62
+ Hands
63
+ Finger snapping
64
+ Clapping
65
+ Heart sounds, heartbeat
66
+ Heart murmur
67
+ Cheering
68
+ Applause
69
+ Chatter
70
+ Crowd
71
+ Hubbub, speech noise, speech babble
72
+ Children playing
73
+ Animal
74
+ Domestic animals, pets
75
+ Dog
76
+ Bark
77
+ Yip
78
+ Howl
79
+ Bow-wow
80
+ Growling
81
+ Whimper (dog)
82
+ Cat
83
+ Purr
84
+ Meow
85
+ Hiss
86
+ Caterwaul
87
+ Livestock, farm animals, working animals
88
+ Horse
89
+ Clip-clop
90
+ Neigh, whinny
91
+ Cattle, bovinae
92
+ Moo
93
+ Cowbell
94
+ Pig
95
+ Oink
96
+ Goat
97
+ Bleat
98
+ Sheep
99
+ Fowl
100
+ Chicken, rooster
101
+ Cluck
102
+ Crowing, cock-a-doodle-doo
103
+ Turkey
104
+ Gobble
105
+ Duck
106
+ Quack
107
+ Goose
108
+ Honk
109
+ Wild animals
110
+ Roaring cats (lions, tigers)
111
+ Roar
112
+ Bird
113
+ Bird vocalization, bird call, bird song
114
+ Chirp, tweet
115
+ Squawk
116
+ Pigeon, dove
117
+ Coo
118
+ Crow
119
+ Caw
120
+ Owl
121
+ Hoot
122
+ Bird flight, flapping wings
123
+ Canidae, dogs, wolves
124
+ Rodents, rats, mice
125
+ Mouse
126
+ Patter
127
+ Insect
128
+ Cricket
129
+ Mosquito
130
+ Fly, housefly
131
+ Buzz
132
+ Bee, wasp, etc.
133
+ Frog
134
+ Croak
135
+ Snake
136
+ Rattle
137
+ Whale vocalization
138
+ Music
139
+ Musical instrument
140
+ Plucked string instrument
141
+ Guitar
142
+ Electric guitar
143
+ Bass guitar
144
+ Acoustic guitar
145
+ Steel guitar, slide guitar
146
+ Tapping (guitar technique)
147
+ Strum
148
+ Banjo
149
+ Sitar
150
+ Mandolin
151
+ Zither
152
+ Ukulele
153
+ Keyboard (musical)
154
+ Piano
155
+ Electric piano
156
+ Organ
157
+ Electronic organ
158
+ Hammond organ
159
+ Synthesizer
160
+ Sampler
161
+ Harpsichord
162
+ Percussion
163
+ Drum kit
164
+ Drum machine
165
+ Drum
166
+ Snare drum
167
+ Rimshot
168
+ Drum roll
169
+ Bass drum
170
+ Timpani
171
+ Tabla
172
+ Cymbal
173
+ Hi-hat
174
+ Wood block
175
+ Tambourine
176
+ Rattle (instrument)
177
+ Maraca
178
+ Gong
179
+ Tubular bells
180
+ Mallet percussion
181
+ Marimba, xylophone
182
+ Glockenspiel
183
+ Vibraphone
184
+ Steelpan
185
+ Orchestra
186
+ Brass instrument
187
+ French horn
188
+ Trumpet
189
+ Trombone
190
+ Bowed string instrument
191
+ String section
192
+ Violin, fiddle
193
+ Pizzicato
194
+ Cello
195
+ Double bass
196
+ Wind instrument, woodwind instrument
197
+ Flute
198
+ Saxophone
199
+ Clarinet
200
+ Harp
201
+ Bell
202
+ Church bell
203
+ Jingle bell
204
+ Bicycle bell
205
+ Tuning fork
206
+ Chime
207
+ Wind chime
208
+ Change ringing (campanology)
209
+ Harmonica
210
+ Accordion
211
+ Bagpipes
212
+ Didgeridoo
213
+ Shofar
214
+ Theremin
215
+ Singing bowl
216
+ Scratching (performance technique)
217
+ Pop music
218
+ Hip hop music
219
+ Beatboxing
220
+ Rock music
221
+ Heavy metal
222
+ Punk rock
223
+ Grunge
224
+ Progressive rock
225
+ Rock and roll
226
+ Psychedelic rock
227
+ Rhythm and blues
228
+ Soul music
229
+ Reggae
230
+ Country
231
+ Swing music
232
+ Bluegrass
233
+ Funk
234
+ Folk music
235
+ Middle Eastern music
236
+ Jazz
237
+ Disco
238
+ Classical music
239
+ Opera
240
+ Electronic music
241
+ House music
242
+ Techno
243
+ Dubstep
244
+ Drum and bass
245
+ Electronica
246
+ Electronic dance music
247
+ Ambient music
248
+ Trance music
249
+ Music of Latin America
250
+ Salsa music
251
+ Flamenco
252
+ Blues
253
+ Music for children
254
+ New-age music
255
+ Vocal music
256
+ A capella
257
+ Music of Africa
258
+ Afrobeat
259
+ Christian music
260
+ Gospel music
261
+ Music of Asia
262
+ Carnatic music
263
+ Music of Bollywood
264
+ Ska
265
+ Traditional music
266
+ Independent music
267
+ Song
268
+ Background music
269
+ Theme music
270
+ Jingle (music)
271
+ Soundtrack music
272
+ Lullaby
273
+ Video game music
274
+ Christmas music
275
+ Dance music
276
+ Wedding music
277
+ Happy music
278
+ Funny music
279
+ Sad music
280
+ Tender music
281
+ Exciting music
282
+ Angry music
283
+ Scary music
284
+ Wind
285
+ Rustling leaves
286
+ Wind noise (microphone)
287
+ Thunderstorm
288
+ Thunder
289
+ Water
290
+ Rain
291
+ Raindrop
292
+ Rain on surface
293
+ Stream
294
+ Waterfall
295
+ Ocean
296
+ Waves, surf
297
+ Steam
298
+ Gurgling
299
+ Fire
300
+ Crackle
301
+ Vehicle
302
+ Boat, Water vehicle
303
+ Sailboat, sailing ship
304
+ Rowboat, canoe, kayak
305
+ Motorboat, speedboat
306
+ Ship
307
+ Motor vehicle (road)
308
+ Car
309
+ Vehicle horn, car horn, honking
310
+ Toot
311
+ Car alarm
312
+ Power windows, electric windows
313
+ Skidding
314
+ Tire squeal
315
+ Car passing by
316
+ Race car, auto racing
317
+ Truck
318
+ Air brake
319
+ Air horn, truck horn
320
+ Reversing beeps
321
+ Ice cream truck, ice cream van
322
+ Bus
323
+ Emergency vehicle
324
+ Police car (siren)
325
+ Ambulance (siren)
326
+ Fire engine, fire truck (siren)
327
+ Motorcycle
328
+ Traffic noise, roadway noise
329
+ Rail transport
330
+ Train
331
+ Train whistle
332
+ Train horn
333
+ Railroad car, train wagon
334
+ Train wheels squealing
335
+ Subway, metro, underground
336
+ Aircraft
337
+ Aircraft engine
338
+ Jet engine
339
+ Propeller, airscrew
340
+ Helicopter
341
+ Fixed-wing aircraft, airplane
342
+ Bicycle
343
+ Skateboard
344
+ Engine
345
+ Light engine (high frequency)
346
+ Dental drill, dentist's drill
347
+ Lawn mower
348
+ Chainsaw
349
+ Medium engine (mid frequency)
350
+ Heavy engine (low frequency)
351
+ Engine knocking
352
+ Engine starting
353
+ Idling
354
+ Accelerating, revving, vroom
355
+ Door
356
+ Doorbell
357
+ Ding-dong
358
+ Sliding door
359
+ Slam
360
+ Knock
361
+ Tap
362
+ Squeak
363
+ Cupboard open or close
364
+ Drawer open or close
365
+ Dishes, pots, and pans
366
+ Cutlery, silverware
367
+ Chopping (food)
368
+ Frying (food)
369
+ Microwave oven
370
+ Blender
371
+ Water tap, faucet
372
+ Sink (filling or washing)
373
+ Bathtub (filling or washing)
374
+ Hair dryer
375
+ Toilet flush
376
+ Toothbrush
377
+ Electric toothbrush
378
+ Vacuum cleaner
379
+ Zipper (clothing)
380
+ Keys jangling
381
+ Coin (dropping)
382
+ Scissors
383
+ Electric shaver, electric razor
384
+ Shuffling cards
385
+ Typing
386
+ Typewriter
387
+ Computer keyboard
388
+ Writing
389
+ Alarm
390
+ Telephone
391
+ Telephone bell ringing
392
+ Ringtone
393
+ Telephone dialing, DTMF
394
+ Dial tone
395
+ Busy signal
396
+ Alarm clock
397
+ Siren
398
+ Civil defense siren
399
+ Buzzer
400
+ Smoke detector, smoke alarm
401
+ Fire alarm
402
+ Foghorn
403
+ Whistle
404
+ Steam whistle
405
+ Mechanisms
406
+ Ratchet, pawl
407
+ Clock
408
+ Tick
409
+ Tick-tock
410
+ Gears
411
+ Pulleys
412
+ Sewing machine
413
+ Mechanical fan
414
+ Air conditioning
415
+ Cash register
416
+ Printer
417
+ Camera
418
+ Single-lens reflex camera
419
+ Tools
420
+ Hammer
421
+ Jackhammer
422
+ Sawing
423
+ Filing (rasp)
424
+ Sanding
425
+ Power tool
426
+ Drill
427
+ Explosion
428
+ Gunshot, gunfire
429
+ Machine gun
430
+ Fusillade
431
+ Artillery fire
432
+ Cap gun
433
+ Fireworks
434
+ Firecracker
435
+ Burst, pop
436
+ Eruption
437
+ Boom
438
+ Wood
439
+ Chop
440
+ Splinter
441
+ Crack
442
+ Glass
443
+ Chink, clink
444
+ Shatter
445
+ Liquid
446
+ Splash, splatter
447
+ Slosh
448
+ Squish
449
+ Drip
450
+ Pour
451
+ Trickle, dribble
452
+ Gush
453
+ Fill (with liquid)
454
+ Spray
455
+ Pump (liquid)
456
+ Stir
457
+ Boiling
458
+ Sonar
459
+ Arrow
460
+ Whoosh, swoosh, swish
461
+ Thump, thud
462
+ Thunk
463
+ Electronic tuner
464
+ Effects unit
465
+ Chorus effect
466
+ Basketball bounce
467
+ Bang
468
+ Slap, smack
469
+ Whack, thwack
470
+ Smash, crash
471
+ Breaking
472
+ Bouncing
473
+ Whip
474
+ Flap
475
+ Scratch
476
+ Scrape
477
+ Rub
478
+ Roll
479
+ Crushing
480
+ Crumpling, crinkling
481
+ Tearing
482
+ Beep, bleep
483
+ Ping
484
+ Ding
485
+ Clang
486
+ Squeal
487
+ Creak
488
+ Rustle
489
+ Whir
490
+ Clatter
491
+ Sizzle
492
+ Clicking
493
+ Clickety-clack
494
+ Rumble
495
+ Plop
496
+ Jingle, tinkle
497
+ Hum
498
+ Zing
499
+ Boing
500
+ Crunch
501
+ Silence
502
+ Sine wave
503
+ Harmonic
504
+ Chirp tone
505
+ Sound effect
506
+ Pulse
507
+ Inside, small room
508
+ Inside, large room or hall
509
+ Inside, public space
510
+ Outside, urban or manmade
511
+ Outside, rural or natural
512
+ Reverberation
513
+ Echo
514
+ Noise
515
+ Environmental noise
516
+ Static
517
+ Mains hum
518
+ Distortion
519
+ Sidetone
520
+ Cacophony
521
+ White noise
522
+ Pink noise
523
+ Throbbing
524
+ Vibration
525
+ Television
526
+ Radio
527
+ Field recording
Models~/ced_tiny_fp32.sentis ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e725666b9e3e7213d8f03c3b5c9fc00dbfe951f33ab3599c53758bbc7fc00fd6
3
+ size 23204224
README.md ADDED
@@ -0,0 +1,76 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ license: apache-2.0
3
+ library_name: unity-sentis
4
+ tags:
5
+ - unity
6
+ - sentis
7
+ - on-device
8
+ - audio-classification
9
+ - audio-tagging
10
+ - sound-event-detection
11
+ - audioset
12
+ ---
13
+
14
+ # sentis-ced-tiny
15
+
16
+ CED-Tiny (Consistent Ensemble Distillation ViT) AudioSet audio tagger converted to **Unity Inference Engine (Sentis)** FP32. Recognizes 527 AudioSet sound classes (animals, alarms,
17
+ vehicles, household and environmental sounds, music, human non-speech sounds, ...) from a raw mono waveform, and returns a
18
+ 192-d embedding usable for your own classes.
19
+
20
+ ## Files
21
+
22
+ ```
23
+ ced_tiny_fp32.sentis # waveform [1, 48000] (16000 Hz mono, 3 s) -> scores [1, 527] (sigmoid), embedding [1, 192]
24
+ audioset_labels.txt # 527 AudioSet display names, index order of the scores
25
+ CedTiny.cs # self-contained Unity Sentis inference
26
+ ```
27
+
28
+ ## Inference
29
+
30
+ The log-mel frontend is baked into the model (STFT as a fixed-basis Conv1d), so the whole pipeline waveform -> scores runs
31
+ in one Sentis graph on CPU or GPU, with no feature extraction in C#. `CedTiny` resamples any input rate to 16000 Hz,
32
+ classifies the last 3 s, and also offers a sliding-window streaming buffer for microphone input.
33
+
34
+ A complete self-contained implementation lives in [`CedTiny.cs`](Runtime/CedTiny.cs). Minimal usage:
35
+
36
+ ```csharp
37
+ var tagger = new CedTiny(BackendType.GPUCompute);
38
+ tagger.Load(modelRoot); // folder holding ced_tiny_fp32.sentis + audioset_labels.txt
39
+ tagger.Warmup();
40
+
41
+ // one-shot: an AudioClip (any sample rate / channel count)
42
+ var data = new float[clip.samples * clip.channels];
43
+ clip.GetData(data, 0);
44
+ var scores = tagger.Classify(CedTiny.ToMono(data, clip.channels), clip.frequency);
45
+ foreach (var p in tagger.Top(scores, k: 5, minScore: 0.1f))
46
+ Debug.Log($"{p.Label}: {p.Score:F2}");
47
+
48
+ // streaming: feed microphone chunks, classify the latest window every 0.25-0.5 s
49
+ tagger.Push(micChunk, micSampleRate);
50
+ var latest = await tagger.ClassifyLatestAsync();
51
+ ```
52
+
53
+ > Scores are independent sigmoid probabilities (multi-label), so several classes can be high at once; threshold per class
54
+ > (0.1-0.3 is a common starting point). `tagger.Embedding` holds the 192-d embedding of the last call for training a small
55
+ > classifier on your own sound classes.
56
+
57
+ ## Conversion notes
58
+
59
+ - Source: [mispeech/ced-tiny](https://huggingface.co/mispeech/ced-tiny) (`model.safetensors`, re-exported from PyTorch; the repo's own `model.onnx` is int8-quantized and does not import into Sentis).
60
+ - Architecture: ViT (12 layers, dim 192, 3 heads, 16x16 patches) trained with consistent ensemble distillation on AudioSet; 5.5M parameters.
61
+ - Frontend: 64-bin log-mel, 16 kHz, n_fft 512 / window 512 / hop 160, power to dB with `top_db = 120` (CED `CedFeatureExtractor`). Reimplemented as a Conv1d with a windowed DFT basis + mel matrix taken from the original library,
62
+ so no STFT / complex op is needed; it matches the original feature extractor.
63
+ - Exported with `torch.onnx.export` (opset 17) as one waveform -> (scores, embedding) graph at a fixed 3 s window
64
+ and imported into Sentis 2.6.1.
65
+ - The exported graph matches the original PyTorch pipeline in ONNX Runtime, and Unity Sentis output matches ONNX Runtime
66
+ on the same input (CPU and GPUCompute).
67
+ - CED splits inputs longer than 10 s into chunks; the 3 s window used here stays within one chunk, so the
68
+ graph is the plain ViT forward (time positional embeddings for the first 18 patches).
69
+ - The model was trained on 10 s clips: windows shorter than ~2 s lose noticeable accuracy, so keep the 3 s
70
+ window (`Push` + `ClassifyLatest`) rather than classifying very short snippets.
71
+
72
+ ## License & attribution
73
+
74
+ Apache License 2.0. Converted from CED by Heinrich Dinkel et al. (https://github.com/RicherMans/CED, https://huggingface.co/mispeech/ced-tiny).
75
+ Paper: [CED: Consistent ensemble distillation for audio tagging](https://arxiv.org/abs/2308.11957) (ICASSP 2024).
76
+ AudioSet class names: [AudioSet ontology](https://research.google.com/audioset/) (CC BY 4.0).
README.md.meta ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ fileFormatVersion: 2
2
+ guid: b10799c7ab7f40588ec407a428b39063
3
+ TextScriptImporter:
4
+ externalObjects: {}
5
+ userData:
6
+ assetBundleName:
7
+ assetBundleVariant:
Runtime.meta ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ fileFormatVersion: 2
2
+ guid: f8dfa59f676344dbb0314f7321225c44
3
+ folderAsset: yes
4
+ DefaultImporter:
5
+ externalObjects: {}
6
+ userData:
7
+ assetBundleName:
8
+ assetBundleVariant:
Runtime/CedTiny.cs ADDED
@@ -0,0 +1,200 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // CED-Tiny (Consistent Ensemble Distillation ViT) AudioSet audio tagger. Self-contained Unity Sentis inference.
2
+ using System;
3
+ using System.Collections.Generic;
4
+ using System.IO;
5
+ using System.Threading.Tasks;
6
+ using Unity.InferenceEngine;
7
+ using UnityEngine;
8
+
9
+ namespace SentisModels
10
+ {
11
+ // Mono waveform -> 527 AudioSet class probabilities (multi-label: independent sigmoids, they do not sum to 1) and a
12
+ // 192-d clip embedding. The log-mel frontend (STFT as a fixed-basis Conv1d) is baked into the model, so the input is
13
+ // the raw 16000 Hz waveform in [-1, 1]; one inference covers a 3 s window. Other sample rates are resampled here.
14
+ public sealed class CedTiny : IDisposable
15
+ {
16
+ public const int SampleRate = 16000;
17
+ public const int WindowSamples = 48000;
18
+ public const int ClassCount = 527;
19
+ public const int EmbeddingSize = 192;
20
+ const string DefaultModelFile = "ced_tiny_fp32.sentis";
21
+ const string DefaultLabelFile = "audioset_labels.txt";
22
+
23
+ public struct Prediction
24
+ {
25
+ public int Index; // AudioSet class index (ontology order, class_labels_indices.csv)
26
+ public string Label;
27
+ public float Score; // sigmoid probability
28
+ }
29
+
30
+ readonly BackendType m_BackendType;
31
+
32
+ Worker m_Worker;
33
+ Tensor<float> m_InputTensor;
34
+ string[] m_Labels;
35
+ readonly float[] m_Window = new float[WindowSamples];
36
+ readonly float[] m_Stream = new float[WindowSamples]; // sliding buffer for Push / ClassifyLatest
37
+
38
+ // Embedding of the most recent Classify call (penultimate features; use for your own classes / similarity).
39
+ public float[] Embedding { get; } = new float[EmbeddingSize];
40
+ public IReadOnlyList<string> Labels => m_Labels;
41
+
42
+ public CedTiny(BackendType backendType = BackendType.GPUCompute)
43
+ {
44
+ m_BackendType = backendType == BackendType.GPUCompute && !SystemInfo.supportsComputeShaders
45
+ ? BackendType.CPU
46
+ : backendType;
47
+ }
48
+
49
+ public bool IsLoaded => m_Worker != null;
50
+
51
+ // modelRoot: absolute path to the folder holding ced_tiny_fp32.sentis and audioset_labels.txt.
52
+ public void Load(string modelRoot, string modelFile = DefaultModelFile)
53
+ {
54
+ Load(ModelLoader.Load(Path.Combine(modelRoot, modelFile)), File.ReadAllLines(Path.Combine(modelRoot, DefaultLabelFile)));
55
+ }
56
+
57
+ // For platforms where the files must be read as bytes / text (e.g. Android StreamingAssets).
58
+ public void Load(byte[] modelBytes, string labelsText)
59
+ {
60
+ using var stream = new MemoryStream(modelBytes);
61
+ Load(ModelLoader.Load(stream), labelsText.Split(new[] { "\r\n", "\n" }, StringSplitOptions.RemoveEmptyEntries));
62
+ }
63
+
64
+ void Load(Model model, string[] labels)
65
+ {
66
+ if (labels.Length != ClassCount)
67
+ throw new ArgumentException($"expected {ClassCount} labels, got {labels.Length}");
68
+ DisposeResources();
69
+ m_Labels = labels;
70
+ m_Worker = new Worker(model, m_BackendType);
71
+ m_InputTensor = new Tensor<float>(new TensorShape(1, WindowSamples));
72
+ Array.Clear(m_Stream, 0, m_Stream.Length);
73
+ }
74
+
75
+ // One silent pass so the first real call doesn't pay shader compilation / allocation.
76
+ public void Warmup()
77
+ {
78
+ Classify(new float[WindowSamples]);
79
+ }
80
+
81
+ // samples: mono PCM at sampleRate. The LAST 3 s are classified (zero-padded in front when shorter).
82
+ public float[] Classify(float[] samples, int sampleRate = SampleRate)
83
+ {
84
+ Schedule(samples, sampleRate);
85
+ return Read();
86
+ }
87
+
88
+ // Async readback variant; prefer this for per-frame use on GPU backends (no main-thread stall).
89
+ public async Task<float[]> ClassifyAsync(float[] samples, int sampleRate = SampleRate)
90
+ {
91
+ Schedule(samples, sampleRate);
92
+ using var scores = await (m_Worker.PeekOutput("scores") as Tensor<float>).ReadbackAndCloneAsync();
93
+ using var embedding = await (m_Worker.PeekOutput("embedding") as Tensor<float>).ReadbackAndCloneAsync();
94
+ embedding.DownloadToArray().CopyTo(Embedding, 0);
95
+ return scores.DownloadToArray();
96
+ }
97
+
98
+ // Streaming: append new microphone / clip samples (mono, any rate) to the sliding window.
99
+ public void Push(float[] samples, int sampleRate = SampleRate)
100
+ {
101
+ var x = sampleRate == SampleRate ? samples : Resample(samples, sampleRate, SampleRate);
102
+ if (x.Length >= WindowSamples)
103
+ {
104
+ Array.Copy(x, x.Length - WindowSamples, m_Stream, 0, WindowSamples);
105
+ return;
106
+ }
107
+ Array.Copy(m_Stream, x.Length, m_Stream, 0, WindowSamples - x.Length);
108
+ Array.Copy(x, 0, m_Stream, WindowSamples - x.Length, x.Length);
109
+ }
110
+
111
+ // Classifies the last 3 s pushed with Push (call e.g. every 0.25-0.5 s for a sliding window).
112
+ public float[] ClassifyLatest() => Classify(m_Stream);
113
+
114
+ public Task<float[]> ClassifyLatestAsync() => ClassifyAsync(m_Stream);
115
+
116
+ // Highest-scoring classes, best first.
117
+ public List<Prediction> Top(float[] scores, int k = 5, float minScore = 0f)
118
+ {
119
+ var order = new int[scores.Length];
120
+ for (var i = 0; i < order.Length; i++)
121
+ order[i] = i;
122
+ Array.Sort(order, (a, b) => scores[b].CompareTo(scores[a]));
123
+ var result = new List<Prediction>();
124
+ for (var i = 0; i < order.Length && result.Count < k; i++)
125
+ {
126
+ if (scores[order[i]] < minScore)
127
+ break;
128
+ result.Add(new Prediction { Index = order[i], Label = m_Labels[order[i]], Score = scores[order[i]] });
129
+ }
130
+ return result;
131
+ }
132
+
133
+ // Average the channels of interleaved PCM (AudioClip.GetData / Microphone data).
134
+ public static float[] ToMono(float[] interleaved, int channels)
135
+ {
136
+ if (channels <= 1)
137
+ return interleaved;
138
+ var mono = new float[interleaved.Length / channels];
139
+ for (var i = 0; i < mono.Length; i++)
140
+ {
141
+ var sum = 0f;
142
+ for (var c = 0; c < channels; c++)
143
+ sum += interleaved[i * channels + c];
144
+ mono[i] = sum / channels;
145
+ }
146
+ return mono;
147
+ }
148
+
149
+ // Linear-interpolation resampler (adequate for tagging; use a band-limited resampler for best fidelity).
150
+ public static float[] Resample(float[] src, int srcRate, int dstRate)
151
+ {
152
+ if (srcRate == dstRate || src.Length == 0)
153
+ return src;
154
+ var n = (int)((long)src.Length * dstRate / srcRate);
155
+ var dst = new float[n];
156
+ var step = (double)srcRate / dstRate;
157
+ for (var i = 0; i < n; i++)
158
+ {
159
+ var p = i * step;
160
+ var j = (int)p;
161
+ var t = (float)(p - j);
162
+ dst[i] = j + 1 < src.Length ? src[j] + (src[j + 1] - src[j]) * t : src[src.Length - 1];
163
+ }
164
+ return dst;
165
+ }
166
+
167
+ void Schedule(float[] samples, int sampleRate)
168
+ {
169
+ if (m_Worker == null)
170
+ throw new InvalidOperationException("CedTiny.Load must be called before Classify.");
171
+ var x = sampleRate == SampleRate ? samples : Resample(samples, sampleRate, SampleRate);
172
+ var n = Math.Min(x.Length, WindowSamples);
173
+ Array.Clear(m_Window, 0, WindowSamples - n);
174
+ Array.Copy(x, x.Length - n, m_Window, WindowSamples - n, n);
175
+ m_InputTensor.Upload(m_Window);
176
+ m_Worker.Schedule(m_InputTensor);
177
+ }
178
+
179
+ float[] Read()
180
+ {
181
+ using var scores = (m_Worker.PeekOutput("scores") as Tensor<float>).ReadbackAndClone();
182
+ using var embedding = (m_Worker.PeekOutput("embedding") as Tensor<float>).ReadbackAndClone();
183
+ embedding.DownloadToArray().CopyTo(Embedding, 0);
184
+ return scores.DownloadToArray();
185
+ }
186
+
187
+ void DisposeResources()
188
+ {
189
+ m_Worker?.Dispose();
190
+ m_Worker = null;
191
+ m_InputTensor?.Dispose();
192
+ m_InputTensor = null;
193
+ }
194
+
195
+ public void Dispose()
196
+ {
197
+ DisposeResources();
198
+ }
199
+ }
200
+ }
Runtime/CedTiny.cs.meta ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ fileFormatVersion: 2
2
+ guid: bf91064d8c734bbfabd217c3600407b1
3
+ MonoImporter:
4
+ externalObjects: {}
5
+ serializedVersion: 2
6
+ defaultReferences: []
7
+ executionOrder: 0
8
+ icon: {instanceID: 0}
9
+ userData:
10
+ assetBundleName:
11
+ assetBundleVariant:
Runtime/Sky.Sentis.CedTiny.asmdef ADDED
@@ -0,0 +1,16 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "name": "Sky.Sentis.CedTiny",
3
+ "rootNamespace": "SentisModels",
4
+ "references": [
5
+ "Unity.InferenceEngine"
6
+ ],
7
+ "includePlatforms": [],
8
+ "excludePlatforms": [],
9
+ "allowUnsafeCode": false,
10
+ "overrideReferences": false,
11
+ "precompiledReferences": [],
12
+ "autoReferenced": true,
13
+ "defineConstraints": [],
14
+ "versionDefines": [],
15
+ "noEngineReferences": false
16
+ }
Runtime/Sky.Sentis.CedTiny.asmdef.meta ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ fileFormatVersion: 2
2
+ guid: 20a66193364c438abaeb54413f39873f
3
+ AssemblyDefinitionImporter:
4
+ externalObjects: {}
5
+ userData:
6
+ assetBundleName:
7
+ assetBundleVariant:
package.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "name": "com.sky.sentis.ced-tiny",
3
+ "version": "0.1.0",
4
+ "displayName": "CED-Tiny Audio Tagger (Sentis)",
5
+ "unity": "6000.0",
6
+ "dependencies": {
7
+ "com.unity.ai.inference": "2.6.1"
8
+ }
9
+ }
package.json.meta ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ fileFormatVersion: 2
2
+ guid: ab637ba17ae0498882302f4dfa1535e5
3
+ PackageManifestImporter:
4
+ externalObjects: {}
5
+ userData:
6
+ assetBundleName:
7
+ assetBundleVariant: