nuonuo/doc/exp02_results.json
Fam Zheng d923aa1e31 NuoNuo: Hippocampal memory module prototype
Hopfield + Hebbian hybrid memory system for LLMs.
Two nights of experiments (16 iterations), validated on LongMemEval (ICLR 2025).

Architecture:
- Single-hop: Two-Stage Hopfield (NN top-20 → softmax settle)
- Multi-hop: Hebbian W matrix with WTA pattern separation
- 64% on LongMemEval (500 questions), retrieval-only, no LLM dependency
- 4ms latency @ 20K memories, ~1GB VRAM

Key findings:
- Hopfield attention solved noise tolerance (20% → 100% vs flat Hebbian)
- WTA pattern separation enables 20K+ capacity
- Multi-hop associative chains (6 hops, CosSim=1.0) — RAG can't do this
- MiniLM-L6 is optimal (discrimination gap > absolute similarity)
- Paraphrase cue augmentation: 55% → 100% on synthetic, 36% → 64% on benchmark
- SNN encoder viable (CosSim 0.99) but not needed for current architecture
2026-04-07 10:37:24 +01:00

808 lines
14 KiB
JSON

[
{
"num_neurons": 2048,
"num_steps": 64,
"num_pairs": 1,
"firing_rate": 0.05,
"num_presentations": 5,
"a_plus": 0.005,
"a_minus": 0.006,
"mean_correct_sim": 0.0,
"mean_wrong_sim": 0,
"discrimination": 0.0,
"correct_sims": [
0.0
],
"recall_firing_rate": 0.0,
"weight_stats": {
"mean": 0.0,
"std": 0.0,
"abs_mean": 0.0,
"sparsity": 1.0,
"max": 0.0,
"min": 0.0
},
"learn_time": 0.13341617584228516,
"test": "single_pair"
},
{
"num_neurons": 2048,
"num_steps": 64,
"num_pairs": 5,
"firing_rate": 0.05,
"num_presentations": 5,
"a_plus": 0.005,
"a_minus": 0.006,
"mean_correct_sim": 0.0,
"mean_wrong_sim": 0.0,
"discrimination": 0.0,
"correct_sims": [
0.0,
0.0,
0.0,
0.0,
0.0
],
"recall_firing_rate": 0.0,
"weight_stats": {
"mean": 0.0,
"std": 0.0,
"abs_mean": 0.0,
"sparsity": 1.0,
"max": 0.0,
"min": 0.0
},
"learn_time": 0.38840293884277344,
"test": "pairs_5"
},
{
"num_neurons": 2048,
"num_steps": 64,
"num_pairs": 10,
"firing_rate": 0.05,
"num_presentations": 5,
"a_plus": 0.005,
"a_minus": 0.006,
"mean_correct_sim": 0.0,
"mean_wrong_sim": 0.0,
"discrimination": 0.0,
"correct_sims": [
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0
],
"recall_firing_rate": 0.0,
"weight_stats": {
"mean": 0.0,
"std": 0.0,
"abs_mean": 0.0,
"sparsity": 1.0,
"max": 0.0,
"min": 0.0
},
"learn_time": 0.7765586376190186,
"test": "pairs_10"
},
{
"num_neurons": 2048,
"num_steps": 64,
"num_pairs": 20,
"firing_rate": 0.05,
"num_presentations": 5,
"a_plus": 0.005,
"a_minus": 0.006,
"mean_correct_sim": 0.0,
"mean_wrong_sim": 0.0,
"discrimination": 0.0,
"correct_sims": [
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0
],
"recall_firing_rate": 0.0,
"weight_stats": {
"mean": 0.0,
"std": 0.0,
"abs_mean": 0.0,
"sparsity": 1.0,
"max": 0.0,
"min": 0.0
},
"learn_time": 1.5450711250305176,
"test": "pairs_20"
},
{
"num_neurons": 2048,
"num_steps": 64,
"num_pairs": 50,
"firing_rate": 0.05,
"num_presentations": 5,
"a_plus": 0.005,
"a_minus": 0.006,
"mean_correct_sim": 0.0,
"mean_wrong_sim": 0.0,
"discrimination": 0.0,
"correct_sims": [
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0
],
"recall_firing_rate": 0.0,
"weight_stats": {
"mean": 0.0,
"std": 0.0,
"abs_mean": 0.0,
"sparsity": 1.0,
"max": 0.0,
"min": 0.0
},
"learn_time": 3.9536848068237305,
"test": "pairs_50"
},
{
"num_neurons": 2048,
"num_steps": 64,
"num_pairs": 10,
"firing_rate": 0.05,
"num_presentations": 5,
"a_plus": 0.001,
"a_minus": 0.0012,
"mean_correct_sim": 0.0,
"mean_wrong_sim": 0.0,
"discrimination": 0.0,
"correct_sims": [
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0
],
"recall_firing_rate": 0.0,
"weight_stats": {
"mean": 0.0,
"std": 0.0,
"abs_mean": 0.0,
"sparsity": 1.0,
"max": 0.0,
"min": 0.0
},
"learn_time": 0.774240255355835,
"test": "lr_0.001"
},
{
"num_neurons": 2048,
"num_steps": 64,
"num_pairs": 10,
"firing_rate": 0.05,
"num_presentations": 5,
"a_plus": 0.005,
"a_minus": 0.006,
"mean_correct_sim": 0.0,
"mean_wrong_sim": 0.0,
"discrimination": 0.0,
"correct_sims": [
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0
],
"recall_firing_rate": 0.0,
"weight_stats": {
"mean": 0.0,
"std": 0.0,
"abs_mean": 0.0,
"sparsity": 1.0,
"max": 0.0,
"min": 0.0
},
"learn_time": 0.8001570701599121,
"test": "lr_0.005"
},
{
"num_neurons": 2048,
"num_steps": 64,
"num_pairs": 10,
"firing_rate": 0.05,
"num_presentations": 5,
"a_plus": 0.01,
"a_minus": 0.012,
"mean_correct_sim": 0.0,
"mean_wrong_sim": 0.0,
"discrimination": 0.0,
"correct_sims": [
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0
],
"recall_firing_rate": 0.0,
"weight_stats": {
"mean": 0.0,
"std": 0.0,
"abs_mean": 0.0,
"sparsity": 1.0,
"max": 0.0,
"min": 0.0
},
"learn_time": 0.7752792835235596,
"test": "lr_0.01"
},
{
"num_neurons": 2048,
"num_steps": 64,
"num_pairs": 10,
"firing_rate": 0.05,
"num_presentations": 5,
"a_plus": 0.05,
"a_minus": 0.06,
"mean_correct_sim": 0.0,
"mean_wrong_sim": 0.0,
"discrimination": 0.0,
"correct_sims": [
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0
],
"recall_firing_rate": 0.0,
"weight_stats": {
"mean": 0.0,
"std": 0.0,
"abs_mean": 0.0,
"sparsity": 1.0,
"max": 0.0,
"min": 0.0
},
"learn_time": 0.7797460556030273,
"test": "lr_0.05"
},
{
"num_neurons": 2048,
"num_steps": 64,
"num_pairs": 10,
"firing_rate": 0.02,
"num_presentations": 5,
"a_plus": 0.005,
"a_minus": 0.006,
"mean_correct_sim": 0.0,
"mean_wrong_sim": 0.0,
"discrimination": 0.0,
"correct_sims": [
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0
],
"recall_firing_rate": 0.0,
"weight_stats": {
"mean": 0.0,
"std": 0.0,
"abs_mean": 0.0,
"sparsity": 1.0,
"max": 0.0,
"min": 0.0
},
"learn_time": 0.778449296951294,
"test": "fr_0.02"
},
{
"num_neurons": 2048,
"num_steps": 64,
"num_pairs": 10,
"firing_rate": 0.05,
"num_presentations": 5,
"a_plus": 0.005,
"a_minus": 0.006,
"mean_correct_sim": 0.0,
"mean_wrong_sim": 0.0,
"discrimination": 0.0,
"correct_sims": [
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0
],
"recall_firing_rate": 0.0,
"weight_stats": {
"mean": 0.0,
"std": 0.0,
"abs_mean": 0.0,
"sparsity": 1.0,
"max": 0.0,
"min": 0.0
},
"learn_time": 0.7686772346496582,
"test": "fr_0.05"
},
{
"num_neurons": 2048,
"num_steps": 64,
"num_pairs": 10,
"firing_rate": 0.1,
"num_presentations": 5,
"a_plus": 0.005,
"a_minus": 0.006,
"mean_correct_sim": 0.0,
"mean_wrong_sim": 0.0,
"discrimination": 0.0,
"correct_sims": [
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0
],
"recall_firing_rate": 0.0,
"weight_stats": {
"mean": 0.0,
"std": 0.0,
"abs_mean": 0.0,
"sparsity": 1.0,
"max": 0.0,
"min": 0.0
},
"learn_time": 0.7901496887207031,
"test": "fr_0.1"
},
{
"num_neurons": 2048,
"num_steps": 64,
"num_pairs": 10,
"firing_rate": 0.2,
"num_presentations": 5,
"a_plus": 0.005,
"a_minus": 0.006,
"mean_correct_sim": 0.0,
"mean_wrong_sim": 0.0,
"discrimination": 0.0,
"correct_sims": [
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0
],
"recall_firing_rate": 0.0,
"weight_stats": {
"mean": 0.0,
"std": 0.0,
"abs_mean": 0.0,
"sparsity": 1.0,
"max": 0.0,
"min": 0.0
},
"learn_time": 0.7785372734069824,
"test": "fr_0.2"
},
{
"num_neurons": 2048,
"num_steps": 64,
"num_pairs": 10,
"firing_rate": 0.05,
"num_presentations": 1,
"a_plus": 0.005,
"a_minus": 0.006,
"mean_correct_sim": 0.0,
"mean_wrong_sim": 0.0,
"discrimination": 0.0,
"correct_sims": [
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0
],
"recall_firing_rate": 0.0,
"weight_stats": {
"mean": 0.0,
"std": 0.0,
"abs_mean": 0.0,
"sparsity": 1.0,
"max": 0.0,
"min": 0.0
},
"learn_time": 0.18436217308044434,
"test": "pres_1"
},
{
"num_neurons": 2048,
"num_steps": 64,
"num_pairs": 10,
"firing_rate": 0.05,
"num_presentations": 3,
"a_plus": 0.005,
"a_minus": 0.006,
"mean_correct_sim": 0.0,
"mean_wrong_sim": 0.0,
"discrimination": 0.0,
"correct_sims": [
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0
],
"recall_firing_rate": 0.0,
"weight_stats": {
"mean": 0.0,
"std": 0.0,
"abs_mean": 0.0,
"sparsity": 1.0,
"max": 0.0,
"min": 0.0
},
"learn_time": 0.4729011058807373,
"test": "pres_3"
},
{
"num_neurons": 2048,
"num_steps": 64,
"num_pairs": 10,
"firing_rate": 0.05,
"num_presentations": 5,
"a_plus": 0.005,
"a_minus": 0.006,
"mean_correct_sim": 0.0,
"mean_wrong_sim": 0.0,
"discrimination": 0.0,
"correct_sims": [
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0
],
"recall_firing_rate": 0.0,
"weight_stats": {
"mean": 0.0,
"std": 0.0,
"abs_mean": 0.0,
"sparsity": 1.0,
"max": 0.0,
"min": 0.0
},
"learn_time": 0.777827262878418,
"test": "pres_5"
},
{
"num_neurons": 2048,
"num_steps": 64,
"num_pairs": 10,
"firing_rate": 0.05,
"num_presentations": 10,
"a_plus": 0.005,
"a_minus": 0.006,
"mean_correct_sim": 0.0,
"mean_wrong_sim": 0.0,
"discrimination": 0.0,
"correct_sims": [
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0
],
"recall_firing_rate": 0.0,
"weight_stats": {
"mean": 0.0,
"std": 0.0,
"abs_mean": 0.0,
"sparsity": 1.0,
"max": 0.0,
"min": 0.0
},
"learn_time": 1.5397796630859375,
"test": "pres_10"
},
{
"num_neurons": 2048,
"num_steps": 64,
"num_pairs": 10,
"firing_rate": 0.05,
"num_presentations": 20,
"a_plus": 0.005,
"a_minus": 0.006,
"mean_correct_sim": 0.0,
"mean_wrong_sim": 0.0,
"discrimination": 0.0,
"correct_sims": [
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0
],
"recall_firing_rate": 0.0,
"weight_stats": {
"mean": 0.0,
"std": 0.0,
"abs_mean": 0.0,
"sparsity": 1.0,
"max": 0.0,
"min": 0.0
},
"learn_time": 3.1238980293273926,
"test": "pres_20"
},
{
"num_neurons": 1024,
"num_steps": 64,
"num_pairs": 10,
"firing_rate": 0.05,
"num_presentations": 5,
"a_plus": 0.005,
"a_minus": 0.006,
"mean_correct_sim": 0.0,
"mean_wrong_sim": 0.0,
"discrimination": 0.0,
"correct_sims": [
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0
],
"recall_firing_rate": 0.0,
"weight_stats": {
"mean": 0.0,
"std": 0.0,
"abs_mean": 0.0,
"sparsity": 1.0,
"max": 0.0,
"min": 0.0
},
"learn_time": 0.7898683547973633,
"test": "width_1024"
},
{
"num_neurons": 2048,
"num_steps": 64,
"num_pairs": 10,
"firing_rate": 0.05,
"num_presentations": 5,
"a_plus": 0.005,
"a_minus": 0.006,
"mean_correct_sim": 0.0,
"mean_wrong_sim": 0.0,
"discrimination": 0.0,
"correct_sims": [
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0
],
"recall_firing_rate": 0.0,
"weight_stats": {
"mean": 0.0,
"std": 0.0,
"abs_mean": 0.0,
"sparsity": 1.0,
"max": 0.0,
"min": 0.0
},
"learn_time": 0.7738516330718994,
"test": "width_2048"
},
{
"num_neurons": 4096,
"num_steps": 64,
"num_pairs": 10,
"firing_rate": 0.05,
"num_presentations": 5,
"a_plus": 0.005,
"a_minus": 0.006,
"mean_correct_sim": 0.0,
"mean_wrong_sim": 0.0,
"discrimination": 0.0,
"correct_sims": [
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0
],
"recall_firing_rate": 0.0,
"weight_stats": {
"mean": 0.0,
"std": 0.0,
"abs_mean": 0.0,
"sparsity": 1.0,
"max": 0.0,
"min": 0.0
},
"learn_time": 6.378566026687622,
"test": "width_4096"
},
{
"num_neurons": 8192,
"num_steps": 64,
"num_pairs": 10,
"firing_rate": 0.05,
"num_presentations": 5,
"a_plus": 0.005,
"a_minus": 0.006,
"mean_correct_sim": 0.0,
"mean_wrong_sim": 0.0,
"discrimination": 0.0,
"correct_sims": [
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0
],
"recall_firing_rate": 0.0,
"weight_stats": {
"mean": 0.0,
"std": 0.0,
"abs_mean": 0.0,
"sparsity": 1.0,
"max": 0.0,
"min": 0.0
},
"learn_time": 27.351831674575806,
"test": "width_8192"
}
]