d923aa1e31
Hopfield + Hebbian hybrid memory system for LLMs. Two nights of experiments (16 iterations), validated on LongMemEval (ICLR 2025). Architecture: - Single-hop: Two-Stage Hopfield (NN top-20 → softmax settle) - Multi-hop: Hebbian W matrix with WTA pattern separation - 64% on LongMemEval (500 questions), retrieval-only, no LLM dependency - 4ms latency @ 20K memories, ~1GB VRAM Key findings: - Hopfield attention solved noise tolerance (20% → 100% vs flat Hebbian) - WTA pattern separation enables 20K+ capacity - Multi-hop associative chains (6 hops, CosSim=1.0) — RAG can't do this - MiniLM-L6 is optimal (discrimination gap > absolute similarity) - Paraphrase cue augmentation: 55% → 100% on synthetic, 36% → 64% on benchmark - SNN encoder viable (CosSim 0.99) but not needed for current architecture
504 lines
12 KiB
JSON
504 lines
12 KiB
JSON
[
|
|
{
|
|
"method": "direct_hebbian",
|
|
"correct": 1.0,
|
|
"wrong": 0,
|
|
"disc": 1.0,
|
|
"w_stats": {
|
|
"mean": 0.0012326654978096485,
|
|
"std": 0.001009981264360249,
|
|
"abs_mean": 0.0012326654978096485,
|
|
"sparsity": 0.5203375816345215,
|
|
"max": 0.01220703125,
|
|
"min": 0.0
|
|
},
|
|
"time": 0.018577098846435547,
|
|
"num_pairs": 1,
|
|
"lr": 0.5,
|
|
"test": "hebb_pairs_1"
|
|
},
|
|
{
|
|
"method": "direct_hebbian",
|
|
"correct": 0.9141042113304139,
|
|
"wrong": 0.8997887462377548,
|
|
"disc": 0.014315465092659019,
|
|
"w_stats": {
|
|
"mean": 0.006221722345799208,
|
|
"std": 0.0023041535168886185,
|
|
"abs_mean": 0.006221722345799208,
|
|
"sparsity": 0.0006265640258789062,
|
|
"max": 0.023681640625,
|
|
"min": 0.0
|
|
},
|
|
"time": 0.0002448558807373047,
|
|
"num_pairs": 5,
|
|
"lr": 0.5,
|
|
"test": "hebb_pairs_5"
|
|
},
|
|
{
|
|
"method": "direct_hebbian",
|
|
"correct": 0.8956584692001343,
|
|
"wrong": 0.8881289852990044,
|
|
"disc": 0.007529483901129841,
|
|
"w_stats": {
|
|
"mean": 0.012574371881783009,
|
|
"std": 0.0033114321995526552,
|
|
"abs_mean": 0.012574371881783009,
|
|
"sparsity": 0.0,
|
|
"max": 0.034423828125,
|
|
"min": 0.0015869140625
|
|
},
|
|
"time": 0.0003325939178466797,
|
|
"num_pairs": 10,
|
|
"lr": 0.5,
|
|
"test": "hebb_pairs_10"
|
|
},
|
|
{
|
|
"method": "direct_hebbian",
|
|
"correct": 0.8879856646060944,
|
|
"wrong": 0.8841540270730068,
|
|
"disc": 0.003831637533087573,
|
|
"w_stats": {
|
|
"mean": 0.024841923266649246,
|
|
"std": 0.004587384406477213,
|
|
"abs_mean": 0.024841923266649246,
|
|
"sparsity": 0.0,
|
|
"max": 0.054443359375,
|
|
"min": 0.0079345703125
|
|
},
|
|
"time": 0.0014731884002685547,
|
|
"num_pairs": 20,
|
|
"lr": 0.5,
|
|
"test": "hebb_pairs_20"
|
|
},
|
|
{
|
|
"method": "direct_hebbian",
|
|
"correct": 0.8821950948238373,
|
|
"wrong": 0.8806546637719991,
|
|
"disc": 0.0015404310518382092,
|
|
"w_stats": {
|
|
"mean": 0.06239410862326622,
|
|
"std": 0.0072029875591397285,
|
|
"abs_mean": 0.06239410862326622,
|
|
"sparsity": 0.0,
|
|
"max": 0.1075439453125,
|
|
"min": 0.0311279296875
|
|
},
|
|
"time": 0.0010445117950439453,
|
|
"num_pairs": 50,
|
|
"lr": 0.5,
|
|
"test": "hebb_pairs_50"
|
|
},
|
|
{
|
|
"method": "direct_hebbian",
|
|
"correct": 0.8799643820524216,
|
|
"wrong": 0.8791960634968498,
|
|
"disc": 0.0007683185555718008,
|
|
"w_stats": {
|
|
"mean": 0.12517985701560974,
|
|
"std": 0.010384579189121723,
|
|
"abs_mean": 0.12517985701560974,
|
|
"sparsity": 0.0,
|
|
"max": 0.181884765625,
|
|
"min": 0.080810546875
|
|
},
|
|
"time": 0.001986265182495117,
|
|
"num_pairs": 100,
|
|
"lr": 0.5,
|
|
"test": "hebb_pairs_100"
|
|
},
|
|
{
|
|
"method": "direct_hebbian",
|
|
"correct": 0.8980890333652496,
|
|
"wrong": 0.8907561533980899,
|
|
"disc": 0.0073328799671597,
|
|
"w_stats": {
|
|
"mean": 0.00025050563272088766,
|
|
"std": 6.504161865450442e-05,
|
|
"abs_mean": 0.00025050563272088766,
|
|
"sparsity": 1.0,
|
|
"max": 0.0007348632207140326,
|
|
"min": 3.9062499126885086e-05
|
|
},
|
|
"time": 0.00024819374084472656,
|
|
"num_pairs": 10,
|
|
"lr": 0.01,
|
|
"test": "hebb_lr_0.01"
|
|
},
|
|
{
|
|
"method": "direct_hebbian",
|
|
"correct": 0.8962267279624939,
|
|
"wrong": 0.8887399951616923,
|
|
"disc": 0.007486732800801588,
|
|
"w_stats": {
|
|
"mean": 0.002498718211427331,
|
|
"std": 0.0006414031959138811,
|
|
"abs_mean": 0.002498718211427331,
|
|
"sparsity": 0.0017719268798828125,
|
|
"max": 0.006787109654396772,
|
|
"min": 0.0003906250058207661
|
|
},
|
|
"time": 0.00022459030151367188,
|
|
"num_pairs": 10,
|
|
"lr": 0.1,
|
|
"test": "hebb_lr_0.1"
|
|
},
|
|
{
|
|
"method": "direct_hebbian",
|
|
"correct": 0.897282725572586,
|
|
"wrong": 0.8897124224238926,
|
|
"disc": 0.007570303148693447,
|
|
"w_stats": {
|
|
"mean": 0.012481803074479103,
|
|
"std": 0.003280544187873602,
|
|
"abs_mean": 0.012481803074479103,
|
|
"sparsity": 0.0,
|
|
"max": 0.035400390625,
|
|
"min": 0.001220703125
|
|
},
|
|
"time": 0.0002167224884033203,
|
|
"num_pairs": 10,
|
|
"lr": 0.5,
|
|
"test": "hebb_lr_0.5"
|
|
},
|
|
{
|
|
"method": "direct_hebbian",
|
|
"correct": 0.8980260252952575,
|
|
"wrong": 0.8906061040030585,
|
|
"disc": 0.007419921292199039,
|
|
"w_stats": {
|
|
"mean": 0.024723926559090614,
|
|
"std": 0.006540043745189905,
|
|
"abs_mean": 0.024723926559090614,
|
|
"sparsity": 0.0,
|
|
"max": 0.07080078125,
|
|
"min": 0.0029296875
|
|
},
|
|
"time": 0.000244140625,
|
|
"num_pairs": 10,
|
|
"lr": 1.0,
|
|
"test": "hebb_lr_1.0"
|
|
},
|
|
{
|
|
"method": "direct_hebbian",
|
|
"correct": 0.8966590642929078,
|
|
"wrong": 0.8892279856734806,
|
|
"disc": 0.007431078619427156,
|
|
"w_stats": {
|
|
"mean": 0.12513691186904907,
|
|
"std": 0.033127814531326294,
|
|
"abs_mean": 0.12513691186904907,
|
|
"sparsity": 0.0,
|
|
"max": 0.34912109375,
|
|
"min": 0.01708984375
|
|
},
|
|
"time": 0.00022292137145996094,
|
|
"num_pairs": 10,
|
|
"lr": 5.0,
|
|
"test": "hebb_lr_5.0"
|
|
},
|
|
{
|
|
"method": "stdp_v2",
|
|
"correct": 0.6561112403869629,
|
|
"wrong": 0,
|
|
"disc": 0.6561112403869629,
|
|
"w_stats": {
|
|
"mean": -0.021826405078172684,
|
|
"std": 0.10513758659362793,
|
|
"abs_mean": 0.0771329402923584,
|
|
"sparsity": 0.01354217529296875,
|
|
"max": 0.9745967388153076,
|
|
"min": -1.0
|
|
},
|
|
"time": 0.020874977111816406,
|
|
"num_pairs": 1,
|
|
"a_plus": 0.01,
|
|
"num_pres": 5,
|
|
"test": "stdp_single"
|
|
},
|
|
{
|
|
"method": "stdp_v2",
|
|
"correct": 0.18890744000673293,
|
|
"wrong": 0.1745286915037367,
|
|
"disc": 0.014378748502996225,
|
|
"w_stats": {
|
|
"mean": -0.022967157885432243,
|
|
"std": 0.0347970575094223,
|
|
"abs_mean": 0.03315284848213196,
|
|
"sparsity": 0.01936030387878418,
|
|
"max": 0.15670186281204224,
|
|
"min": -0.2564994990825653
|
|
},
|
|
"time": 0.26916003227233887,
|
|
"num_pairs": 10,
|
|
"a_plus": 0.001,
|
|
"num_pres": 5,
|
|
"test": "stdp_ap_0.001"
|
|
},
|
|
{
|
|
"method": "stdp_v2",
|
|
"correct": 0.2575246155261993,
|
|
"wrong": 0.24171155989170073,
|
|
"disc": 0.015813055634498585,
|
|
"w_stats": {
|
|
"mean": -0.11097157001495361,
|
|
"std": 0.16529808938503265,
|
|
"abs_mean": 0.15848013758659363,
|
|
"sparsity": 0.003999948501586914,
|
|
"max": 0.7865057587623596,
|
|
"min": -1.0
|
|
},
|
|
"time": 0.2617373466491699,
|
|
"num_pairs": 10,
|
|
"a_plus": 0.005,
|
|
"num_pres": 5,
|
|
"test": "stdp_ap_0.005"
|
|
},
|
|
{
|
|
"method": "stdp_v2",
|
|
"correct": 0.2595768585801125,
|
|
"wrong": 0.24608167906602224,
|
|
"disc": 0.013495179514090239,
|
|
"w_stats": {
|
|
"mean": -0.2241937518119812,
|
|
"std": 0.3288184702396393,
|
|
"abs_mean": 0.31941741704940796,
|
|
"sparsity": 0.0019876956939697266,
|
|
"max": 1.0,
|
|
"min": -1.0
|
|
},
|
|
"time": 0.2628958225250244,
|
|
"num_pairs": 10,
|
|
"a_plus": 0.01,
|
|
"num_pres": 5,
|
|
"test": "stdp_ap_0.01"
|
|
},
|
|
{
|
|
"method": "stdp_v2",
|
|
"correct": 0.2949586361646652,
|
|
"wrong": 0.2823951015869776,
|
|
"disc": 0.012563534577687607,
|
|
"w_stats": {
|
|
"mean": -0.3816400170326233,
|
|
"std": 0.6254727244377136,
|
|
"abs_mean": 0.6577693819999695,
|
|
"sparsity": 0.0006313323974609375,
|
|
"max": 1.0,
|
|
"min": -1.0
|
|
},
|
|
"time": 0.26494669914245605,
|
|
"num_pairs": 10,
|
|
"a_plus": 0.05,
|
|
"num_pres": 5,
|
|
"test": "stdp_ap_0.05"
|
|
},
|
|
{
|
|
"method": "stdp_v2",
|
|
"correct": 0.4278454571962357,
|
|
"wrong": 0.4212547073761622,
|
|
"disc": 0.0065907498200734604,
|
|
"w_stats": {
|
|
"mean": -0.2731684446334839,
|
|
"std": 0.7176912426948547,
|
|
"abs_mean": 0.6977914571762085,
|
|
"sparsity": 0.0005943775177001953,
|
|
"max": 1.0,
|
|
"min": -1.0
|
|
},
|
|
"time": 0.263822078704834,
|
|
"num_pairs": 10,
|
|
"a_plus": 0.1,
|
|
"num_pres": 5,
|
|
"test": "stdp_ap_0.1"
|
|
},
|
|
{
|
|
"method": "stdp_v2",
|
|
"correct": 0.23125857561826707,
|
|
"wrong": 0.2177843016054895,
|
|
"disc": 0.013474274012777565,
|
|
"w_stats": {
|
|
"mean": -0.04514722526073456,
|
|
"std": 0.06740628927946091,
|
|
"abs_mean": 0.06436040252447128,
|
|
"sparsity": 0.009995222091674805,
|
|
"max": 0.39074867963790894,
|
|
"min": -0.4776502847671509
|
|
},
|
|
"time": 0.04672598838806152,
|
|
"num_pairs": 10,
|
|
"a_plus": 0.01,
|
|
"num_pres": 1,
|
|
"test": "stdp_pres_1"
|
|
},
|
|
{
|
|
"method": "stdp_v2",
|
|
"correct": 0.2634019389748573,
|
|
"wrong": 0.25012489590379927,
|
|
"disc": 0.013277043071058037,
|
|
"w_stats": {
|
|
"mean": -0.1368531435728073,
|
|
"std": 0.20195430517196655,
|
|
"abs_mean": 0.19379907846450806,
|
|
"sparsity": 0.0032529830932617188,
|
|
"max": 0.9678819179534912,
|
|
"min": -1.0
|
|
},
|
|
"time": 0.15973997116088867,
|
|
"num_pairs": 10,
|
|
"a_plus": 0.01,
|
|
"num_pres": 3,
|
|
"test": "stdp_pres_3"
|
|
},
|
|
{
|
|
"method": "stdp_v2",
|
|
"correct": 0.2491248592734337,
|
|
"wrong": 0.23636625193887287,
|
|
"disc": 0.012758607334560829,
|
|
"w_stats": {
|
|
"mean": -0.23120653629302979,
|
|
"std": 0.3264971673488617,
|
|
"abs_mean": 0.3213178515434265,
|
|
"sparsity": 0.0019659996032714844,
|
|
"max": 1.0,
|
|
"min": -1.0
|
|
},
|
|
"time": 0.2647593021392822,
|
|
"num_pairs": 10,
|
|
"a_plus": 0.01,
|
|
"num_pres": 5,
|
|
"test": "stdp_pres_5"
|
|
},
|
|
{
|
|
"method": "stdp_v2",
|
|
"correct": 0.2780441254377365,
|
|
"wrong": 0.2647830416758855,
|
|
"disc": 0.013261083761850978,
|
|
"w_stats": {
|
|
"mean": -0.35562774538993835,
|
|
"std": 0.5160319805145264,
|
|
"abs_mean": 0.5376336574554443,
|
|
"sparsity": 0.0010309219360351562,
|
|
"max": 1.0,
|
|
"min": -1.0
|
|
},
|
|
"time": 0.5369422435760498,
|
|
"num_pairs": 10,
|
|
"a_plus": 0.01,
|
|
"num_pres": 10,
|
|
"test": "stdp_pres_10"
|
|
},
|
|
{
|
|
"method": "stdp_v2",
|
|
"correct": 0.2884770005941391,
|
|
"wrong": 0.27335384570890003,
|
|
"disc": 0.015123154885239076,
|
|
"w_stats": {
|
|
"mean": -0.39499378204345703,
|
|
"std": 0.616945207118988,
|
|
"abs_mean": 0.6560592651367188,
|
|
"sparsity": 0.0006542205810546875,
|
|
"max": 1.0,
|
|
"min": -1.0
|
|
},
|
|
"time": 1.07774019241333,
|
|
"num_pairs": 10,
|
|
"a_plus": 0.01,
|
|
"num_pres": 20,
|
|
"test": "stdp_pres_20"
|
|
},
|
|
{
|
|
"method": "stdp_v2",
|
|
"correct": 0.6523751020431519,
|
|
"wrong": 0,
|
|
"disc": 0.6523751020431519,
|
|
"w_stats": {
|
|
"mean": -0.021571537479758263,
|
|
"std": 0.10514378547668457,
|
|
"abs_mean": 0.07724925875663757,
|
|
"sparsity": 0.013613700866699219,
|
|
"max": 0.8662262558937073,
|
|
"min": -1.0
|
|
},
|
|
"time": 0.019371509552001953,
|
|
"num_pairs": 1,
|
|
"a_plus": 0.01,
|
|
"num_pres": 5,
|
|
"test": "stdp_pairs_1"
|
|
},
|
|
{
|
|
"method": "stdp_v2",
|
|
"correct": 0.38826957941055296,
|
|
"wrong": 0.3618871793150902,
|
|
"disc": 0.026382400095462777,
|
|
"w_stats": {
|
|
"mean": -0.11600317060947418,
|
|
"std": 0.23469609022140503,
|
|
"abs_mean": 0.20440348982810974,
|
|
"sparsity": 0.003264188766479492,
|
|
"max": 1.0,
|
|
"min": -1.0
|
|
},
|
|
"time": 0.13651704788208008,
|
|
"num_pairs": 5,
|
|
"a_plus": 0.01,
|
|
"num_pres": 5,
|
|
"test": "stdp_pairs_5"
|
|
},
|
|
{
|
|
"method": "stdp_v2",
|
|
"correct": 0.24784058183431626,
|
|
"wrong": 0.23306812014844683,
|
|
"disc": 0.014772461685869431,
|
|
"w_stats": {
|
|
"mean": -0.22731736302375793,
|
|
"std": 0.3267463445663452,
|
|
"abs_mean": 0.3194453716278076,
|
|
"sparsity": 0.001964569091796875,
|
|
"max": 1.0,
|
|
"min": -1.0
|
|
},
|
|
"time": 0.28417372703552246,
|
|
"num_pairs": 10,
|
|
"a_plus": 0.01,
|
|
"num_pres": 5,
|
|
"test": "stdp_pairs_10"
|
|
},
|
|
{
|
|
"method": "stdp_v2",
|
|
"correct": 0.12115511521697045,
|
|
"wrong": 0.11465478504174634,
|
|
"disc": 0.006500330175224112,
|
|
"w_stats": {
|
|
"mean": -0.42493927478790283,
|
|
"std": 0.4062454402446747,
|
|
"abs_mean": 0.5013920068740845,
|
|
"sparsity": 0.0010747909545898438,
|
|
"max": 1.0,
|
|
"min": -1.0
|
|
},
|
|
"time": 0.5374035835266113,
|
|
"num_pairs": 20,
|
|
"a_plus": 0.01,
|
|
"num_pres": 5,
|
|
"test": "stdp_pairs_20"
|
|
},
|
|
{
|
|
"method": "stdp_v2",
|
|
"correct": 0.01958512845332734,
|
|
"wrong": 0.01876905316739089,
|
|
"disc": 0.000816075285936451,
|
|
"w_stats": {
|
|
"mean": -0.6996303796768188,
|
|
"std": 0.3604925274848938,
|
|
"abs_mean": 0.7365255355834961,
|
|
"sparsity": 0.0003490447998046875,
|
|
"max": 1.0,
|
|
"min": -1.0
|
|
},
|
|
"time": 1.3608872890472412,
|
|
"num_pairs": 50,
|
|
"a_plus": 0.01,
|
|
"num_pres": 5,
|
|
"test": "stdp_pairs_50"
|
|
}
|
|
] |