diff --git "a/reduced_ppl_average.json" "b/reduced_ppl_average.json" new file mode 100644--- /dev/null +++ "b/reduced_ppl_average.json" @@ -0,0 +1,33680 @@ +{ + "original_ppl": 4.002291202545166, + "reduced_ppl": [ + { + "bit": 1, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 4057.61865234375 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 4064.404052734375 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 3790.370849609375 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 3414.189697265625 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 3719.7109375 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 3788.780517578125 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 3439.8291015625 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 4053.673583984375 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 4067.114013671875 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 4105.2294921875 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 3826.388427734375 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 3893.549560546875 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 3735.858154296875 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 3954.819091796875 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 3971.82373046875 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 4032.228271484375 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 4034.7978515625 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 3987.445068359375 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 3931.552490234375 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 3873.842041015625 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 3941.677978515625 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 4028.519287109375 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 4046.539306640625 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 4030.136962890625 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 4005.552734375 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 3988.019287109375 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 3998.293701171875 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 3979.129638671875 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 4049.033203125 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 4042.755615234375 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 4015.248046875 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 4070.447265625 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 3999.277587890625 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 4007.875732421875 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 3999.670654296875 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 4053.681396484375 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 4048.1181640625 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 4022.25 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 4051.9150390625 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 3984.449462890625 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 4001.268798828125 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 3998.717041015625 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 4057.28564453125 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 4053.642578125 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 4027.992919921875 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 4039.94970703125 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 4010.81640625 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 4010.437744140625 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 3986.06494140625 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 4053.495849609375 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 4049.2880859375 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 4106.9482421875 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 3997.821044921875 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 3943.933837890625 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 3975.556640625 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 3991.702392578125 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 4052.1044921875 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 4044.3095703125 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 4037.288330078125 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 4054.24169921875 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 3975.09375 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 3925.382080078125 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 3989.44970703125 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 4067.6298828125 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 4038.470458984375 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 4098.2509765625 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 4183.53955078125 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 4037.149658203125 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 3968.3857421875 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 4030.1640625 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 4041.390869140625 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 4062.71435546875 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 4112.36865234375 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 4261.72998046875 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 4066.136962890625 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 3976.974609375 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 4042.9521484375 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 4132.4541015625 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 4023.08642578125 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 4055.21240234375 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 4152.29345703125 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 4075.193603515625 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 4117.3369140625 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 4036.42578125 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 4169.078125 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 4089.017822265625 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 4048.647216796875 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 3953.51416015625 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 4001.688720703125 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 3989.194580078125 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 4015.3974609375 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 4052.285888671875 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 4051.551513671875 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 4042.23876953125 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 3993.103515625 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 3963.442626953125 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 3980.30615234375 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 4054.794921875 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 4068.29345703125 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 4000.12451171875 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 4126.26806640625 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 4036.722412109375 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 3988.6240234375 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 4020.091064453125 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 4010.586669921875 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 4098.16064453125 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 4101.71533203125 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 4025.220458984375 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 3846.75 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 3970.21435546875 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 3996.56298828125 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 4048.66650390625 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 4052.057861328125 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 4071.16943359375 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 4074.19921875 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 4103.5615234375 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 4055.2666015625 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 3966.91015625 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 4001.520751953125 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 4028.6884765625 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 4044.298095703125 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 3959.460693359375 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 4076.305419921875 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 4039.47216796875 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 4016.339599609375 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 3996.68115234375 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 4052.95068359375 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 4075.944091796875 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 3979.87353515625 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 4082.17578125 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 4047.558349609375 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 3984.41552734375 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 3952.552734375 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 4358.7978515625 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 4199.17236328125 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 4235.70458984375 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 4265.53173828125 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 3917.90234375 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 3955.090576171875 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 4044.220947265625 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 4000.841796875 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 4022.399658203125 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 4063.9892578125 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 4109.43603515625 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 4010.5791015625 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 4014.754150390625 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 3918.6494140625 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 4074.03173828125 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 4067.990966796875 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 4137.06396484375 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 3934.703369140625 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 3886.89013671875 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 3882.88134765625 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 3905.152099609375 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 4029.4375 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 4108.53515625 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 3894.678466796875 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 4074.3427734375 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 3993.484619140625 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 3636.754150390625 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 3928.60302734375 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 4066.88525390625 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 4055.03857421875 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 4030.96728515625 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 4156.5166015625 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 3902.19970703125 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 4009.3671875 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 3923.705322265625 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 4038.732421875 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 4045.60205078125 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 4104.3994140625 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 3964.198486328125 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 3999.07177734375 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 4049.47705078125 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 4049.1376953125 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 4174.95458984375 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 4085.373291015625 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 4050.728759765625 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 4196.47021484375 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 4022.87158203125 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 3882.581298828125 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 3910.0859375 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 4060.9052734375 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 4039.545166015625 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 3880.227294921875 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 4078.938232421875 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 4077.46435546875 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 4004.456787109375 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 4077.40966796875 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 4004.3837890625 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 4041.568115234375 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 4169.12158203125 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 4021.912841796875 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 4088.1015625 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 4056.60888671875 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 4053.116943359375 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 4029.867919921875 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 4043.72705078125 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 4020.386474609375 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 3882.43701171875 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 4061.38916015625 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 3926.494140625 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 4068.355712890625 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 4016.10986328125 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 4050.404052734375 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 4022.618408203125 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 4030.159912109375 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 4085.762939453125 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 4010.10498046875 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 3983.99755859375 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 4060.630126953125 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 4028.599853515625 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 4174.69970703125 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 4156.33837890625 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 3949.45166015625 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 4054.13720703125 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 4089.8916015625 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 4026.437255859375 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 4025.849853515625 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 3972.35400390625 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 4087.02587890625 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 3946.394775390625 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 3891.433349609375 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 3957.6181640625 + }, + { + "layer": "layer.32.self_attn.q_proj", + "ppl": 4017.511962890625 + }, + { + "layer": "layer.32.self_attn.k_proj", + "ppl": 4028.788330078125 + }, + { + "layer": "layer.32.self_attn.v_proj", + "ppl": 4011.983154296875 + }, + { + "layer": "layer.32.self_attn.o_proj", + "ppl": 4002.666015625 + }, + { + "layer": "layer.32.mlp.gate_proj", + "ppl": 4063.79541015625 + }, + { + "layer": "layer.32.mlp.up_proj", + "ppl": 3991.055419921875 + }, + { + "layer": "layer.32.mlp.down_proj", + "ppl": 4182.55029296875 + }, + { + "layer": "layer.33.self_attn.q_proj", + "ppl": 4050.771240234375 + }, + { + "layer": "layer.33.self_attn.k_proj", + "ppl": 4073.17333984375 + }, + { + "layer": "layer.33.self_attn.v_proj", + "ppl": 3997.965576171875 + }, + { + "layer": "layer.33.self_attn.o_proj", + "ppl": 4100.11962890625 + }, + { + "layer": "layer.33.mlp.gate_proj", + "ppl": 4074.653564453125 + }, + { + "layer": "layer.33.mlp.up_proj", + "ppl": 4058.17578125 + }, + { + "layer": "layer.33.mlp.down_proj", + "ppl": 4010.743896484375 + }, + { + "layer": "layer.34.self_attn.q_proj", + "ppl": 4039.271484375 + }, + { + "layer": "layer.34.self_attn.k_proj", + "ppl": 4054.5703125 + }, + { + "layer": "layer.34.self_attn.v_proj", + "ppl": 4032.8935546875 + }, + { + "layer": "layer.34.self_attn.o_proj", + "ppl": 4063.31103515625 + }, + { + "layer": "layer.34.mlp.gate_proj", + "ppl": 3980.8564453125 + }, + { + "layer": "layer.34.mlp.up_proj", + "ppl": 4082.658935546875 + }, + { + "layer": "layer.34.mlp.down_proj", + "ppl": 3990.263671875 + }, + { + "layer": "layer.35.self_attn.q_proj", + "ppl": 4040.85107421875 + }, + { + "layer": "layer.35.self_attn.k_proj", + "ppl": 4046.813720703125 + }, + { + "layer": "layer.35.self_attn.v_proj", + "ppl": 4035.3984375 + }, + { + "layer": "layer.35.self_attn.o_proj", + "ppl": 4139.25048828125 + }, + { + "layer": "layer.35.mlp.gate_proj", + "ppl": 4083.332275390625 + }, + { + "layer": "layer.35.mlp.up_proj", + "ppl": 4135.8017578125 + }, + { + "layer": "layer.35.mlp.down_proj", + "ppl": 4043.592041015625 + }, + { + "layer": "layer.36.self_attn.q_proj", + "ppl": 4062.7333984375 + }, + { + "layer": "layer.36.self_attn.k_proj", + "ppl": 4037.94677734375 + }, + { + "layer": "layer.36.self_attn.v_proj", + "ppl": 4031.678466796875 + }, + { + "layer": "layer.36.self_attn.o_proj", + "ppl": 4077.2734375 + }, + { + "layer": "layer.36.mlp.gate_proj", + "ppl": 4018.017578125 + }, + { + "layer": "layer.36.mlp.up_proj", + "ppl": 3949.369140625 + }, + { + "layer": "layer.36.mlp.down_proj", + "ppl": 3944.10693359375 + }, + { + "layer": "layer.37.self_attn.q_proj", + "ppl": 4030.4521484375 + }, + { + "layer": "layer.37.self_attn.k_proj", + "ppl": 4035.09033203125 + }, + { + "layer": "layer.37.self_attn.v_proj", + "ppl": 3913.72705078125 + }, + { + "layer": "layer.37.self_attn.o_proj", + "ppl": 4049.639404296875 + }, + { + "layer": "layer.37.mlp.gate_proj", + "ppl": 3950.9736328125 + }, + { + "layer": "layer.37.mlp.up_proj", + "ppl": 3952.202392578125 + }, + { + "layer": "layer.37.mlp.down_proj", + "ppl": 3985.98486328125 + }, + { + "layer": "layer.38.self_attn.q_proj", + "ppl": 3979.311767578125 + }, + { + "layer": "layer.38.self_attn.k_proj", + "ppl": 4075.275634765625 + }, + { + "layer": "layer.38.self_attn.v_proj", + "ppl": 4113.27880859375 + }, + { + "layer": "layer.38.self_attn.o_proj", + "ppl": 4038.1083984375 + }, + { + "layer": "layer.38.mlp.gate_proj", + "ppl": 3995.031005859375 + }, + { + "layer": "layer.38.mlp.up_proj", + "ppl": 3999.384521484375 + }, + { + "layer": "layer.38.mlp.down_proj", + "ppl": 4038.439453125 + }, + { + "layer": "layer.39.self_attn.q_proj", + "ppl": 4039.186767578125 + }, + { + "layer": "layer.39.self_attn.k_proj", + "ppl": 4021.724609375 + }, + { + "layer": "layer.39.self_attn.v_proj", + "ppl": 4061.009765625 + }, + { + "layer": "layer.39.self_attn.o_proj", + "ppl": 3977.145263671875 + }, + { + "layer": "layer.39.mlp.gate_proj", + "ppl": 4046.62841796875 + }, + { + "layer": "layer.39.mlp.up_proj", + "ppl": 4085.396728515625 + }, + { + "layer": "layer.39.mlp.down_proj", + "ppl": 4028.507568359375 + }, + { + "layer": "layer.40.self_attn.q_proj", + "ppl": 4012.970458984375 + }, + { + "layer": "layer.40.self_attn.k_proj", + "ppl": 4010.61767578125 + }, + { + "layer": "layer.40.self_attn.v_proj", + "ppl": 4178.7587890625 + }, + { + "layer": "layer.40.self_attn.o_proj", + "ppl": 4092.54833984375 + }, + { + "layer": "layer.40.mlp.gate_proj", + "ppl": 3956.0185546875 + }, + { + "layer": "layer.40.mlp.up_proj", + "ppl": 4065.08203125 + }, + { + "layer": "layer.40.mlp.down_proj", + "ppl": 4038.501220703125 + }, + { + "layer": "layer.41.self_attn.q_proj", + "ppl": 3994.4443359375 + }, + { + "layer": "layer.41.self_attn.k_proj", + "ppl": 4052.537109375 + }, + { + "layer": "layer.41.self_attn.v_proj", + "ppl": 3922.725341796875 + }, + { + "layer": "layer.41.self_attn.o_proj", + "ppl": 3871.54833984375 + }, + { + "layer": "layer.41.mlp.gate_proj", + "ppl": 4223.7568359375 + }, + { + "layer": "layer.41.mlp.up_proj", + "ppl": 4079.506103515625 + }, + { + "layer": "layer.41.mlp.down_proj", + "ppl": 4037.087890625 + }, + { + "layer": "layer.42.self_attn.q_proj", + "ppl": 4008.556640625 + }, + { + "layer": "layer.42.self_attn.k_proj", + "ppl": 4086.347412109375 + }, + { + "layer": "layer.42.self_attn.v_proj", + "ppl": 4093.96142578125 + }, + { + "layer": "layer.42.self_attn.o_proj", + "ppl": 3994.09375 + }, + { + "layer": "layer.42.mlp.gate_proj", + "ppl": 4071.18115234375 + }, + { + "layer": "layer.42.mlp.up_proj", + "ppl": 4045.2158203125 + }, + { + "layer": "layer.42.mlp.down_proj", + "ppl": 4120.40869140625 + }, + { + "layer": "layer.43.self_attn.q_proj", + "ppl": 4052.34375 + }, + { + "layer": "layer.43.self_attn.k_proj", + "ppl": 4096.9921875 + }, + { + "layer": "layer.43.self_attn.v_proj", + "ppl": 4193.23779296875 + }, + { + "layer": "layer.43.self_attn.o_proj", + "ppl": 3970.812255859375 + }, + { + "layer": "layer.43.mlp.gate_proj", + "ppl": 4014.89990234375 + }, + { + "layer": "layer.43.mlp.up_proj", + "ppl": 3957.27490234375 + }, + { + "layer": "layer.43.mlp.down_proj", + "ppl": 4092.388427734375 + }, + { + "layer": "layer.44.self_attn.q_proj", + "ppl": 4091.90087890625 + }, + { + "layer": "layer.44.self_attn.k_proj", + "ppl": 4066.536376953125 + }, + { + "layer": "layer.44.self_attn.v_proj", + "ppl": 4079.330810546875 + }, + { + "layer": "layer.44.self_attn.o_proj", + "ppl": 4048.840087890625 + }, + { + "layer": "layer.44.mlp.gate_proj", + "ppl": 4067.1025390625 + }, + { + "layer": "layer.44.mlp.up_proj", + "ppl": 3997.74462890625 + }, + { + "layer": "layer.44.mlp.down_proj", + "ppl": 4175.27294921875 + }, + { + "layer": "layer.45.self_attn.q_proj", + "ppl": 4171.9296875 + }, + { + "layer": "layer.45.self_attn.k_proj", + "ppl": 4092.98193359375 + }, + { + "layer": "layer.45.self_attn.v_proj", + "ppl": 4048.828369140625 + }, + { + "layer": "layer.45.self_attn.o_proj", + "ppl": 4333.6298828125 + }, + { + "layer": "layer.45.mlp.gate_proj", + "ppl": 4307.85986328125 + }, + { + "layer": "layer.45.mlp.up_proj", + "ppl": 3608.6865234375 + }, + { + "layer": "layer.45.mlp.down_proj", + "ppl": 4116.748046875 + }, + { + "layer": "layer.46.self_attn.q_proj", + "ppl": 4008.6787109375 + }, + { + "layer": "layer.46.self_attn.k_proj", + "ppl": 4041.13671875 + }, + { + "layer": "layer.46.self_attn.v_proj", + "ppl": 4466.2080078125 + }, + { + "layer": "layer.46.self_attn.o_proj", + "ppl": 3992.814208984375 + }, + { + "layer": "layer.46.mlp.gate_proj", + "ppl": 4098.03173828125 + }, + { + "layer": "layer.46.mlp.up_proj", + "ppl": 4029.7294921875 + }, + { + "layer": "layer.46.mlp.down_proj", + "ppl": 4136.6103515625 + }, + { + "layer": "layer.47.self_attn.q_proj", + "ppl": 4147.919921875 + }, + { + "layer": "layer.47.self_attn.k_proj", + "ppl": 4088.33935546875 + }, + { + "layer": "layer.47.self_attn.v_proj", + "ppl": 4145.53955078125 + }, + { + "layer": "layer.47.self_attn.o_proj", + "ppl": 3911.3837890625 + }, + { + "layer": "layer.47.mlp.gate_proj", + "ppl": 4030.843994140625 + }, + { + "layer": "layer.47.mlp.up_proj", + "ppl": 4257.30615234375 + }, + { + "layer": "layer.47.mlp.down_proj", + "ppl": 4124.91064453125 + }, + { + "layer": "layer.48.self_attn.q_proj", + "ppl": 4036.645263671875 + }, + { + "layer": "layer.48.self_attn.k_proj", + "ppl": 4007.207275390625 + }, + { + "layer": "layer.48.self_attn.v_proj", + "ppl": 4212.11572265625 + }, + { + "layer": "layer.48.self_attn.o_proj", + "ppl": 3905.986328125 + }, + { + "layer": "layer.48.mlp.gate_proj", + "ppl": 4023.194091796875 + }, + { + "layer": "layer.48.mlp.up_proj", + "ppl": 3944.622314453125 + }, + { + "layer": "layer.48.mlp.down_proj", + "ppl": 3909.44091796875 + }, + { + "layer": "layer.49.self_attn.q_proj", + "ppl": 4026.602783203125 + }, + { + "layer": "layer.49.self_attn.k_proj", + "ppl": 4069.958251953125 + }, + { + "layer": "layer.49.self_attn.v_proj", + "ppl": 3855.134521484375 + }, + { + "layer": "layer.49.self_attn.o_proj", + "ppl": 3982.827392578125 + }, + { + "layer": "layer.49.mlp.gate_proj", + "ppl": 3916.378173828125 + }, + { + "layer": "layer.49.mlp.up_proj", + "ppl": 4046.635986328125 + }, + { + "layer": "layer.49.mlp.down_proj", + "ppl": 4216.57275390625 + }, + { + "layer": "layer.50.self_attn.q_proj", + "ppl": 4039.341064453125 + }, + { + "layer": "layer.50.self_attn.k_proj", + "ppl": 4166.56982421875 + }, + { + "layer": "layer.50.self_attn.v_proj", + "ppl": 3927.98095703125 + }, + { + "layer": "layer.50.self_attn.o_proj", + "ppl": 4322.353515625 + }, + { + "layer": "layer.50.mlp.gate_proj", + "ppl": 4012.071044921875 + }, + { + "layer": "layer.50.mlp.up_proj", + "ppl": 4167.876953125 + }, + { + "layer": "layer.50.mlp.down_proj", + "ppl": 3917.513916015625 + }, + { + "layer": "layer.51.self_attn.q_proj", + "ppl": 4125.49658203125 + }, + { + "layer": "layer.51.self_attn.k_proj", + "ppl": 4104.63427734375 + }, + { + "layer": "layer.51.self_attn.v_proj", + "ppl": 4029.779296875 + }, + { + "layer": "layer.51.self_attn.o_proj", + "ppl": 3686.321044921875 + }, + { + "layer": "layer.51.mlp.gate_proj", + "ppl": 3452.50537109375 + }, + { + "layer": "layer.51.mlp.up_proj", + "ppl": 3793.723388671875 + }, + { + "layer": "layer.51.mlp.down_proj", + "ppl": 3714.078369140625 + }, + { + "layer": "layer.52.self_attn.q_proj", + "ppl": 4027.643310546875 + }, + { + "layer": "layer.52.self_attn.k_proj", + "ppl": 4069.822265625 + }, + { + "layer": "layer.52.self_attn.v_proj", + "ppl": 4266.93994140625 + }, + { + "layer": "layer.52.self_attn.o_proj", + "ppl": 3937.50732421875 + }, + { + "layer": "layer.52.mlp.gate_proj", + "ppl": 3911.555419921875 + }, + { + "layer": "layer.52.mlp.up_proj", + "ppl": 3654.45751953125 + }, + { + "layer": "layer.52.mlp.down_proj", + "ppl": 3883.973876953125 + }, + { + "layer": "layer.53.self_attn.q_proj", + "ppl": 4046.3388671875 + }, + { + "layer": "layer.53.self_attn.k_proj", + "ppl": 3981.3916015625 + }, + { + "layer": "layer.53.self_attn.v_proj", + "ppl": 4051.122802734375 + }, + { + "layer": "layer.53.self_attn.o_proj", + "ppl": 4007.85302734375 + }, + { + "layer": "layer.53.mlp.gate_proj", + "ppl": 3522.5107421875 + }, + { + "layer": "layer.53.mlp.up_proj", + "ppl": 3782.448486328125 + }, + { + "layer": "layer.53.mlp.down_proj", + "ppl": 3772.832763671875 + }, + { + "layer": "layer.54.self_attn.q_proj", + "ppl": 4011.451416015625 + }, + { + "layer": "layer.54.self_attn.k_proj", + "ppl": 4033.359130859375 + }, + { + "layer": "layer.54.self_attn.v_proj", + "ppl": 3768.9130859375 + }, + { + "layer": "layer.54.self_attn.o_proj", + "ppl": 3954.336181640625 + }, + { + "layer": "layer.54.mlp.gate_proj", + "ppl": 3857.138916015625 + }, + { + "layer": "layer.54.mlp.up_proj", + "ppl": 3958.048583984375 + }, + { + "layer": "layer.54.mlp.down_proj", + "ppl": 4152.57470703125 + }, + { + "layer": "layer.55.self_attn.q_proj", + "ppl": 4054.5859375 + }, + { + "layer": "layer.55.self_attn.k_proj", + "ppl": 4031.467041015625 + }, + { + "layer": "layer.55.self_attn.v_proj", + "ppl": 4144.08056640625 + }, + { + "layer": "layer.55.self_attn.o_proj", + "ppl": 3843.255615234375 + }, + { + "layer": "layer.55.mlp.gate_proj", + "ppl": 3973.608154296875 + }, + { + "layer": "layer.55.mlp.up_proj", + "ppl": 3997.9736328125 + }, + { + "layer": "layer.55.mlp.down_proj", + "ppl": 3950.574462890625 + }, + { + "layer": "layer.56.self_attn.q_proj", + "ppl": 4023.520263671875 + }, + { + "layer": "layer.56.self_attn.k_proj", + "ppl": 4055.807861328125 + }, + { + "layer": "layer.56.self_attn.v_proj", + "ppl": 3972.7705078125 + }, + { + "layer": "layer.56.self_attn.o_proj", + "ppl": 3947.501220703125 + }, + { + "layer": "layer.56.mlp.gate_proj", + "ppl": 3970.668701171875 + }, + { + "layer": "layer.56.mlp.up_proj", + "ppl": 3935.735595703125 + }, + { + "layer": "layer.56.mlp.down_proj", + "ppl": 4007.937255859375 + }, + { + "layer": "layer.57.self_attn.q_proj", + "ppl": 4091.1787109375 + }, + { + "layer": "layer.57.self_attn.k_proj", + "ppl": 4065.718017578125 + }, + { + "layer": "layer.57.self_attn.v_proj", + "ppl": 3900.01220703125 + }, + { + "layer": "layer.57.self_attn.o_proj", + "ppl": 3646.4296875 + }, + { + "layer": "layer.57.mlp.gate_proj", + "ppl": 4079.08984375 + }, + { + "layer": "layer.57.mlp.up_proj", + "ppl": 3773.84375 + }, + { + "layer": "layer.57.mlp.down_proj", + "ppl": 3962.376953125 + }, + { + "layer": "layer.58.self_attn.q_proj", + "ppl": 4050.00244140625 + }, + { + "layer": "layer.58.self_attn.k_proj", + "ppl": 4044.07421875 + }, + { + "layer": "layer.58.self_attn.v_proj", + "ppl": 4001.772705078125 + }, + { + "layer": "layer.58.self_attn.o_proj", + "ppl": 3688.3115234375 + }, + { + "layer": "layer.58.mlp.gate_proj", + "ppl": 3872.035888671875 + }, + { + "layer": "layer.58.mlp.up_proj", + "ppl": 3981.4375 + }, + { + "layer": "layer.58.mlp.down_proj", + "ppl": 3963.321533203125 + }, + { + "layer": "layer.59.self_attn.q_proj", + "ppl": 4022.6259765625 + }, + { + "layer": "layer.59.self_attn.k_proj", + "ppl": 4015.722900390625 + }, + { + "layer": "layer.59.self_attn.v_proj", + "ppl": 4012.721923828125 + }, + { + "layer": "layer.59.self_attn.o_proj", + "ppl": 3725.04296875 + }, + { + "layer": "layer.59.mlp.gate_proj", + "ppl": 3900.640625 + }, + { + "layer": "layer.59.mlp.up_proj", + "ppl": 3974.540283203125 + }, + { + "layer": "layer.59.mlp.down_proj", + "ppl": 3912.428466796875 + }, + { + "layer": "layer.60.self_attn.q_proj", + "ppl": 4043.677001953125 + }, + { + "layer": "layer.60.self_attn.k_proj", + "ppl": 4036.768310546875 + }, + { + "layer": "layer.60.self_attn.v_proj", + "ppl": 4110.85107421875 + }, + { + "layer": "layer.60.self_attn.o_proj", + "ppl": 3840.05712890625 + }, + { + "layer": "layer.60.mlp.gate_proj", + "ppl": 3906.097900390625 + }, + { + "layer": "layer.60.mlp.up_proj", + "ppl": 4127.63720703125 + }, + { + "layer": "layer.60.mlp.down_proj", + "ppl": 3959.815673828125 + }, + { + "layer": "layer.61.self_attn.q_proj", + "ppl": 4049.47705078125 + }, + { + "layer": "layer.61.self_attn.k_proj", + "ppl": 4043.623046875 + }, + { + "layer": "layer.61.self_attn.v_proj", + "ppl": 3887.34619140625 + }, + { + "layer": "layer.61.self_attn.o_proj", + "ppl": 3705.283447265625 + }, + { + "layer": "layer.61.mlp.gate_proj", + "ppl": 3892.4765625 + }, + { + "layer": "layer.61.mlp.up_proj", + "ppl": 3927.2841796875 + }, + { + "layer": "layer.61.mlp.down_proj", + "ppl": 3878.680419921875 + }, + { + "layer": "layer.62.self_attn.q_proj", + "ppl": 4001.666015625 + }, + { + "layer": "layer.62.self_attn.k_proj", + "ppl": 4031.793701171875 + }, + { + "layer": "layer.62.self_attn.v_proj", + "ppl": 4055.6611328125 + }, + { + "layer": "layer.62.self_attn.o_proj", + "ppl": 3947.6669921875 + }, + { + "layer": "layer.62.mlp.gate_proj", + "ppl": 4070.121337890625 + }, + { + "layer": "layer.62.mlp.up_proj", + "ppl": 3985.034912109375 + }, + { + "layer": "layer.62.mlp.down_proj", + "ppl": 3968.359619140625 + }, + { + "layer": "layer.63.self_attn.q_proj", + "ppl": 4004.116455078125 + }, + { + "layer": "layer.63.self_attn.k_proj", + "ppl": 3977.15673828125 + }, + { + "layer": "layer.63.self_attn.v_proj", + "ppl": 4070.462890625 + }, + { + "layer": "layer.63.self_attn.o_proj", + "ppl": 3775.276611328125 + }, + { + "layer": "layer.63.mlp.gate_proj", + "ppl": 3925.846435546875 + }, + { + "layer": "layer.63.mlp.up_proj", + "ppl": 4040.92822265625 + }, + { + "layer": "layer.63.mlp.down_proj", + "ppl": 3943.4638671875 + }, + { + "layer": "layer.64.self_attn.q_proj", + "ppl": 4032.8359375 + }, + { + "layer": "layer.64.self_attn.k_proj", + "ppl": 4036.3681640625 + }, + { + "layer": "layer.64.self_attn.v_proj", + "ppl": 4061.19970703125 + }, + { + "layer": "layer.64.self_attn.o_proj", + "ppl": 4083.550537109375 + }, + { + "layer": "layer.64.mlp.gate_proj", + "ppl": 3919.135498046875 + }, + { + "layer": "layer.64.mlp.up_proj", + "ppl": 4014.394287109375 + }, + { + "layer": "layer.64.mlp.down_proj", + "ppl": 4090.90185546875 + }, + { + "layer": "layer.65.self_attn.q_proj", + "ppl": 4051.207763671875 + }, + { + "layer": "layer.65.self_attn.k_proj", + "ppl": 4024.27587890625 + }, + { + "layer": "layer.65.self_attn.v_proj", + "ppl": 4094.94970703125 + }, + { + "layer": "layer.65.self_attn.o_proj", + "ppl": 3905.2265625 + }, + { + "layer": "layer.65.mlp.gate_proj", + "ppl": 3976.640869140625 + }, + { + "layer": "layer.65.mlp.up_proj", + "ppl": 4039.522216796875 + }, + { + "layer": "layer.65.mlp.down_proj", + "ppl": 4090.3984375 + }, + { + "layer": "layer.66.self_attn.q_proj", + "ppl": 4038.312744140625 + }, + { + "layer": "layer.66.self_attn.k_proj", + "ppl": 4053.1904296875 + }, + { + "layer": "layer.66.self_attn.v_proj", + "ppl": 4086.7529296875 + }, + { + "layer": "layer.66.self_attn.o_proj", + "ppl": 4123.79736328125 + }, + { + "layer": "layer.66.mlp.gate_proj", + "ppl": 3988.71142578125 + }, + { + "layer": "layer.66.mlp.up_proj", + "ppl": 3988.156005859375 + }, + { + "layer": "layer.66.mlp.down_proj", + "ppl": 3935.352783203125 + }, + { + "layer": "layer.67.self_attn.q_proj", + "ppl": 4045.8564453125 + }, + { + "layer": "layer.67.self_attn.k_proj", + "ppl": 4023.19775390625 + }, + { + "layer": "layer.67.self_attn.v_proj", + "ppl": 4181.0908203125 + }, + { + "layer": "layer.67.self_attn.o_proj", + "ppl": 4000.132080078125 + }, + { + "layer": "layer.67.mlp.gate_proj", + "ppl": 3993.572021484375 + }, + { + "layer": "layer.67.mlp.up_proj", + "ppl": 4057.993896484375 + }, + { + "layer": "layer.67.mlp.down_proj", + "ppl": 4078.930419921875 + }, + { + "layer": "layer.68.self_attn.q_proj", + "ppl": 4028.4921875 + }, + { + "layer": "layer.68.self_attn.k_proj", + "ppl": 4056.79833984375 + }, + { + "layer": "layer.68.self_attn.v_proj", + "ppl": 4006.44677734375 + }, + { + "layer": "layer.68.self_attn.o_proj", + "ppl": 4176.08154296875 + }, + { + "layer": "layer.68.mlp.gate_proj", + "ppl": 3972.695068359375 + }, + { + "layer": "layer.68.mlp.up_proj", + "ppl": 4014.685302734375 + }, + { + "layer": "layer.68.mlp.down_proj", + "ppl": 4018.945068359375 + }, + { + "layer": "layer.69.self_attn.q_proj", + "ppl": 4037.218994140625 + }, + { + "layer": "layer.69.self_attn.k_proj", + "ppl": 4055.281982421875 + }, + { + "layer": "layer.69.self_attn.v_proj", + "ppl": 4064.601318359375 + }, + { + "layer": "layer.69.self_attn.o_proj", + "ppl": 4151.703125 + }, + { + "layer": "layer.69.mlp.gate_proj", + "ppl": 4011.275390625 + }, + { + "layer": "layer.69.mlp.up_proj", + "ppl": 4018.5732421875 + }, + { + "layer": "layer.69.mlp.down_proj", + "ppl": 4163.654296875 + }, + { + "layer": "layer.70.self_attn.q_proj", + "ppl": 4043.3955078125 + }, + { + "layer": "layer.70.self_attn.k_proj", + "ppl": 4046.44677734375 + }, + { + "layer": "layer.70.self_attn.v_proj", + "ppl": 4084.60205078125 + }, + { + "layer": "layer.70.self_attn.o_proj", + "ppl": 4034.928955078125 + }, + { + "layer": "layer.70.mlp.gate_proj", + "ppl": 3888.743896484375 + }, + { + "layer": "layer.70.mlp.up_proj", + "ppl": 4056.6513671875 + }, + { + "layer": "layer.70.mlp.down_proj", + "ppl": 4094.859619140625 + }, + { + "layer": "layer.71.self_attn.q_proj", + "ppl": 4042.55517578125 + }, + { + "layer": "layer.71.self_attn.k_proj", + "ppl": 4034.190185546875 + }, + { + "layer": "layer.71.self_attn.v_proj", + "ppl": 4093.3681640625 + }, + { + "layer": "layer.71.self_attn.o_proj", + "ppl": 3984.848388671875 + }, + { + "layer": "layer.71.mlp.gate_proj", + "ppl": 3956.30908203125 + }, + { + "layer": "layer.71.mlp.up_proj", + "ppl": 4130.9609375 + }, + { + "layer": "layer.71.mlp.down_proj", + "ppl": 4191.10302734375 + }, + { + "layer": "layer.72.self_attn.q_proj", + "ppl": 4037.5 + }, + { + "layer": "layer.72.self_attn.k_proj", + "ppl": 4006.7294921875 + }, + { + "layer": "layer.72.self_attn.v_proj", + "ppl": 4066.63720703125 + }, + { + "layer": "layer.72.self_attn.o_proj", + "ppl": 3990.438720703125 + }, + { + "layer": "layer.72.mlp.gate_proj", + "ppl": 3950.9287109375 + }, + { + "layer": "layer.72.mlp.up_proj", + "ppl": 4124.32421875 + }, + { + "layer": "layer.72.mlp.down_proj", + "ppl": 3947.07958984375 + }, + { + "layer": "layer.73.self_attn.q_proj", + "ppl": 4035.463623046875 + }, + { + "layer": "layer.73.self_attn.k_proj", + "ppl": 4058.469970703125 + }, + { + "layer": "layer.73.self_attn.v_proj", + "ppl": 4061.025146484375 + }, + { + "layer": "layer.73.self_attn.o_proj", + "ppl": 4060.061279296875 + }, + { + "layer": "layer.73.mlp.gate_proj", + "ppl": 3908.758544921875 + }, + { + "layer": "layer.73.mlp.up_proj", + "ppl": 4099.43115234375 + }, + { + "layer": "layer.73.mlp.down_proj", + "ppl": 4002.490478515625 + }, + { + "layer": "layer.74.self_attn.q_proj", + "ppl": 4057.564208984375 + }, + { + "layer": "layer.74.self_attn.k_proj", + "ppl": 4062.70654296875 + }, + { + "layer": "layer.74.self_attn.v_proj", + "ppl": 3973.562744140625 + }, + { + "layer": "layer.74.self_attn.o_proj", + "ppl": 3869.876220703125 + }, + { + "layer": "layer.74.mlp.gate_proj", + "ppl": 3860.0458984375 + }, + { + "layer": "layer.74.mlp.up_proj", + "ppl": 4180.34521484375 + }, + { + "layer": "layer.74.mlp.down_proj", + "ppl": 3962.486572265625 + }, + { + "layer": "layer.75.self_attn.q_proj", + "ppl": 4031.60546875 + }, + { + "layer": "layer.75.self_attn.k_proj", + "ppl": 4037.711669921875 + }, + { + "layer": "layer.75.self_attn.v_proj", + "ppl": 4165.8349609375 + }, + { + "layer": "layer.75.self_attn.o_proj", + "ppl": 3975.465576171875 + }, + { + "layer": "layer.75.mlp.gate_proj", + "ppl": 3918.1376953125 + }, + { + "layer": "layer.75.mlp.up_proj", + "ppl": 3905.800048828125 + }, + { + "layer": "layer.75.mlp.down_proj", + "ppl": 4217.90380859375 + }, + { + "layer": "layer.76.self_attn.q_proj", + "ppl": 4027.585693359375 + }, + { + "layer": "layer.76.self_attn.k_proj", + "ppl": 4054.694091796875 + }, + { + "layer": "layer.76.self_attn.v_proj", + "ppl": 3893.389892578125 + }, + { + "layer": "layer.76.self_attn.o_proj", + "ppl": 3996.471435546875 + }, + { + "layer": "layer.76.mlp.gate_proj", + "ppl": 4049.3076171875 + }, + { + "layer": "layer.76.mlp.up_proj", + "ppl": 4278.31640625 + }, + { + "layer": "layer.76.mlp.down_proj", + "ppl": 3947.546142578125 + }, + { + "layer": "layer.77.self_attn.q_proj", + "ppl": 4074.494384765625 + }, + { + "layer": "layer.77.self_attn.k_proj", + "ppl": 3993.119140625 + }, + { + "layer": "layer.77.self_attn.v_proj", + "ppl": 3932.546142578125 + }, + { + "layer": "layer.77.self_attn.o_proj", + "ppl": 3823.48486328125 + }, + { + "layer": "layer.77.mlp.gate_proj", + "ppl": 4204.4501953125 + }, + { + "layer": "layer.77.mlp.up_proj", + "ppl": 4197.634765625 + }, + { + "layer": "layer.77.mlp.down_proj", + "ppl": 3226.13330078125 + }, + { + "layer": "layer.78.self_attn.q_proj", + "ppl": 4057.21240234375 + }, + { + "layer": "layer.78.self_attn.k_proj", + "ppl": 4028.692138671875 + }, + { + "layer": "layer.78.self_attn.v_proj", + "ppl": 4204.09716796875 + }, + { + "layer": "layer.78.self_attn.o_proj", + "ppl": 4955.37109375 + }, + { + "layer": "layer.78.mlp.gate_proj", + "ppl": 3842.6435546875 + }, + { + "layer": "layer.78.mlp.up_proj", + "ppl": 3744.383056640625 + }, + { + "layer": "layer.78.mlp.down_proj", + "ppl": 2970.520263671875 + }, + { + "layer": "layer.79.self_attn.q_proj", + "ppl": 4048.32666015625 + }, + { + "layer": "layer.79.self_attn.k_proj", + "ppl": 4044.7109375 + }, + { + "layer": "layer.79.self_attn.v_proj", + "ppl": 4306.5205078125 + }, + { + "layer": "layer.79.self_attn.o_proj", + "ppl": 4341.7587890625 + }, + { + "layer": "layer.79.mlp.gate_proj", + "ppl": 3362.815673828125 + }, + { + "layer": "layer.79.mlp.up_proj", + "ppl": 4342.87255859375 + }, + { + "layer": "layer.79.mlp.down_proj", + "ppl": 4722.39599609375 + } + ] + }, + { + "bit": 2, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 8.43564224243164 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 8.437524795532227 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 8.43498706817627 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 8.427589416503906 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 8.445396423339844 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 8.446619987487793 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 8.43922233581543 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 8.436372756958008 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 8.438374519348145 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 8.436844825744629 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 8.425411224365234 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 8.44064712524414 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 8.443014144897461 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 8.429457664489746 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 8.42896556854248 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 8.431224822998047 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 8.429108619689941 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 8.425543785095215 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 8.432868957519531 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 8.433671951293945 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 8.42186450958252 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 8.426974296569824 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 8.432634353637695 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 8.421029090881348 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 8.420162200927734 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 8.427379608154297 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 8.428032875061035 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 8.424278259277344 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 8.425654411315918 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 8.429655075073242 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 8.42428207397461 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 8.426841735839844 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 8.419339179992676 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 8.42158317565918 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 8.41919994354248 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 8.43621826171875 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 8.438799858093262 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 8.435945510864258 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 8.437633514404297 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 8.41896915435791 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 8.42092514038086 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 8.423081398010254 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 8.435125350952148 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 8.435023307800293 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 8.4324951171875 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 8.434490203857422 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 8.42373275756836 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 8.428262710571289 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 8.423206329345703 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 8.439522743225098 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 8.437919616699219 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 8.438152313232422 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 8.433655738830566 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 8.425433158874512 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 8.425338745117188 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 8.429433822631836 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 8.437040328979492 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 8.43490219116211 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 8.434725761413574 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 8.434242248535156 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 8.428468704223633 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 8.430088996887207 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 8.426487922668457 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 8.433794021606445 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 8.435678482055664 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 8.43436336517334 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 8.433426856994629 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 8.43116283416748 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 8.430636405944824 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 8.430039405822754 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 8.436454772949219 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 8.436762809753418 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 8.436182022094727 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 8.436012268066406 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 8.428727149963379 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 8.43027400970459 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 8.429945945739746 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 8.434135437011719 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 8.434330940246582 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 8.434443473815918 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 8.436481475830078 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 8.428342819213867 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 8.43081283569336 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 8.42811393737793 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 8.441267967224121 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 8.441010475158691 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 8.435694694519043 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 8.439888954162598 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 8.435272216796875 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 8.433072090148926 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 8.43282699584961 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 8.438908576965332 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 8.437477111816406 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 8.437593460083008 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 8.437588691711426 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 8.434142112731934 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 8.435012817382812 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 8.435725212097168 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 8.436697959899902 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 8.436084747314453 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 8.436195373535156 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 8.43680477142334 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 8.43702220916748 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 8.435705184936523 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 8.435388565063477 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 8.438748359680176 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 8.438558578491211 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 8.438247680664062 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 8.43887710571289 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 8.43350887298584 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 8.434762001037598 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 8.433570861816406 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 8.437925338745117 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 8.437990188598633 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 8.436911582946777 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 8.436458587646484 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 8.438631057739258 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 8.438324928283691 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 8.435429573059082 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 8.431514739990234 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 8.432132720947266 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 8.43448543548584 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 8.432870864868164 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 8.436659812927246 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 8.434829711914062 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 8.436728477478027 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 8.441239356994629 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 8.440478324890137 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 8.437847137451172 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 8.435585975646973 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 8.438322067260742 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 8.434710502624512 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 8.438307762145996 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 8.440774917602539 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 8.441875457763672 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 8.435638427734375 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 8.436945915222168 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 8.433937072753906 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 8.436145782470703 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 8.433124542236328 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 8.436080932617188 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 8.435593605041504 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 8.435093879699707 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 8.43710708618164 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 8.435489654541016 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 8.435393333435059 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 8.431818008422852 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 8.441170692443848 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 8.44063949584961 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 8.439764976501465 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 8.435338973999023 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 8.436933517456055 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 8.434887886047363 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 8.436017990112305 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 8.441094398498535 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 8.439885139465332 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 8.432727813720703 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 8.42893123626709 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 8.433084487915039 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 8.433737754821777 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 8.428447723388672 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 8.434825897216797 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 8.434234619140625 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 8.434174537658691 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 8.430766105651855 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 8.434310913085938 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 8.435553550720215 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 8.422091484069824 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 8.436507225036621 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 8.436579704284668 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 8.433835983276367 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 8.437484741210938 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 8.427059173583984 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 8.434847831726074 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 8.42279052734375 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 8.441654205322266 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 8.438356399536133 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 8.433293342590332 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 8.435223579406738 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 8.429208755493164 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 8.42949390411377 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 8.42726993560791 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 8.43423080444336 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 8.434049606323242 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 8.435521125793457 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 8.434357643127441 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 8.424609184265137 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 8.423760414123535 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 8.42556381225586 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 8.436766624450684 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 8.435264587402344 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 8.435279846191406 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 8.4384126663208 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 8.424924850463867 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 8.42708683013916 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 8.424490928649902 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 8.437066078186035 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 8.436057090759277 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 8.434680938720703 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 8.434684753417969 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 8.428306579589844 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 8.430377960205078 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 8.426164627075195 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 8.434882164001465 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 8.435696601867676 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 8.430434226989746 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 8.430800437927246 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 8.4283447265625 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 8.437504768371582 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 8.43061351776123 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 8.437710762023926 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 8.436641693115234 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 8.436982154846191 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 8.43384838104248 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 8.42482852935791 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 8.426912307739258 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 8.420632362365723 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 8.433625221252441 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 8.434727668762207 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 8.433165550231934 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 8.436358451843262 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 8.430925369262695 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 8.425955772399902 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 8.43330192565918 + }, + { + "layer": "layer.32.self_attn.q_proj", + "ppl": 8.436102867126465 + }, + { + "layer": "layer.32.self_attn.k_proj", + "ppl": 8.437047958374023 + }, + { + "layer": "layer.32.self_attn.v_proj", + "ppl": 8.43574333190918 + }, + { + "layer": "layer.32.self_attn.o_proj", + "ppl": 8.434962272644043 + }, + { + "layer": "layer.32.mlp.gate_proj", + "ppl": 8.430072784423828 + }, + { + "layer": "layer.32.mlp.up_proj", + "ppl": 8.430746078491211 + }, + { + "layer": "layer.32.mlp.down_proj", + "ppl": 8.428507804870605 + }, + { + "layer": "layer.33.self_attn.q_proj", + "ppl": 8.437520980834961 + }, + { + "layer": "layer.33.self_attn.k_proj", + "ppl": 8.436543464660645 + }, + { + "layer": "layer.33.self_attn.v_proj", + "ppl": 8.437517166137695 + }, + { + "layer": "layer.33.self_attn.o_proj", + "ppl": 8.435332298278809 + }, + { + "layer": "layer.33.mlp.gate_proj", + "ppl": 8.427555084228516 + }, + { + "layer": "layer.33.mlp.up_proj", + "ppl": 8.426220893859863 + }, + { + "layer": "layer.33.mlp.down_proj", + "ppl": 8.423362731933594 + }, + { + "layer": "layer.34.self_attn.q_proj", + "ppl": 8.43657112121582 + }, + { + "layer": "layer.34.self_attn.k_proj", + "ppl": 8.437261581420898 + }, + { + "layer": "layer.34.self_attn.v_proj", + "ppl": 8.434369087219238 + }, + { + "layer": "layer.34.self_attn.o_proj", + "ppl": 8.435103416442871 + }, + { + "layer": "layer.34.mlp.gate_proj", + "ppl": 8.428606033325195 + }, + { + "layer": "layer.34.mlp.up_proj", + "ppl": 8.429444313049316 + }, + { + "layer": "layer.34.mlp.down_proj", + "ppl": 8.422531127929688 + }, + { + "layer": "layer.35.self_attn.q_proj", + "ppl": 8.43641471862793 + }, + { + "layer": "layer.35.self_attn.k_proj", + "ppl": 8.437368392944336 + }, + { + "layer": "layer.35.self_attn.v_proj", + "ppl": 8.434858322143555 + }, + { + "layer": "layer.35.self_attn.o_proj", + "ppl": 8.43416690826416 + }, + { + "layer": "layer.35.mlp.gate_proj", + "ppl": 8.42488956451416 + }, + { + "layer": "layer.35.mlp.up_proj", + "ppl": 8.432917594909668 + }, + { + "layer": "layer.35.mlp.down_proj", + "ppl": 8.433645248413086 + }, + { + "layer": "layer.36.self_attn.q_proj", + "ppl": 8.437766075134277 + }, + { + "layer": "layer.36.self_attn.k_proj", + "ppl": 8.437522888183594 + }, + { + "layer": "layer.36.self_attn.v_proj", + "ppl": 8.434865951538086 + }, + { + "layer": "layer.36.self_attn.o_proj", + "ppl": 8.436670303344727 + }, + { + "layer": "layer.36.mlp.gate_proj", + "ppl": 8.420511245727539 + }, + { + "layer": "layer.36.mlp.up_proj", + "ppl": 8.4296875 + }, + { + "layer": "layer.36.mlp.down_proj", + "ppl": 8.431825637817383 + }, + { + "layer": "layer.37.self_attn.q_proj", + "ppl": 8.437307357788086 + }, + { + "layer": "layer.37.self_attn.k_proj", + "ppl": 8.436229705810547 + }, + { + "layer": "layer.37.self_attn.v_proj", + "ppl": 8.436366081237793 + }, + { + "layer": "layer.37.self_attn.o_proj", + "ppl": 8.430000305175781 + }, + { + "layer": "layer.37.mlp.gate_proj", + "ppl": 8.421354293823242 + }, + { + "layer": "layer.37.mlp.up_proj", + "ppl": 8.427504539489746 + }, + { + "layer": "layer.37.mlp.down_proj", + "ppl": 8.428503036499023 + }, + { + "layer": "layer.38.self_attn.q_proj", + "ppl": 8.43630599975586 + }, + { + "layer": "layer.38.self_attn.k_proj", + "ppl": 8.437217712402344 + }, + { + "layer": "layer.38.self_attn.v_proj", + "ppl": 8.437470436096191 + }, + { + "layer": "layer.38.self_attn.o_proj", + "ppl": 8.434889793395996 + }, + { + "layer": "layer.38.mlp.gate_proj", + "ppl": 8.420053482055664 + }, + { + "layer": "layer.38.mlp.up_proj", + "ppl": 8.433385848999023 + }, + { + "layer": "layer.38.mlp.down_proj", + "ppl": 8.424708366394043 + }, + { + "layer": "layer.39.self_attn.q_proj", + "ppl": 8.437549591064453 + }, + { + "layer": "layer.39.self_attn.k_proj", + "ppl": 8.4358549118042 + }, + { + "layer": "layer.39.self_attn.v_proj", + "ppl": 8.433711051940918 + }, + { + "layer": "layer.39.self_attn.o_proj", + "ppl": 8.434562683105469 + }, + { + "layer": "layer.39.mlp.gate_proj", + "ppl": 8.42741584777832 + }, + { + "layer": "layer.39.mlp.up_proj", + "ppl": 8.432836532592773 + }, + { + "layer": "layer.39.mlp.down_proj", + "ppl": 8.426741600036621 + }, + { + "layer": "layer.40.self_attn.q_proj", + "ppl": 8.43327522277832 + }, + { + "layer": "layer.40.self_attn.k_proj", + "ppl": 8.435662269592285 + }, + { + "layer": "layer.40.self_attn.v_proj", + "ppl": 8.436982154846191 + }, + { + "layer": "layer.40.self_attn.o_proj", + "ppl": 8.43522834777832 + }, + { + "layer": "layer.40.mlp.gate_proj", + "ppl": 8.42845344543457 + }, + { + "layer": "layer.40.mlp.up_proj", + "ppl": 8.430708885192871 + }, + { + "layer": "layer.40.mlp.down_proj", + "ppl": 8.426847457885742 + }, + { + "layer": "layer.41.self_attn.q_proj", + "ppl": 8.440418243408203 + }, + { + "layer": "layer.41.self_attn.k_proj", + "ppl": 8.438772201538086 + }, + { + "layer": "layer.41.self_attn.v_proj", + "ppl": 8.435318946838379 + }, + { + "layer": "layer.41.self_attn.o_proj", + "ppl": 8.43427848815918 + }, + { + "layer": "layer.41.mlp.gate_proj", + "ppl": 8.424784660339355 + }, + { + "layer": "layer.41.mlp.up_proj", + "ppl": 8.428284645080566 + }, + { + "layer": "layer.41.mlp.down_proj", + "ppl": 8.42488956451416 + }, + { + "layer": "layer.42.self_attn.q_proj", + "ppl": 8.438796997070312 + }, + { + "layer": "layer.42.self_attn.k_proj", + "ppl": 8.438762664794922 + }, + { + "layer": "layer.42.self_attn.v_proj", + "ppl": 8.437187194824219 + }, + { + "layer": "layer.42.self_attn.o_proj", + "ppl": 8.434998512268066 + }, + { + "layer": "layer.42.mlp.gate_proj", + "ppl": 8.420053482055664 + }, + { + "layer": "layer.42.mlp.up_proj", + "ppl": 8.42577075958252 + }, + { + "layer": "layer.42.mlp.down_proj", + "ppl": 8.424676895141602 + }, + { + "layer": "layer.43.self_attn.q_proj", + "ppl": 8.436668395996094 + }, + { + "layer": "layer.43.self_attn.k_proj", + "ppl": 8.432026863098145 + }, + { + "layer": "layer.43.self_attn.v_proj", + "ppl": 8.4293851852417 + }, + { + "layer": "layer.43.self_attn.o_proj", + "ppl": 8.434035301208496 + }, + { + "layer": "layer.43.mlp.gate_proj", + "ppl": 8.420001983642578 + }, + { + "layer": "layer.43.mlp.up_proj", + "ppl": 8.4243745803833 + }, + { + "layer": "layer.43.mlp.down_proj", + "ppl": 8.423195838928223 + }, + { + "layer": "layer.44.self_attn.q_proj", + "ppl": 8.441418647766113 + }, + { + "layer": "layer.44.self_attn.k_proj", + "ppl": 8.439558982849121 + }, + { + "layer": "layer.44.self_attn.v_proj", + "ppl": 8.436285972595215 + }, + { + "layer": "layer.44.self_attn.o_proj", + "ppl": 8.43936824798584 + }, + { + "layer": "layer.44.mlp.gate_proj", + "ppl": 8.42698860168457 + }, + { + "layer": "layer.44.mlp.up_proj", + "ppl": 8.428807258605957 + }, + { + "layer": "layer.44.mlp.down_proj", + "ppl": 8.422372817993164 + }, + { + "layer": "layer.45.self_attn.q_proj", + "ppl": 8.442127227783203 + }, + { + "layer": "layer.45.self_attn.k_proj", + "ppl": 8.43857192993164 + }, + { + "layer": "layer.45.self_attn.v_proj", + "ppl": 8.441853523254395 + }, + { + "layer": "layer.45.self_attn.o_proj", + "ppl": 8.438545227050781 + }, + { + "layer": "layer.45.mlp.gate_proj", + "ppl": 8.409388542175293 + }, + { + "layer": "layer.45.mlp.up_proj", + "ppl": 8.39021110534668 + }, + { + "layer": "layer.45.mlp.down_proj", + "ppl": 8.43354320526123 + }, + { + "layer": "layer.46.self_attn.q_proj", + "ppl": 8.43249797821045 + }, + { + "layer": "layer.46.self_attn.k_proj", + "ppl": 8.43188762664795 + }, + { + "layer": "layer.46.self_attn.v_proj", + "ppl": 8.439230918884277 + }, + { + "layer": "layer.46.self_attn.o_proj", + "ppl": 8.437259674072266 + }, + { + "layer": "layer.46.mlp.gate_proj", + "ppl": 8.428834915161133 + }, + { + "layer": "layer.46.mlp.up_proj", + "ppl": 8.427484512329102 + }, + { + "layer": "layer.46.mlp.down_proj", + "ppl": 8.4328031539917 + }, + { + "layer": "layer.47.self_attn.q_proj", + "ppl": 8.43511962890625 + }, + { + "layer": "layer.47.self_attn.k_proj", + "ppl": 8.435903549194336 + }, + { + "layer": "layer.47.self_attn.v_proj", + "ppl": 8.439955711364746 + }, + { + "layer": "layer.47.self_attn.o_proj", + "ppl": 8.437845230102539 + }, + { + "layer": "layer.47.mlp.gate_proj", + "ppl": 8.428847312927246 + }, + { + "layer": "layer.47.mlp.up_proj", + "ppl": 8.433257102966309 + }, + { + "layer": "layer.47.mlp.down_proj", + "ppl": 8.432239532470703 + }, + { + "layer": "layer.48.self_attn.q_proj", + "ppl": 8.441060066223145 + }, + { + "layer": "layer.48.self_attn.k_proj", + "ppl": 8.43871021270752 + }, + { + "layer": "layer.48.self_attn.v_proj", + "ppl": 8.441734313964844 + }, + { + "layer": "layer.48.self_attn.o_proj", + "ppl": 8.44025707244873 + }, + { + "layer": "layer.48.mlp.gate_proj", + "ppl": 8.418546676635742 + }, + { + "layer": "layer.48.mlp.up_proj", + "ppl": 8.424736022949219 + }, + { + "layer": "layer.48.mlp.down_proj", + "ppl": 8.433752059936523 + }, + { + "layer": "layer.49.self_attn.q_proj", + "ppl": 8.438011169433594 + }, + { + "layer": "layer.49.self_attn.k_proj", + "ppl": 8.436895370483398 + }, + { + "layer": "layer.49.self_attn.v_proj", + "ppl": 8.434091567993164 + }, + { + "layer": "layer.49.self_attn.o_proj", + "ppl": 8.431085586547852 + }, + { + "layer": "layer.49.mlp.gate_proj", + "ppl": 8.424025535583496 + }, + { + "layer": "layer.49.mlp.up_proj", + "ppl": 8.4315767288208 + }, + { + "layer": "layer.49.mlp.down_proj", + "ppl": 8.43051528930664 + }, + { + "layer": "layer.50.self_attn.q_proj", + "ppl": 8.436450958251953 + }, + { + "layer": "layer.50.self_attn.k_proj", + "ppl": 8.433028221130371 + }, + { + "layer": "layer.50.self_attn.v_proj", + "ppl": 8.43865966796875 + }, + { + "layer": "layer.50.self_attn.o_proj", + "ppl": 8.43064022064209 + }, + { + "layer": "layer.50.mlp.gate_proj", + "ppl": 8.447053909301758 + }, + { + "layer": "layer.50.mlp.up_proj", + "ppl": 8.436088562011719 + }, + { + "layer": "layer.50.mlp.down_proj", + "ppl": 8.440068244934082 + }, + { + "layer": "layer.51.self_attn.q_proj", + "ppl": 8.443805694580078 + }, + { + "layer": "layer.51.self_attn.k_proj", + "ppl": 8.439712524414062 + }, + { + "layer": "layer.51.self_attn.v_proj", + "ppl": 8.438864707946777 + }, + { + "layer": "layer.51.self_attn.o_proj", + "ppl": 8.43405532836914 + }, + { + "layer": "layer.51.mlp.gate_proj", + "ppl": 8.439555168151855 + }, + { + "layer": "layer.51.mlp.up_proj", + "ppl": 8.438813209533691 + }, + { + "layer": "layer.51.mlp.down_proj", + "ppl": 8.43155288696289 + }, + { + "layer": "layer.52.self_attn.q_proj", + "ppl": 8.435537338256836 + }, + { + "layer": "layer.52.self_attn.k_proj", + "ppl": 8.437905311584473 + }, + { + "layer": "layer.52.self_attn.v_proj", + "ppl": 8.42921257019043 + }, + { + "layer": "layer.52.self_attn.o_proj", + "ppl": 8.42943000793457 + }, + { + "layer": "layer.52.mlp.gate_proj", + "ppl": 8.43620491027832 + }, + { + "layer": "layer.52.mlp.up_proj", + "ppl": 8.42921257019043 + }, + { + "layer": "layer.52.mlp.down_proj", + "ppl": 8.432991981506348 + }, + { + "layer": "layer.53.self_attn.q_proj", + "ppl": 8.439638137817383 + }, + { + "layer": "layer.53.self_attn.k_proj", + "ppl": 8.437965393066406 + }, + { + "layer": "layer.53.self_attn.v_proj", + "ppl": 8.429112434387207 + }, + { + "layer": "layer.53.self_attn.o_proj", + "ppl": 8.428083419799805 + }, + { + "layer": "layer.53.mlp.gate_proj", + "ppl": 8.443294525146484 + }, + { + "layer": "layer.53.mlp.up_proj", + "ppl": 8.449249267578125 + }, + { + "layer": "layer.53.mlp.down_proj", + "ppl": 8.429181098937988 + }, + { + "layer": "layer.54.self_attn.q_proj", + "ppl": 8.428827285766602 + }, + { + "layer": "layer.54.self_attn.k_proj", + "ppl": 8.435393333435059 + }, + { + "layer": "layer.54.self_attn.v_proj", + "ppl": 8.433772087097168 + }, + { + "layer": "layer.54.self_attn.o_proj", + "ppl": 8.427745819091797 + }, + { + "layer": "layer.54.mlp.gate_proj", + "ppl": 8.4327974319458 + }, + { + "layer": "layer.54.mlp.up_proj", + "ppl": 8.43395709991455 + }, + { + "layer": "layer.54.mlp.down_proj", + "ppl": 8.436086654663086 + }, + { + "layer": "layer.55.self_attn.q_proj", + "ppl": 8.422378540039062 + }, + { + "layer": "layer.55.self_attn.k_proj", + "ppl": 8.428742408752441 + }, + { + "layer": "layer.55.self_attn.v_proj", + "ppl": 8.430257797241211 + }, + { + "layer": "layer.55.self_attn.o_proj", + "ppl": 8.433120727539062 + }, + { + "layer": "layer.55.mlp.gate_proj", + "ppl": 8.428827285766602 + }, + { + "layer": "layer.55.mlp.up_proj", + "ppl": 8.436744689941406 + }, + { + "layer": "layer.55.mlp.down_proj", + "ppl": 8.426239013671875 + }, + { + "layer": "layer.56.self_attn.q_proj", + "ppl": 8.42364501953125 + }, + { + "layer": "layer.56.self_attn.k_proj", + "ppl": 8.427255630493164 + }, + { + "layer": "layer.56.self_attn.v_proj", + "ppl": 8.424510955810547 + }, + { + "layer": "layer.56.self_attn.o_proj", + "ppl": 8.423696517944336 + }, + { + "layer": "layer.56.mlp.gate_proj", + "ppl": 8.421873092651367 + }, + { + "layer": "layer.56.mlp.up_proj", + "ppl": 8.431665420532227 + }, + { + "layer": "layer.56.mlp.down_proj", + "ppl": 8.431578636169434 + }, + { + "layer": "layer.57.self_attn.q_proj", + "ppl": 8.433673858642578 + }, + { + "layer": "layer.57.self_attn.k_proj", + "ppl": 8.444967269897461 + }, + { + "layer": "layer.57.self_attn.v_proj", + "ppl": 8.427983283996582 + }, + { + "layer": "layer.57.self_attn.o_proj", + "ppl": 8.421324729919434 + }, + { + "layer": "layer.57.mlp.gate_proj", + "ppl": 8.436768531799316 + }, + { + "layer": "layer.57.mlp.up_proj", + "ppl": 8.439785957336426 + }, + { + "layer": "layer.57.mlp.down_proj", + "ppl": 8.434102058410645 + }, + { + "layer": "layer.58.self_attn.q_proj", + "ppl": 8.434475898742676 + }, + { + "layer": "layer.58.self_attn.k_proj", + "ppl": 8.435399055480957 + }, + { + "layer": "layer.58.self_attn.v_proj", + "ppl": 8.441018104553223 + }, + { + "layer": "layer.58.self_attn.o_proj", + "ppl": 8.419066429138184 + }, + { + "layer": "layer.58.mlp.gate_proj", + "ppl": 8.414403915405273 + }, + { + "layer": "layer.58.mlp.up_proj", + "ppl": 8.435708999633789 + }, + { + "layer": "layer.58.mlp.down_proj", + "ppl": 8.441485404968262 + }, + { + "layer": "layer.59.self_attn.q_proj", + "ppl": 8.427878379821777 + }, + { + "layer": "layer.59.self_attn.k_proj", + "ppl": 8.429011344909668 + }, + { + "layer": "layer.59.self_attn.v_proj", + "ppl": 8.425684928894043 + }, + { + "layer": "layer.59.self_attn.o_proj", + "ppl": 8.409010887145996 + }, + { + "layer": "layer.59.mlp.gate_proj", + "ppl": 8.423354148864746 + }, + { + "layer": "layer.59.mlp.up_proj", + "ppl": 8.433398246765137 + }, + { + "layer": "layer.59.mlp.down_proj", + "ppl": 8.428047180175781 + }, + { + "layer": "layer.60.self_attn.q_proj", + "ppl": 8.439579010009766 + }, + { + "layer": "layer.60.self_attn.k_proj", + "ppl": 8.432422637939453 + }, + { + "layer": "layer.60.self_attn.v_proj", + "ppl": 8.436028480529785 + }, + { + "layer": "layer.60.self_attn.o_proj", + "ppl": 8.42306900024414 + }, + { + "layer": "layer.60.mlp.gate_proj", + "ppl": 8.422463417053223 + }, + { + "layer": "layer.60.mlp.up_proj", + "ppl": 8.429208755493164 + }, + { + "layer": "layer.60.mlp.down_proj", + "ppl": 8.420985221862793 + }, + { + "layer": "layer.61.self_attn.q_proj", + "ppl": 8.438478469848633 + }, + { + "layer": "layer.61.self_attn.k_proj", + "ppl": 8.436612129211426 + }, + { + "layer": "layer.61.self_attn.v_proj", + "ppl": 8.434247016906738 + }, + { + "layer": "layer.61.self_attn.o_proj", + "ppl": 8.426660537719727 + }, + { + "layer": "layer.61.mlp.gate_proj", + "ppl": 8.417984008789062 + }, + { + "layer": "layer.61.mlp.up_proj", + "ppl": 8.438193321228027 + }, + { + "layer": "layer.61.mlp.down_proj", + "ppl": 8.437392234802246 + }, + { + "layer": "layer.62.self_attn.q_proj", + "ppl": 8.393028259277344 + }, + { + "layer": "layer.62.self_attn.k_proj", + "ppl": 8.424243927001953 + }, + { + "layer": "layer.62.self_attn.v_proj", + "ppl": 8.428461074829102 + }, + { + "layer": "layer.62.self_attn.o_proj", + "ppl": 8.420551300048828 + }, + { + "layer": "layer.62.mlp.gate_proj", + "ppl": 8.41268539428711 + }, + { + "layer": "layer.62.mlp.up_proj", + "ppl": 8.43774700164795 + }, + { + "layer": "layer.62.mlp.down_proj", + "ppl": 8.43399715423584 + }, + { + "layer": "layer.63.self_attn.q_proj", + "ppl": 8.426970481872559 + }, + { + "layer": "layer.63.self_attn.k_proj", + "ppl": 8.434990882873535 + }, + { + "layer": "layer.63.self_attn.v_proj", + "ppl": 8.429498672485352 + }, + { + "layer": "layer.63.self_attn.o_proj", + "ppl": 8.419757843017578 + }, + { + "layer": "layer.63.mlp.gate_proj", + "ppl": 8.41634464263916 + }, + { + "layer": "layer.63.mlp.up_proj", + "ppl": 8.428559303283691 + }, + { + "layer": "layer.63.mlp.down_proj", + "ppl": 8.420547485351562 + }, + { + "layer": "layer.64.self_attn.q_proj", + "ppl": 8.433804512023926 + }, + { + "layer": "layer.64.self_attn.k_proj", + "ppl": 8.427645683288574 + }, + { + "layer": "layer.64.self_attn.v_proj", + "ppl": 8.427717208862305 + }, + { + "layer": "layer.64.self_attn.o_proj", + "ppl": 8.414321899414062 + }, + { + "layer": "layer.64.mlp.gate_proj", + "ppl": 8.405219078063965 + }, + { + "layer": "layer.64.mlp.up_proj", + "ppl": 8.43010139465332 + }, + { + "layer": "layer.64.mlp.down_proj", + "ppl": 8.422719955444336 + }, + { + "layer": "layer.65.self_attn.q_proj", + "ppl": 8.426337242126465 + }, + { + "layer": "layer.65.self_attn.k_proj", + "ppl": 8.429618835449219 + }, + { + "layer": "layer.65.self_attn.v_proj", + "ppl": 8.432389259338379 + }, + { + "layer": "layer.65.self_attn.o_proj", + "ppl": 8.423429489135742 + }, + { + "layer": "layer.65.mlp.gate_proj", + "ppl": 8.405235290527344 + }, + { + "layer": "layer.65.mlp.up_proj", + "ppl": 8.435107231140137 + }, + { + "layer": "layer.65.mlp.down_proj", + "ppl": 8.420900344848633 + }, + { + "layer": "layer.66.self_attn.q_proj", + "ppl": 8.431001663208008 + }, + { + "layer": "layer.66.self_attn.k_proj", + "ppl": 8.431371688842773 + }, + { + "layer": "layer.66.self_attn.v_proj", + "ppl": 8.431331634521484 + }, + { + "layer": "layer.66.self_attn.o_proj", + "ppl": 8.430304527282715 + }, + { + "layer": "layer.66.mlp.gate_proj", + "ppl": 8.401182174682617 + }, + { + "layer": "layer.66.mlp.up_proj", + "ppl": 8.426507949829102 + }, + { + "layer": "layer.66.mlp.down_proj", + "ppl": 8.415746688842773 + }, + { + "layer": "layer.67.self_attn.q_proj", + "ppl": 8.436956405639648 + }, + { + "layer": "layer.67.self_attn.k_proj", + "ppl": 8.442124366760254 + }, + { + "layer": "layer.67.self_attn.v_proj", + "ppl": 8.445108413696289 + }, + { + "layer": "layer.67.self_attn.o_proj", + "ppl": 8.43814468383789 + }, + { + "layer": "layer.67.mlp.gate_proj", + "ppl": 8.404167175292969 + }, + { + "layer": "layer.67.mlp.up_proj", + "ppl": 8.43177604675293 + }, + { + "layer": "layer.67.mlp.down_proj", + "ppl": 8.421366691589355 + }, + { + "layer": "layer.68.self_attn.q_proj", + "ppl": 8.432328224182129 + }, + { + "layer": "layer.68.self_attn.k_proj", + "ppl": 8.433271408081055 + }, + { + "layer": "layer.68.self_attn.v_proj", + "ppl": 8.436177253723145 + }, + { + "layer": "layer.68.self_attn.o_proj", + "ppl": 8.43358325958252 + }, + { + "layer": "layer.68.mlp.gate_proj", + "ppl": 8.401236534118652 + }, + { + "layer": "layer.68.mlp.up_proj", + "ppl": 8.43631362915039 + }, + { + "layer": "layer.68.mlp.down_proj", + "ppl": 8.416448593139648 + }, + { + "layer": "layer.69.self_attn.q_proj", + "ppl": 8.434109687805176 + }, + { + "layer": "layer.69.self_attn.k_proj", + "ppl": 8.434283256530762 + }, + { + "layer": "layer.69.self_attn.v_proj", + "ppl": 8.431624412536621 + }, + { + "layer": "layer.69.self_attn.o_proj", + "ppl": 8.433683395385742 + }, + { + "layer": "layer.69.mlp.gate_proj", + "ppl": 8.40113639831543 + }, + { + "layer": "layer.69.mlp.up_proj", + "ppl": 8.427767753601074 + }, + { + "layer": "layer.69.mlp.down_proj", + "ppl": 8.415373802185059 + }, + { + "layer": "layer.70.self_attn.q_proj", + "ppl": 8.43480110168457 + }, + { + "layer": "layer.70.self_attn.k_proj", + "ppl": 8.437477111816406 + }, + { + "layer": "layer.70.self_attn.v_proj", + "ppl": 8.437143325805664 + }, + { + "layer": "layer.70.self_attn.o_proj", + "ppl": 8.433358192443848 + }, + { + "layer": "layer.70.mlp.gate_proj", + "ppl": 8.392451286315918 + }, + { + "layer": "layer.70.mlp.up_proj", + "ppl": 8.425031661987305 + }, + { + "layer": "layer.70.mlp.down_proj", + "ppl": 8.41224479675293 + }, + { + "layer": "layer.71.self_attn.q_proj", + "ppl": 8.437508583068848 + }, + { + "layer": "layer.71.self_attn.k_proj", + "ppl": 8.437609672546387 + }, + { + "layer": "layer.71.self_attn.v_proj", + "ppl": 8.429072380065918 + }, + { + "layer": "layer.71.self_attn.o_proj", + "ppl": 8.433160781860352 + }, + { + "layer": "layer.71.mlp.gate_proj", + "ppl": 8.397431373596191 + }, + { + "layer": "layer.71.mlp.up_proj", + "ppl": 8.427343368530273 + }, + { + "layer": "layer.71.mlp.down_proj", + "ppl": 8.411385536193848 + }, + { + "layer": "layer.72.self_attn.q_proj", + "ppl": 8.435614585876465 + }, + { + "layer": "layer.72.self_attn.k_proj", + "ppl": 8.437150955200195 + }, + { + "layer": "layer.72.self_attn.v_proj", + "ppl": 8.43068790435791 + }, + { + "layer": "layer.72.self_attn.o_proj", + "ppl": 8.436574935913086 + }, + { + "layer": "layer.72.mlp.gate_proj", + "ppl": 8.38936710357666 + }, + { + "layer": "layer.72.mlp.up_proj", + "ppl": 8.420513153076172 + }, + { + "layer": "layer.72.mlp.down_proj", + "ppl": 8.415781021118164 + }, + { + "layer": "layer.73.self_attn.q_proj", + "ppl": 8.431303024291992 + }, + { + "layer": "layer.73.self_attn.k_proj", + "ppl": 8.435384750366211 + }, + { + "layer": "layer.73.self_attn.v_proj", + "ppl": 8.439432144165039 + }, + { + "layer": "layer.73.self_attn.o_proj", + "ppl": 8.439098358154297 + }, + { + "layer": "layer.73.mlp.gate_proj", + "ppl": 8.385502815246582 + }, + { + "layer": "layer.73.mlp.up_proj", + "ppl": 8.416946411132812 + }, + { + "layer": "layer.73.mlp.down_proj", + "ppl": 8.414170265197754 + }, + { + "layer": "layer.74.self_attn.q_proj", + "ppl": 8.435903549194336 + }, + { + "layer": "layer.74.self_attn.k_proj", + "ppl": 8.438231468200684 + }, + { + "layer": "layer.74.self_attn.v_proj", + "ppl": 8.437339782714844 + }, + { + "layer": "layer.74.self_attn.o_proj", + "ppl": 8.436285972595215 + }, + { + "layer": "layer.74.mlp.gate_proj", + "ppl": 8.384054183959961 + }, + { + "layer": "layer.74.mlp.up_proj", + "ppl": 8.413816452026367 + }, + { + "layer": "layer.74.mlp.down_proj", + "ppl": 8.401280403137207 + }, + { + "layer": "layer.75.self_attn.q_proj", + "ppl": 8.433924674987793 + }, + { + "layer": "layer.75.self_attn.k_proj", + "ppl": 8.43947696685791 + }, + { + "layer": "layer.75.self_attn.v_proj", + "ppl": 8.440195083618164 + }, + { + "layer": "layer.75.self_attn.o_proj", + "ppl": 8.442523002624512 + }, + { + "layer": "layer.75.mlp.gate_proj", + "ppl": 8.404383659362793 + }, + { + "layer": "layer.75.mlp.up_proj", + "ppl": 8.412436485290527 + }, + { + "layer": "layer.75.mlp.down_proj", + "ppl": 8.41076374053955 + }, + { + "layer": "layer.76.self_attn.q_proj", + "ppl": 8.432023048400879 + }, + { + "layer": "layer.76.self_attn.k_proj", + "ppl": 8.43589973449707 + }, + { + "layer": "layer.76.self_attn.v_proj", + "ppl": 8.438014030456543 + }, + { + "layer": "layer.76.self_attn.o_proj", + "ppl": 8.438255310058594 + }, + { + "layer": "layer.76.mlp.gate_proj", + "ppl": 8.401926040649414 + }, + { + "layer": "layer.76.mlp.up_proj", + "ppl": 8.423157691955566 + }, + { + "layer": "layer.76.mlp.down_proj", + "ppl": 8.434475898742676 + }, + { + "layer": "layer.77.self_attn.q_proj", + "ppl": 8.449577331542969 + }, + { + "layer": "layer.77.self_attn.k_proj", + "ppl": 8.439894676208496 + }, + { + "layer": "layer.77.self_attn.v_proj", + "ppl": 8.450078964233398 + }, + { + "layer": "layer.77.self_attn.o_proj", + "ppl": 8.443574905395508 + }, + { + "layer": "layer.77.mlp.gate_proj", + "ppl": 8.402032852172852 + }, + { + "layer": "layer.77.mlp.up_proj", + "ppl": 8.409648895263672 + }, + { + "layer": "layer.77.mlp.down_proj", + "ppl": 8.399707794189453 + }, + { + "layer": "layer.78.self_attn.q_proj", + "ppl": 8.438583374023438 + }, + { + "layer": "layer.78.self_attn.k_proj", + "ppl": 8.43503475189209 + }, + { + "layer": "layer.78.self_attn.v_proj", + "ppl": 8.445510864257812 + }, + { + "layer": "layer.78.self_attn.o_proj", + "ppl": 8.448753356933594 + }, + { + "layer": "layer.78.mlp.gate_proj", + "ppl": 8.367792129516602 + }, + { + "layer": "layer.78.mlp.up_proj", + "ppl": 8.385520935058594 + }, + { + "layer": "layer.78.mlp.down_proj", + "ppl": 8.32874584197998 + }, + { + "layer": "layer.79.self_attn.q_proj", + "ppl": 8.431244850158691 + }, + { + "layer": "layer.79.self_attn.k_proj", + "ppl": 8.437352180480957 + }, + { + "layer": "layer.79.self_attn.v_proj", + "ppl": 8.466424942016602 + }, + { + "layer": "layer.79.self_attn.o_proj", + "ppl": 8.45706844329834 + }, + { + "layer": "layer.79.mlp.gate_proj", + "ppl": 8.28991985321045 + }, + { + "layer": "layer.79.mlp.up_proj", + "ppl": 8.279684066772461 + }, + { + "layer": "layer.79.mlp.down_proj", + "ppl": 8.125868797302246 + } + ] + }, + { + "bit": 3, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 5.51397180557251 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 5.51422643661499 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 5.513911247253418 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 5.51437520980835 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 5.515405178070068 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 5.51629638671875 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 5.516295433044434 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 5.514007091522217 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 5.514129638671875 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 5.512594223022461 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 5.5136942863464355 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 5.514279365539551 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 5.515121936798096 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 5.513031482696533 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 5.514398574829102 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 5.5141072273254395 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 5.5138726234436035 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 5.51340389251709 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 5.513745307922363 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 5.51397705078125 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 5.51280403137207 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 5.513675212860107 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 5.513891696929932 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 5.513644218444824 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 5.513322353363037 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 5.514100551605225 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 5.514010429382324 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 5.5131611824035645 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 5.513298511505127 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 5.513147354125977 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 5.51292610168457 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 5.513442039489746 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 5.512918472290039 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 5.513444423675537 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 5.512770175933838 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 5.514232158660889 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 5.514081954956055 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 5.513560771942139 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 5.514223098754883 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 5.513411521911621 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 5.513289928436279 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 5.512571811676025 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 5.513644218444824 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 5.51361608505249 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 5.5132060050964355 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 5.513375282287598 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 5.513425827026367 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 5.513505935668945 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 5.513365268707275 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 5.5145263671875 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 5.514232158660889 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 5.514213562011719 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 5.514138698577881 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 5.513579368591309 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 5.5136799812316895 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 5.514121055603027 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 5.513885974884033 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 5.514011859893799 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 5.513660430908203 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 5.514126777648926 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 5.512823581695557 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 5.513332843780518 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 5.512930393218994 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 5.514090061187744 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 5.514464855194092 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 5.514238357543945 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 5.5143232345581055 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 5.512932777404785 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 5.513247489929199 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 5.512648105621338 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 5.514298915863037 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 5.514169692993164 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 5.513681411743164 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 5.513892650604248 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 5.513113021850586 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 5.513467788696289 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 5.513406276702881 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 5.513876914978027 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 5.513513088226318 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 5.513662338256836 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 5.513980865478516 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 5.513000965118408 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 5.513343334197998 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 5.513463020324707 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 5.514689922332764 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 5.514402866363525 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 5.513946056365967 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 5.513828277587891 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 5.513600826263428 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 5.513535499572754 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 5.5131378173828125 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 5.514350891113281 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 5.5141191482543945 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 5.514345169067383 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 5.514044761657715 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 5.513665199279785 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 5.513286113739014 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 5.51343297958374 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 5.514315128326416 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 5.514000415802002 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 5.513766288757324 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 5.513926982879639 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 5.5138139724731445 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 5.513794422149658 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 5.5139007568359375 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 5.513778209686279 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 5.514236927032471 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 5.51413106918335 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 5.513671398162842 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 5.513309955596924 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 5.5134100914001465 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 5.5139265060424805 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 5.5144195556640625 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 5.51423454284668 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 5.513548374176025 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 5.513986110687256 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 5.513934135437012 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 5.513958930969238 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 5.513915061950684 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 5.51381778717041 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 5.5141072273254395 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 5.514083385467529 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 5.513371467590332 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 5.514061450958252 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 5.514786720275879 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 5.514520168304443 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 5.514592170715332 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 5.513669490814209 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 5.514636993408203 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 5.5143818855285645 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 5.513758182525635 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 5.514090061187744 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 5.513650417327881 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 5.514255046844482 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 5.5141472816467285 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 5.513840198516846 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 5.513849258422852 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 5.513794422149658 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 5.51401948928833 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 5.513054847717285 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 5.514159679412842 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 5.513828277587891 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 5.51411771774292 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 5.51405668258667 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 5.512775421142578 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 5.513950347900391 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 5.5131754875183105 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 5.513874530792236 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 5.513851642608643 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 5.513923168182373 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 5.513763427734375 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 5.514312267303467 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 5.514688014984131 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 5.513576030731201 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 5.514050483703613 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 5.5140767097473145 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 5.513402462005615 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 5.5131354331970215 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 5.513488292694092 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 5.514022350311279 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 5.512223720550537 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 5.514122486114502 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 5.514091968536377 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 5.51326847076416 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 5.514473915100098 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 5.513852119445801 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 5.514284133911133 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 5.512829303741455 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 5.513879776000977 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 5.513846397399902 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 5.513102054595947 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 5.512949466705322 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 5.512349605560303 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 5.5133233070373535 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 5.512219429016113 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 5.5136260986328125 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 5.513734817504883 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 5.513322353363037 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 5.513257026672363 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 5.512722492218018 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 5.512813568115234 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 5.5105299949646 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 5.513760566711426 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 5.5136871337890625 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 5.513035774230957 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 5.514194488525391 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 5.5121893882751465 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 5.512491703033447 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 5.5115227699279785 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 5.513722896575928 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 5.513642311096191 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 5.513746738433838 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 5.513479709625244 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 5.513130187988281 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 5.511776924133301 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 5.514010429382324 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 5.514023780822754 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 5.514068603515625 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 5.513552188873291 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 5.513955116271973 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 5.512467861175537 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 5.5138020515441895 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 5.512082099914551 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 5.5135955810546875 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 5.513489246368408 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 5.513948917388916 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 5.5129923820495605 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 5.512391567230225 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 5.512238025665283 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 5.51271915435791 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 5.51405668258667 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 5.514118194580078 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 5.512923240661621 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 5.513586044311523 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 5.511960506439209 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 5.513425827026367 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 5.512727737426758 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 5.51340913772583 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 5.513899803161621 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 5.513736248016357 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 5.513524532318115 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 5.513425827026367 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 5.5147881507873535 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 5.5131025314331055 + }, + { + "layer": "layer.32.self_attn.q_proj", + "ppl": 5.5139384269714355 + }, + { + "layer": "layer.32.self_attn.k_proj", + "ppl": 5.513479709625244 + }, + { + "layer": "layer.32.self_attn.v_proj", + "ppl": 5.514422416687012 + }, + { + "layer": "layer.32.self_attn.o_proj", + "ppl": 5.514009475708008 + }, + { + "layer": "layer.32.mlp.gate_proj", + "ppl": 5.5118818283081055 + }, + { + "layer": "layer.32.mlp.up_proj", + "ppl": 5.512892723083496 + }, + { + "layer": "layer.32.mlp.down_proj", + "ppl": 5.510375499725342 + }, + { + "layer": "layer.33.self_attn.q_proj", + "ppl": 5.514034748077393 + }, + { + "layer": "layer.33.self_attn.k_proj", + "ppl": 5.5135064125061035 + }, + { + "layer": "layer.33.self_attn.v_proj", + "ppl": 5.514549732208252 + }, + { + "layer": "layer.33.self_attn.o_proj", + "ppl": 5.513619422912598 + }, + { + "layer": "layer.33.mlp.gate_proj", + "ppl": 5.510427474975586 + }, + { + "layer": "layer.33.mlp.up_proj", + "ppl": 5.513306617736816 + }, + { + "layer": "layer.33.mlp.down_proj", + "ppl": 5.510874271392822 + }, + { + "layer": "layer.34.self_attn.q_proj", + "ppl": 5.5138044357299805 + }, + { + "layer": "layer.34.self_attn.k_proj", + "ppl": 5.514336585998535 + }, + { + "layer": "layer.34.self_attn.v_proj", + "ppl": 5.513259410858154 + }, + { + "layer": "layer.34.self_attn.o_proj", + "ppl": 5.512522220611572 + }, + { + "layer": "layer.34.mlp.gate_proj", + "ppl": 5.511094570159912 + }, + { + "layer": "layer.34.mlp.up_proj", + "ppl": 5.512248516082764 + }, + { + "layer": "layer.34.mlp.down_proj", + "ppl": 5.5107951164245605 + }, + { + "layer": "layer.35.self_attn.q_proj", + "ppl": 5.513169765472412 + }, + { + "layer": "layer.35.self_attn.k_proj", + "ppl": 5.513501167297363 + }, + { + "layer": "layer.35.self_attn.v_proj", + "ppl": 5.513176441192627 + }, + { + "layer": "layer.35.self_attn.o_proj", + "ppl": 5.513312816619873 + }, + { + "layer": "layer.35.mlp.gate_proj", + "ppl": 5.512849807739258 + }, + { + "layer": "layer.35.mlp.up_proj", + "ppl": 5.511414051055908 + }, + { + "layer": "layer.35.mlp.down_proj", + "ppl": 5.512011528015137 + }, + { + "layer": "layer.36.self_attn.q_proj", + "ppl": 5.513412952423096 + }, + { + "layer": "layer.36.self_attn.k_proj", + "ppl": 5.5136871337890625 + }, + { + "layer": "layer.36.self_attn.v_proj", + "ppl": 5.513221263885498 + }, + { + "layer": "layer.36.self_attn.o_proj", + "ppl": 5.513271331787109 + }, + { + "layer": "layer.36.mlp.gate_proj", + "ppl": 5.511714935302734 + }, + { + "layer": "layer.36.mlp.up_proj", + "ppl": 5.512826442718506 + }, + { + "layer": "layer.36.mlp.down_proj", + "ppl": 5.511132717132568 + }, + { + "layer": "layer.37.self_attn.q_proj", + "ppl": 5.513824939727783 + }, + { + "layer": "layer.37.self_attn.k_proj", + "ppl": 5.513746738433838 + }, + { + "layer": "layer.37.self_attn.v_proj", + "ppl": 5.5130414962768555 + }, + { + "layer": "layer.37.self_attn.o_proj", + "ppl": 5.5140228271484375 + }, + { + "layer": "layer.37.mlp.gate_proj", + "ppl": 5.512037754058838 + }, + { + "layer": "layer.37.mlp.up_proj", + "ppl": 5.512011528015137 + }, + { + "layer": "layer.37.mlp.down_proj", + "ppl": 5.512300968170166 + }, + { + "layer": "layer.38.self_attn.q_proj", + "ppl": 5.514552593231201 + }, + { + "layer": "layer.38.self_attn.k_proj", + "ppl": 5.51467752456665 + }, + { + "layer": "layer.38.self_attn.v_proj", + "ppl": 5.5142083168029785 + }, + { + "layer": "layer.38.self_attn.o_proj", + "ppl": 5.5134196281433105 + }, + { + "layer": "layer.38.mlp.gate_proj", + "ppl": 5.511679172515869 + }, + { + "layer": "layer.38.mlp.up_proj", + "ppl": 5.511322975158691 + }, + { + "layer": "layer.38.mlp.down_proj", + "ppl": 5.511824131011963 + }, + { + "layer": "layer.39.self_attn.q_proj", + "ppl": 5.513850688934326 + }, + { + "layer": "layer.39.self_attn.k_proj", + "ppl": 5.5136919021606445 + }, + { + "layer": "layer.39.self_attn.v_proj", + "ppl": 5.512905120849609 + }, + { + "layer": "layer.39.self_attn.o_proj", + "ppl": 5.513373851776123 + }, + { + "layer": "layer.39.mlp.gate_proj", + "ppl": 5.512655735015869 + }, + { + "layer": "layer.39.mlp.up_proj", + "ppl": 5.511218070983887 + }, + { + "layer": "layer.39.mlp.down_proj", + "ppl": 5.512871265411377 + }, + { + "layer": "layer.40.self_attn.q_proj", + "ppl": 5.5134053230285645 + }, + { + "layer": "layer.40.self_attn.k_proj", + "ppl": 5.514036178588867 + }, + { + "layer": "layer.40.self_attn.v_proj", + "ppl": 5.514383316040039 + }, + { + "layer": "layer.40.self_attn.o_proj", + "ppl": 5.513573169708252 + }, + { + "layer": "layer.40.mlp.gate_proj", + "ppl": 5.5106520652771 + }, + { + "layer": "layer.40.mlp.up_proj", + "ppl": 5.512678146362305 + }, + { + "layer": "layer.40.mlp.down_proj", + "ppl": 5.510904312133789 + }, + { + "layer": "layer.41.self_attn.q_proj", + "ppl": 5.513747692108154 + }, + { + "layer": "layer.41.self_attn.k_proj", + "ppl": 5.513740539550781 + }, + { + "layer": "layer.41.self_attn.v_proj", + "ppl": 5.513632297515869 + }, + { + "layer": "layer.41.self_attn.o_proj", + "ppl": 5.5131001472473145 + }, + { + "layer": "layer.41.mlp.gate_proj", + "ppl": 5.511781215667725 + }, + { + "layer": "layer.41.mlp.up_proj", + "ppl": 5.513725757598877 + }, + { + "layer": "layer.41.mlp.down_proj", + "ppl": 5.509703636169434 + }, + { + "layer": "layer.42.self_attn.q_proj", + "ppl": 5.51454496383667 + }, + { + "layer": "layer.42.self_attn.k_proj", + "ppl": 5.514172554016113 + }, + { + "layer": "layer.42.self_attn.v_proj", + "ppl": 5.513256549835205 + }, + { + "layer": "layer.42.self_attn.o_proj", + "ppl": 5.514164924621582 + }, + { + "layer": "layer.42.mlp.gate_proj", + "ppl": 5.511033058166504 + }, + { + "layer": "layer.42.mlp.up_proj", + "ppl": 5.510975360870361 + }, + { + "layer": "layer.42.mlp.down_proj", + "ppl": 5.512882232666016 + }, + { + "layer": "layer.43.self_attn.q_proj", + "ppl": 5.513648509979248 + }, + { + "layer": "layer.43.self_attn.k_proj", + "ppl": 5.5141801834106445 + }, + { + "layer": "layer.43.self_attn.v_proj", + "ppl": 5.514577865600586 + }, + { + "layer": "layer.43.self_attn.o_proj", + "ppl": 5.513094902038574 + }, + { + "layer": "layer.43.mlp.gate_proj", + "ppl": 5.512804985046387 + }, + { + "layer": "layer.43.mlp.up_proj", + "ppl": 5.512004852294922 + }, + { + "layer": "layer.43.mlp.down_proj", + "ppl": 5.511534690856934 + }, + { + "layer": "layer.44.self_attn.q_proj", + "ppl": 5.51483154296875 + }, + { + "layer": "layer.44.self_attn.k_proj", + "ppl": 5.514404296875 + }, + { + "layer": "layer.44.self_attn.v_proj", + "ppl": 5.514501094818115 + }, + { + "layer": "layer.44.self_attn.o_proj", + "ppl": 5.512596607208252 + }, + { + "layer": "layer.44.mlp.gate_proj", + "ppl": 5.5130486488342285 + }, + { + "layer": "layer.44.mlp.up_proj", + "ppl": 5.512814044952393 + }, + { + "layer": "layer.44.mlp.down_proj", + "ppl": 5.512773036956787 + }, + { + "layer": "layer.45.self_attn.q_proj", + "ppl": 5.513573169708252 + }, + { + "layer": "layer.45.self_attn.k_proj", + "ppl": 5.514266014099121 + }, + { + "layer": "layer.45.self_attn.v_proj", + "ppl": 5.514160633087158 + }, + { + "layer": "layer.45.self_attn.o_proj", + "ppl": 5.513456344604492 + }, + { + "layer": "layer.45.mlp.gate_proj", + "ppl": 5.511527061462402 + }, + { + "layer": "layer.45.mlp.up_proj", + "ppl": 5.511292934417725 + }, + { + "layer": "layer.45.mlp.down_proj", + "ppl": 5.511380195617676 + }, + { + "layer": "layer.46.self_attn.q_proj", + "ppl": 5.51367712020874 + }, + { + "layer": "layer.46.self_attn.k_proj", + "ppl": 5.513549327850342 + }, + { + "layer": "layer.46.self_attn.v_proj", + "ppl": 5.514069557189941 + }, + { + "layer": "layer.46.self_attn.o_proj", + "ppl": 5.51354455947876 + }, + { + "layer": "layer.46.mlp.gate_proj", + "ppl": 5.510519981384277 + }, + { + "layer": "layer.46.mlp.up_proj", + "ppl": 5.511793613433838 + }, + { + "layer": "layer.46.mlp.down_proj", + "ppl": 5.512061595916748 + }, + { + "layer": "layer.47.self_attn.q_proj", + "ppl": 5.513991355895996 + }, + { + "layer": "layer.47.self_attn.k_proj", + "ppl": 5.514088153839111 + }, + { + "layer": "layer.47.self_attn.v_proj", + "ppl": 5.515133857727051 + }, + { + "layer": "layer.47.self_attn.o_proj", + "ppl": 5.512912273406982 + }, + { + "layer": "layer.47.mlp.gate_proj", + "ppl": 5.51165771484375 + }, + { + "layer": "layer.47.mlp.up_proj", + "ppl": 5.512274265289307 + }, + { + "layer": "layer.47.mlp.down_proj", + "ppl": 5.5102009773254395 + }, + { + "layer": "layer.48.self_attn.q_proj", + "ppl": 5.513919830322266 + }, + { + "layer": "layer.48.self_attn.k_proj", + "ppl": 5.513689994812012 + }, + { + "layer": "layer.48.self_attn.v_proj", + "ppl": 5.513068199157715 + }, + { + "layer": "layer.48.self_attn.o_proj", + "ppl": 5.5139384269714355 + }, + { + "layer": "layer.48.mlp.gate_proj", + "ppl": 5.509477138519287 + }, + { + "layer": "layer.48.mlp.up_proj", + "ppl": 5.5110063552856445 + }, + { + "layer": "layer.48.mlp.down_proj", + "ppl": 5.511894226074219 + }, + { + "layer": "layer.49.self_attn.q_proj", + "ppl": 5.513146877288818 + }, + { + "layer": "layer.49.self_attn.k_proj", + "ppl": 5.513696670532227 + }, + { + "layer": "layer.49.self_attn.v_proj", + "ppl": 5.513857364654541 + }, + { + "layer": "layer.49.self_attn.o_proj", + "ppl": 5.514798641204834 + }, + { + "layer": "layer.49.mlp.gate_proj", + "ppl": 5.506067276000977 + }, + { + "layer": "layer.49.mlp.up_proj", + "ppl": 5.506082057952881 + }, + { + "layer": "layer.49.mlp.down_proj", + "ppl": 5.512669086456299 + }, + { + "layer": "layer.50.self_attn.q_proj", + "ppl": 5.5133442878723145 + }, + { + "layer": "layer.50.self_attn.k_proj", + "ppl": 5.513556480407715 + }, + { + "layer": "layer.50.self_attn.v_proj", + "ppl": 5.514441013336182 + }, + { + "layer": "layer.50.self_attn.o_proj", + "ppl": 5.513786792755127 + }, + { + "layer": "layer.50.mlp.gate_proj", + "ppl": 5.511912822723389 + }, + { + "layer": "layer.50.mlp.up_proj", + "ppl": 5.511435031890869 + }, + { + "layer": "layer.50.mlp.down_proj", + "ppl": 5.51371955871582 + }, + { + "layer": "layer.51.self_attn.q_proj", + "ppl": 5.515671730041504 + }, + { + "layer": "layer.51.self_attn.k_proj", + "ppl": 5.514225006103516 + }, + { + "layer": "layer.51.self_attn.v_proj", + "ppl": 5.514513969421387 + }, + { + "layer": "layer.51.self_attn.o_proj", + "ppl": 5.51266622543335 + }, + { + "layer": "layer.51.mlp.gate_proj", + "ppl": 5.5121378898620605 + }, + { + "layer": "layer.51.mlp.up_proj", + "ppl": 5.513200283050537 + }, + { + "layer": "layer.51.mlp.down_proj", + "ppl": 5.513415813446045 + }, + { + "layer": "layer.52.self_attn.q_proj", + "ppl": 5.513866424560547 + }, + { + "layer": "layer.52.self_attn.k_proj", + "ppl": 5.513350009918213 + }, + { + "layer": "layer.52.self_attn.v_proj", + "ppl": 5.514607906341553 + }, + { + "layer": "layer.52.self_attn.o_proj", + "ppl": 5.511905670166016 + }, + { + "layer": "layer.52.mlp.gate_proj", + "ppl": 5.511734962463379 + }, + { + "layer": "layer.52.mlp.up_proj", + "ppl": 5.513733386993408 + }, + { + "layer": "layer.52.mlp.down_proj", + "ppl": 5.513326644897461 + }, + { + "layer": "layer.53.self_attn.q_proj", + "ppl": 5.514603137969971 + }, + { + "layer": "layer.53.self_attn.k_proj", + "ppl": 5.514461994171143 + }, + { + "layer": "layer.53.self_attn.v_proj", + "ppl": 5.513237953186035 + }, + { + "layer": "layer.53.self_attn.o_proj", + "ppl": 5.5129265785217285 + }, + { + "layer": "layer.53.mlp.gate_proj", + "ppl": 5.512051105499268 + }, + { + "layer": "layer.53.mlp.up_proj", + "ppl": 5.5126519203186035 + }, + { + "layer": "layer.53.mlp.down_proj", + "ppl": 5.511776447296143 + }, + { + "layer": "layer.54.self_attn.q_proj", + "ppl": 5.513978004455566 + }, + { + "layer": "layer.54.self_attn.k_proj", + "ppl": 5.514357089996338 + }, + { + "layer": "layer.54.self_attn.v_proj", + "ppl": 5.513978004455566 + }, + { + "layer": "layer.54.self_attn.o_proj", + "ppl": 5.513510704040527 + }, + { + "layer": "layer.54.mlp.gate_proj", + "ppl": 5.513111114501953 + }, + { + "layer": "layer.54.mlp.up_proj", + "ppl": 5.514275550842285 + }, + { + "layer": "layer.54.mlp.down_proj", + "ppl": 5.513550758361816 + }, + { + "layer": "layer.55.self_attn.q_proj", + "ppl": 5.5129289627075195 + }, + { + "layer": "layer.55.self_attn.k_proj", + "ppl": 5.513218402862549 + }, + { + "layer": "layer.55.self_attn.v_proj", + "ppl": 5.513338088989258 + }, + { + "layer": "layer.55.self_attn.o_proj", + "ppl": 5.510605812072754 + }, + { + "layer": "layer.55.mlp.gate_proj", + "ppl": 5.511655330657959 + }, + { + "layer": "layer.55.mlp.up_proj", + "ppl": 5.5130486488342285 + }, + { + "layer": "layer.55.mlp.down_proj", + "ppl": 5.510458946228027 + }, + { + "layer": "layer.56.self_attn.q_proj", + "ppl": 5.51286506652832 + }, + { + "layer": "layer.56.self_attn.k_proj", + "ppl": 5.512111663818359 + }, + { + "layer": "layer.56.self_attn.v_proj", + "ppl": 5.512587547302246 + }, + { + "layer": "layer.56.self_attn.o_proj", + "ppl": 5.510988235473633 + }, + { + "layer": "layer.56.mlp.gate_proj", + "ppl": 5.509881973266602 + }, + { + "layer": "layer.56.mlp.up_proj", + "ppl": 5.513911247253418 + }, + { + "layer": "layer.56.mlp.down_proj", + "ppl": 5.511500835418701 + }, + { + "layer": "layer.57.self_attn.q_proj", + "ppl": 5.513285160064697 + }, + { + "layer": "layer.57.self_attn.k_proj", + "ppl": 5.514214515686035 + }, + { + "layer": "layer.57.self_attn.v_proj", + "ppl": 5.513558387756348 + }, + { + "layer": "layer.57.self_attn.o_proj", + "ppl": 5.512728214263916 + }, + { + "layer": "layer.57.mlp.gate_proj", + "ppl": 5.512491703033447 + }, + { + "layer": "layer.57.mlp.up_proj", + "ppl": 5.514166831970215 + }, + { + "layer": "layer.57.mlp.down_proj", + "ppl": 5.510583400726318 + }, + { + "layer": "layer.58.self_attn.q_proj", + "ppl": 5.513945579528809 + }, + { + "layer": "layer.58.self_attn.k_proj", + "ppl": 5.5135087966918945 + }, + { + "layer": "layer.58.self_attn.v_proj", + "ppl": 5.514957904815674 + }, + { + "layer": "layer.58.self_attn.o_proj", + "ppl": 5.512248992919922 + }, + { + "layer": "layer.58.mlp.gate_proj", + "ppl": 5.511964797973633 + }, + { + "layer": "layer.58.mlp.up_proj", + "ppl": 5.514004707336426 + }, + { + "layer": "layer.58.mlp.down_proj", + "ppl": 5.5123515129089355 + }, + { + "layer": "layer.59.self_attn.q_proj", + "ppl": 5.512457847595215 + }, + { + "layer": "layer.59.self_attn.k_proj", + "ppl": 5.51309061050415 + }, + { + "layer": "layer.59.self_attn.v_proj", + "ppl": 5.513375759124756 + }, + { + "layer": "layer.59.self_attn.o_proj", + "ppl": 5.512128829956055 + }, + { + "layer": "layer.59.mlp.gate_proj", + "ppl": 5.512197017669678 + }, + { + "layer": "layer.59.mlp.up_proj", + "ppl": 5.511456489562988 + }, + { + "layer": "layer.59.mlp.down_proj", + "ppl": 5.5095744132995605 + }, + { + "layer": "layer.60.self_attn.q_proj", + "ppl": 5.514479160308838 + }, + { + "layer": "layer.60.self_attn.k_proj", + "ppl": 5.513430118560791 + }, + { + "layer": "layer.60.self_attn.v_proj", + "ppl": 5.513364791870117 + }, + { + "layer": "layer.60.self_attn.o_proj", + "ppl": 5.511868476867676 + }, + { + "layer": "layer.60.mlp.gate_proj", + "ppl": 5.50831937789917 + }, + { + "layer": "layer.60.mlp.up_proj", + "ppl": 5.513257026672363 + }, + { + "layer": "layer.60.mlp.down_proj", + "ppl": 5.5099358558654785 + }, + { + "layer": "layer.61.self_attn.q_proj", + "ppl": 5.513772010803223 + }, + { + "layer": "layer.61.self_attn.k_proj", + "ppl": 5.513573169708252 + }, + { + "layer": "layer.61.self_attn.v_proj", + "ppl": 5.514030456542969 + }, + { + "layer": "layer.61.self_attn.o_proj", + "ppl": 5.512800693511963 + }, + { + "layer": "layer.61.mlp.gate_proj", + "ppl": 5.509766101837158 + }, + { + "layer": "layer.61.mlp.up_proj", + "ppl": 5.510016441345215 + }, + { + "layer": "layer.61.mlp.down_proj", + "ppl": 5.510794162750244 + }, + { + "layer": "layer.62.self_attn.q_proj", + "ppl": 5.511543273925781 + }, + { + "layer": "layer.62.self_attn.k_proj", + "ppl": 5.511427879333496 + }, + { + "layer": "layer.62.self_attn.v_proj", + "ppl": 5.512821674346924 + }, + { + "layer": "layer.62.self_attn.o_proj", + "ppl": 5.512712001800537 + }, + { + "layer": "layer.62.mlp.gate_proj", + "ppl": 5.508686542510986 + }, + { + "layer": "layer.62.mlp.up_proj", + "ppl": 5.512994289398193 + }, + { + "layer": "layer.62.mlp.down_proj", + "ppl": 5.508646011352539 + }, + { + "layer": "layer.63.self_attn.q_proj", + "ppl": 5.512796401977539 + }, + { + "layer": "layer.63.self_attn.k_proj", + "ppl": 5.513285160064697 + }, + { + "layer": "layer.63.self_attn.v_proj", + "ppl": 5.512679576873779 + }, + { + "layer": "layer.63.self_attn.o_proj", + "ppl": 5.511402130126953 + }, + { + "layer": "layer.63.mlp.gate_proj", + "ppl": 5.510396957397461 + }, + { + "layer": "layer.63.mlp.up_proj", + "ppl": 5.510629177093506 + }, + { + "layer": "layer.63.mlp.down_proj", + "ppl": 5.5104193687438965 + }, + { + "layer": "layer.64.self_attn.q_proj", + "ppl": 5.512805461883545 + }, + { + "layer": "layer.64.self_attn.k_proj", + "ppl": 5.512998580932617 + }, + { + "layer": "layer.64.self_attn.v_proj", + "ppl": 5.512772560119629 + }, + { + "layer": "layer.64.self_attn.o_proj", + "ppl": 5.511561393737793 + }, + { + "layer": "layer.64.mlp.gate_proj", + "ppl": 5.507650375366211 + }, + { + "layer": "layer.64.mlp.up_proj", + "ppl": 5.513155937194824 + }, + { + "layer": "layer.64.mlp.down_proj", + "ppl": 5.509214401245117 + }, + { + "layer": "layer.65.self_attn.q_proj", + "ppl": 5.509038925170898 + }, + { + "layer": "layer.65.self_attn.k_proj", + "ppl": 5.512480735778809 + }, + { + "layer": "layer.65.self_attn.v_proj", + "ppl": 5.512462139129639 + }, + { + "layer": "layer.65.self_attn.o_proj", + "ppl": 5.51257848739624 + }, + { + "layer": "layer.65.mlp.gate_proj", + "ppl": 5.507838726043701 + }, + { + "layer": "layer.65.mlp.up_proj", + "ppl": 5.509129524230957 + }, + { + "layer": "layer.65.mlp.down_proj", + "ppl": 5.510698318481445 + }, + { + "layer": "layer.66.self_attn.q_proj", + "ppl": 5.514170169830322 + }, + { + "layer": "layer.66.self_attn.k_proj", + "ppl": 5.512954235076904 + }, + { + "layer": "layer.66.self_attn.v_proj", + "ppl": 5.514279365539551 + }, + { + "layer": "layer.66.self_attn.o_proj", + "ppl": 5.512680530548096 + }, + { + "layer": "layer.66.mlp.gate_proj", + "ppl": 5.508090972900391 + }, + { + "layer": "layer.66.mlp.up_proj", + "ppl": 5.5128984451293945 + }, + { + "layer": "layer.66.mlp.down_proj", + "ppl": 5.512160301208496 + }, + { + "layer": "layer.67.self_attn.q_proj", + "ppl": 5.5147857666015625 + }, + { + "layer": "layer.67.self_attn.k_proj", + "ppl": 5.5145721435546875 + }, + { + "layer": "layer.67.self_attn.v_proj", + "ppl": 5.514556884765625 + }, + { + "layer": "layer.67.self_attn.o_proj", + "ppl": 5.513469219207764 + }, + { + "layer": "layer.67.mlp.gate_proj", + "ppl": 5.5077385902404785 + }, + { + "layer": "layer.67.mlp.up_proj", + "ppl": 5.512131690979004 + }, + { + "layer": "layer.67.mlp.down_proj", + "ppl": 5.511162757873535 + }, + { + "layer": "layer.68.self_attn.q_proj", + "ppl": 5.512897491455078 + }, + { + "layer": "layer.68.self_attn.k_proj", + "ppl": 5.512921333312988 + }, + { + "layer": "layer.68.self_attn.v_proj", + "ppl": 5.514300346374512 + }, + { + "layer": "layer.68.self_attn.o_proj", + "ppl": 5.512375354766846 + }, + { + "layer": "layer.68.mlp.gate_proj", + "ppl": 5.5089497566223145 + }, + { + "layer": "layer.68.mlp.up_proj", + "ppl": 5.511641502380371 + }, + { + "layer": "layer.68.mlp.down_proj", + "ppl": 5.5099592208862305 + }, + { + "layer": "layer.69.self_attn.q_proj", + "ppl": 5.513253688812256 + }, + { + "layer": "layer.69.self_attn.k_proj", + "ppl": 5.514280796051025 + }, + { + "layer": "layer.69.self_attn.v_proj", + "ppl": 5.513470649719238 + }, + { + "layer": "layer.69.self_attn.o_proj", + "ppl": 5.513514041900635 + }, + { + "layer": "layer.69.mlp.gate_proj", + "ppl": 5.506117343902588 + }, + { + "layer": "layer.69.mlp.up_proj", + "ppl": 5.511744022369385 + }, + { + "layer": "layer.69.mlp.down_proj", + "ppl": 5.511817932128906 + }, + { + "layer": "layer.70.self_attn.q_proj", + "ppl": 5.51414155960083 + }, + { + "layer": "layer.70.self_attn.k_proj", + "ppl": 5.514233589172363 + }, + { + "layer": "layer.70.self_attn.v_proj", + "ppl": 5.513031482696533 + }, + { + "layer": "layer.70.self_attn.o_proj", + "ppl": 5.513054370880127 + }, + { + "layer": "layer.70.mlp.gate_proj", + "ppl": 5.504940032958984 + }, + { + "layer": "layer.70.mlp.up_proj", + "ppl": 5.510495662689209 + }, + { + "layer": "layer.70.mlp.down_proj", + "ppl": 5.510429859161377 + }, + { + "layer": "layer.71.self_attn.q_proj", + "ppl": 5.514801025390625 + }, + { + "layer": "layer.71.self_attn.k_proj", + "ppl": 5.51496696472168 + }, + { + "layer": "layer.71.self_attn.v_proj", + "ppl": 5.514010906219482 + }, + { + "layer": "layer.71.self_attn.o_proj", + "ppl": 5.513473033905029 + }, + { + "layer": "layer.71.mlp.gate_proj", + "ppl": 5.506985187530518 + }, + { + "layer": "layer.71.mlp.up_proj", + "ppl": 5.509762763977051 + }, + { + "layer": "layer.71.mlp.down_proj", + "ppl": 5.510087013244629 + }, + { + "layer": "layer.72.self_attn.q_proj", + "ppl": 5.5131449699401855 + }, + { + "layer": "layer.72.self_attn.k_proj", + "ppl": 5.514153480529785 + }, + { + "layer": "layer.72.self_attn.v_proj", + "ppl": 5.512856960296631 + }, + { + "layer": "layer.72.self_attn.o_proj", + "ppl": 5.5128068923950195 + }, + { + "layer": "layer.72.mlp.gate_proj", + "ppl": 5.504757881164551 + }, + { + "layer": "layer.72.mlp.up_proj", + "ppl": 5.509491443634033 + }, + { + "layer": "layer.72.mlp.down_proj", + "ppl": 5.512383937835693 + }, + { + "layer": "layer.73.self_attn.q_proj", + "ppl": 5.514010429382324 + }, + { + "layer": "layer.73.self_attn.k_proj", + "ppl": 5.515251636505127 + }, + { + "layer": "layer.73.self_attn.v_proj", + "ppl": 5.514477729797363 + }, + { + "layer": "layer.73.self_attn.o_proj", + "ppl": 5.513573169708252 + }, + { + "layer": "layer.73.mlp.gate_proj", + "ppl": 5.5038065910339355 + }, + { + "layer": "layer.73.mlp.up_proj", + "ppl": 5.5098419189453125 + }, + { + "layer": "layer.73.mlp.down_proj", + "ppl": 5.5074286460876465 + }, + { + "layer": "layer.74.self_attn.q_proj", + "ppl": 5.513381004333496 + }, + { + "layer": "layer.74.self_attn.k_proj", + "ppl": 5.5133490562438965 + }, + { + "layer": "layer.74.self_attn.v_proj", + "ppl": 5.513709545135498 + }, + { + "layer": "layer.74.self_attn.o_proj", + "ppl": 5.514651298522949 + }, + { + "layer": "layer.74.mlp.gate_proj", + "ppl": 5.503017902374268 + }, + { + "layer": "layer.74.mlp.up_proj", + "ppl": 5.510184288024902 + }, + { + "layer": "layer.74.mlp.down_proj", + "ppl": 5.510400772094727 + }, + { + "layer": "layer.75.self_attn.q_proj", + "ppl": 5.514217376708984 + }, + { + "layer": "layer.75.self_attn.k_proj", + "ppl": 5.514402866363525 + }, + { + "layer": "layer.75.self_attn.v_proj", + "ppl": 5.513424396514893 + }, + { + "layer": "layer.75.self_attn.o_proj", + "ppl": 5.513917446136475 + }, + { + "layer": "layer.75.mlp.gate_proj", + "ppl": 5.505828380584717 + }, + { + "layer": "layer.75.mlp.up_proj", + "ppl": 5.501712799072266 + }, + { + "layer": "layer.75.mlp.down_proj", + "ppl": 5.51049280166626 + }, + { + "layer": "layer.76.self_attn.q_proj", + "ppl": 5.514885425567627 + }, + { + "layer": "layer.76.self_attn.k_proj", + "ppl": 5.5153350830078125 + }, + { + "layer": "layer.76.self_attn.v_proj", + "ppl": 5.515736103057861 + }, + { + "layer": "layer.76.self_attn.o_proj", + "ppl": 5.513675689697266 + }, + { + "layer": "layer.76.mlp.gate_proj", + "ppl": 5.505327224731445 + }, + { + "layer": "layer.76.mlp.up_proj", + "ppl": 5.500453472137451 + }, + { + "layer": "layer.76.mlp.down_proj", + "ppl": 5.50971794128418 + }, + { + "layer": "layer.77.self_attn.q_proj", + "ppl": 5.516908645629883 + }, + { + "layer": "layer.77.self_attn.k_proj", + "ppl": 5.516780376434326 + }, + { + "layer": "layer.77.self_attn.v_proj", + "ppl": 5.515347480773926 + }, + { + "layer": "layer.77.self_attn.o_proj", + "ppl": 5.516739368438721 + }, + { + "layer": "layer.77.mlp.gate_proj", + "ppl": 5.501150608062744 + }, + { + "layer": "layer.77.mlp.up_proj", + "ppl": 5.502776145935059 + }, + { + "layer": "layer.77.mlp.down_proj", + "ppl": 5.500026702880859 + }, + { + "layer": "layer.78.self_attn.q_proj", + "ppl": 5.5133843421936035 + }, + { + "layer": "layer.78.self_attn.k_proj", + "ppl": 5.513864040374756 + }, + { + "layer": "layer.78.self_attn.v_proj", + "ppl": 5.514043807983398 + }, + { + "layer": "layer.78.self_attn.o_proj", + "ppl": 5.5189008712768555 + }, + { + "layer": "layer.78.mlp.gate_proj", + "ppl": 5.503373622894287 + }, + { + "layer": "layer.78.mlp.up_proj", + "ppl": 5.501648426055908 + }, + { + "layer": "layer.78.mlp.down_proj", + "ppl": 5.497072696685791 + }, + { + "layer": "layer.79.self_attn.q_proj", + "ppl": 5.512425422668457 + }, + { + "layer": "layer.79.self_attn.k_proj", + "ppl": 5.514126777648926 + }, + { + "layer": "layer.79.self_attn.v_proj", + "ppl": 5.514781951904297 + }, + { + "layer": "layer.79.self_attn.o_proj", + "ppl": 5.517577648162842 + }, + { + "layer": "layer.79.mlp.gate_proj", + "ppl": 5.485888957977295 + }, + { + "layer": "layer.79.mlp.up_proj", + "ppl": 5.480979919433594 + }, + { + "layer": "layer.79.mlp.down_proj", + "ppl": 5.452732563018799 + } + ] + }, + { + "bit": 4, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 4.703306198120117 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 4.703546047210693 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 4.7030930519104 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 4.703035831451416 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 4.703409671783447 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 4.703556060791016 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 4.703685283660889 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 4.70335578918457 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 4.703385353088379 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 4.7030744552612305 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 4.703028202056885 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 4.7033891677856445 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 4.703714370727539 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 4.702980995178223 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 4.703360080718994 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 4.703396797180176 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 4.703269958496094 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 4.703349590301514 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 4.703120708465576 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 4.703151226043701 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 4.703115940093994 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 4.703261375427246 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 4.70341157913208 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 4.703495025634766 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 4.703186511993408 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 4.703306198120117 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 4.703193664550781 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 4.703367710113525 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 4.703091621398926 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 4.703164577484131 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 4.70321798324585 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 4.703062534332275 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 4.703206539154053 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 4.7033162117004395 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 4.703061580657959 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 4.70341157913208 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 4.70335578918457 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 4.703524112701416 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 4.703357696533203 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 4.70306396484375 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 4.70307731628418 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 4.703205108642578 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 4.703166961669922 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 4.703246593475342 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 4.703227996826172 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 4.703239917755127 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 4.703414440155029 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 4.703061580657959 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 4.70311164855957 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 4.703576564788818 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 4.703375339508057 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 4.703101634979248 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 4.703277111053467 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 4.702792167663574 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 4.703282833099365 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 4.703125476837158 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 4.7032904624938965 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 4.703240394592285 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 4.703152179718018 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 4.7033891677856445 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 4.703114032745361 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 4.70308256149292 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 4.7031331062316895 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 4.7033257484436035 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 4.703278064727783 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 4.70325231552124 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 4.703062534332275 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 4.703256130218506 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 4.7032084465026855 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 4.703369617462158 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 4.70322322845459 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 4.703521728515625 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 4.703433513641357 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 4.703418254852295 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 4.703183650970459 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 4.70338249206543 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 4.702739238739014 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 4.703144550323486 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 4.7032670974731445 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 4.7031025886535645 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 4.703111171722412 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 4.7030510902404785 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 4.703457355499268 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 4.702816009521484 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 4.703540802001953 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 4.70344877243042 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 4.703365325927734 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 4.703069686889648 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 4.703339576721191 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 4.702874660491943 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 4.703277111053467 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 4.703598976135254 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 4.703429222106934 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 4.70334529876709 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 4.703056812286377 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 4.703105449676514 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 4.703003406524658 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 4.7033562660217285 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 4.703425884246826 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 4.703315258026123 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 4.703395843505859 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 4.703577518463135 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 4.7033162117004395 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 4.703385353088379 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 4.703244686126709 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 4.703118324279785 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 4.703269004821777 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 4.703227519989014 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 4.703094959259033 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 4.703187465667725 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 4.703117847442627 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 4.70308780670166 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 4.703098773956299 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 4.703523635864258 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 4.703177452087402 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 4.703394889831543 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 4.7035112380981445 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 4.703671455383301 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 4.7030534744262695 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 4.7031941413879395 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 4.703291416168213 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 4.703142166137695 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 4.703141212463379 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 4.703173637390137 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 4.703357696533203 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 4.703214645385742 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 4.7035088539123535 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 4.703372478485107 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 4.703258514404297 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 4.703344345092773 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 4.703479766845703 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 4.703032970428467 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 4.7032790184021 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 4.703478813171387 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 4.703563690185547 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 4.7031354904174805 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 4.7035112380981445 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 4.7034759521484375 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 4.703341960906982 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 4.703130722045898 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 4.7030816078186035 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 4.703278541564941 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 4.703341960906982 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 4.703061580657959 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 4.703128337860107 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 4.70302152633667 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 4.702645778656006 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 4.703374862670898 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 4.703371524810791 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 4.703172206878662 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 4.703670978546143 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 4.7036285400390625 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 4.703752040863037 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 4.703222751617432 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 4.703233242034912 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 4.702937602996826 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 4.70341682434082 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 4.703284740447998 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 4.70328426361084 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 4.703672885894775 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 4.703067779541016 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 4.703205108642578 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 4.703408241271973 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 4.703414440155029 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 4.703305244445801 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 4.702952861785889 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 4.7039642333984375 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 4.703002452850342 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 4.70338773727417 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 4.703513145446777 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 4.702845573425293 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 4.703558444976807 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 4.703225612640381 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 4.703124523162842 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 4.702138900756836 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 4.703087329864502 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 4.7032294273376465 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 4.703183174133301 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 4.703104019165039 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 4.702307224273682 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 4.702949523925781 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 4.7030863761901855 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 4.703282356262207 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 4.703244209289551 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 4.70322847366333 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 4.703383922576904 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 4.702417373657227 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 4.7028422355651855 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 4.701864242553711 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 4.70318603515625 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 4.703343391418457 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 4.703274726867676 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 4.7031941413879395 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 4.702741622924805 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 4.703006744384766 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 4.702725410461426 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 4.70323371887207 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 4.703474044799805 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 4.702986240386963 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 4.703278541564941 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 4.702480316162109 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 4.702804088592529 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 4.7029876708984375 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 4.702785968780518 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 4.703184127807617 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 4.702970027923584 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 4.7023797035217285 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 4.702879905700684 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 4.702817916870117 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 4.702239036560059 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 4.703287601470947 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 4.703435897827148 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 4.702757835388184 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 4.7027788162231445 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 4.702541351318359 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 4.703281402587891 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 4.702696323394775 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 4.703107833862305 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 4.703114986419678 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 4.702877521514893 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 4.703268527984619 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 4.702597618103027 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 4.703221797943115 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 4.701953887939453 + }, + { + "layer": "layer.32.self_attn.q_proj", + "ppl": 4.703347206115723 + }, + { + "layer": "layer.32.self_attn.k_proj", + "ppl": 4.703389644622803 + }, + { + "layer": "layer.32.self_attn.v_proj", + "ppl": 4.703434944152832 + }, + { + "layer": "layer.32.self_attn.o_proj", + "ppl": 4.70301628112793 + }, + { + "layer": "layer.32.mlp.gate_proj", + "ppl": 4.702639102935791 + }, + { + "layer": "layer.32.mlp.up_proj", + "ppl": 4.702162265777588 + }, + { + "layer": "layer.32.mlp.down_proj", + "ppl": 4.702122211456299 + }, + { + "layer": "layer.33.self_attn.q_proj", + "ppl": 4.703301906585693 + }, + { + "layer": "layer.33.self_attn.k_proj", + "ppl": 4.7036333084106445 + }, + { + "layer": "layer.33.self_attn.v_proj", + "ppl": 4.703132152557373 + }, + { + "layer": "layer.33.self_attn.o_proj", + "ppl": 4.703286170959473 + }, + { + "layer": "layer.33.mlp.gate_proj", + "ppl": 4.703076362609863 + }, + { + "layer": "layer.33.mlp.up_proj", + "ppl": 4.703280925750732 + }, + { + "layer": "layer.33.mlp.down_proj", + "ppl": 4.702764511108398 + }, + { + "layer": "layer.34.self_attn.q_proj", + "ppl": 4.703279972076416 + }, + { + "layer": "layer.34.self_attn.k_proj", + "ppl": 4.703351974487305 + }, + { + "layer": "layer.34.self_attn.v_proj", + "ppl": 4.702850341796875 + }, + { + "layer": "layer.34.self_attn.o_proj", + "ppl": 4.702753067016602 + }, + { + "layer": "layer.34.mlp.gate_proj", + "ppl": 4.70250129699707 + }, + { + "layer": "layer.34.mlp.up_proj", + "ppl": 4.702744007110596 + }, + { + "layer": "layer.34.mlp.down_proj", + "ppl": 4.701454162597656 + }, + { + "layer": "layer.35.self_attn.q_proj", + "ppl": 4.703097343444824 + }, + { + "layer": "layer.35.self_attn.k_proj", + "ppl": 4.7032599449157715 + }, + { + "layer": "layer.35.self_attn.v_proj", + "ppl": 4.703059673309326 + }, + { + "layer": "layer.35.self_attn.o_proj", + "ppl": 4.702810287475586 + }, + { + "layer": "layer.35.mlp.gate_proj", + "ppl": 4.702864170074463 + }, + { + "layer": "layer.35.mlp.up_proj", + "ppl": 4.7031402587890625 + }, + { + "layer": "layer.35.mlp.down_proj", + "ppl": 4.701979637145996 + }, + { + "layer": "layer.36.self_attn.q_proj", + "ppl": 4.702988624572754 + }, + { + "layer": "layer.36.self_attn.k_proj", + "ppl": 4.703162670135498 + }, + { + "layer": "layer.36.self_attn.v_proj", + "ppl": 4.702914714813232 + }, + { + "layer": "layer.36.self_attn.o_proj", + "ppl": 4.7030534744262695 + }, + { + "layer": "layer.36.mlp.gate_proj", + "ppl": 4.701683044433594 + }, + { + "layer": "layer.36.mlp.up_proj", + "ppl": 4.703291416168213 + }, + { + "layer": "layer.36.mlp.down_proj", + "ppl": 4.701124668121338 + }, + { + "layer": "layer.37.self_attn.q_proj", + "ppl": 4.703120231628418 + }, + { + "layer": "layer.37.self_attn.k_proj", + "ppl": 4.7031168937683105 + }, + { + "layer": "layer.37.self_attn.v_proj", + "ppl": 4.703031539916992 + }, + { + "layer": "layer.37.self_attn.o_proj", + "ppl": 4.703766822814941 + }, + { + "layer": "layer.37.mlp.gate_proj", + "ppl": 4.702314853668213 + }, + { + "layer": "layer.37.mlp.up_proj", + "ppl": 4.702218532562256 + }, + { + "layer": "layer.37.mlp.down_proj", + "ppl": 4.702141284942627 + }, + { + "layer": "layer.38.self_attn.q_proj", + "ppl": 4.703257083892822 + }, + { + "layer": "layer.38.self_attn.k_proj", + "ppl": 4.703250885009766 + }, + { + "layer": "layer.38.self_attn.v_proj", + "ppl": 4.703441619873047 + }, + { + "layer": "layer.38.self_attn.o_proj", + "ppl": 4.7026896476745605 + }, + { + "layer": "layer.38.mlp.gate_proj", + "ppl": 4.702215671539307 + }, + { + "layer": "layer.38.mlp.up_proj", + "ppl": 4.701983451843262 + }, + { + "layer": "layer.38.mlp.down_proj", + "ppl": 4.701565265655518 + }, + { + "layer": "layer.39.self_attn.q_proj", + "ppl": 4.702978134155273 + }, + { + "layer": "layer.39.self_attn.k_proj", + "ppl": 4.70322847366333 + }, + { + "layer": "layer.39.self_attn.v_proj", + "ppl": 4.703524589538574 + }, + { + "layer": "layer.39.self_attn.o_proj", + "ppl": 4.702984809875488 + }, + { + "layer": "layer.39.mlp.gate_proj", + "ppl": 4.702021598815918 + }, + { + "layer": "layer.39.mlp.up_proj", + "ppl": 4.702653408050537 + }, + { + "layer": "layer.39.mlp.down_proj", + "ppl": 4.7035932540893555 + }, + { + "layer": "layer.40.self_attn.q_proj", + "ppl": 4.702979564666748 + }, + { + "layer": "layer.40.self_attn.k_proj", + "ppl": 4.702965259552002 + }, + { + "layer": "layer.40.self_attn.v_proj", + "ppl": 4.70286226272583 + }, + { + "layer": "layer.40.self_attn.o_proj", + "ppl": 4.702841758728027 + }, + { + "layer": "layer.40.mlp.gate_proj", + "ppl": 4.702483654022217 + }, + { + "layer": "layer.40.mlp.up_proj", + "ppl": 4.703090190887451 + }, + { + "layer": "layer.40.mlp.down_proj", + "ppl": 4.702702522277832 + }, + { + "layer": "layer.41.self_attn.q_proj", + "ppl": 4.703005313873291 + }, + { + "layer": "layer.41.self_attn.k_proj", + "ppl": 4.703117847442627 + }, + { + "layer": "layer.41.self_attn.v_proj", + "ppl": 4.702739238739014 + }, + { + "layer": "layer.41.self_attn.o_proj", + "ppl": 4.702536582946777 + }, + { + "layer": "layer.41.mlp.gate_proj", + "ppl": 4.702151298522949 + }, + { + "layer": "layer.41.mlp.up_proj", + "ppl": 4.703165054321289 + }, + { + "layer": "layer.41.mlp.down_proj", + "ppl": 4.703639984130859 + }, + { + "layer": "layer.42.self_attn.q_proj", + "ppl": 4.703484058380127 + }, + { + "layer": "layer.42.self_attn.k_proj", + "ppl": 4.703462600708008 + }, + { + "layer": "layer.42.self_attn.v_proj", + "ppl": 4.703326225280762 + }, + { + "layer": "layer.42.self_attn.o_proj", + "ppl": 4.703029155731201 + }, + { + "layer": "layer.42.mlp.gate_proj", + "ppl": 4.702334403991699 + }, + { + "layer": "layer.42.mlp.up_proj", + "ppl": 4.702983379364014 + }, + { + "layer": "layer.42.mlp.down_proj", + "ppl": 4.70245885848999 + }, + { + "layer": "layer.43.self_attn.q_proj", + "ppl": 4.70327615737915 + }, + { + "layer": "layer.43.self_attn.k_proj", + "ppl": 4.7034077644348145 + }, + { + "layer": "layer.43.self_attn.v_proj", + "ppl": 4.703312873840332 + }, + { + "layer": "layer.43.self_attn.o_proj", + "ppl": 4.70300817489624 + }, + { + "layer": "layer.43.mlp.gate_proj", + "ppl": 4.70225191116333 + }, + { + "layer": "layer.43.mlp.up_proj", + "ppl": 4.701954364776611 + }, + { + "layer": "layer.43.mlp.down_proj", + "ppl": 4.703003406524658 + }, + { + "layer": "layer.44.self_attn.q_proj", + "ppl": 4.703310012817383 + }, + { + "layer": "layer.44.self_attn.k_proj", + "ppl": 4.703115463256836 + }, + { + "layer": "layer.44.self_attn.v_proj", + "ppl": 4.702903747558594 + }, + { + "layer": "layer.44.self_attn.o_proj", + "ppl": 4.702939510345459 + }, + { + "layer": "layer.44.mlp.gate_proj", + "ppl": 4.702269077301025 + }, + { + "layer": "layer.44.mlp.up_proj", + "ppl": 4.702744007110596 + }, + { + "layer": "layer.44.mlp.down_proj", + "ppl": 4.702698707580566 + }, + { + "layer": "layer.45.self_attn.q_proj", + "ppl": 4.703321933746338 + }, + { + "layer": "layer.45.self_attn.k_proj", + "ppl": 4.703393936157227 + }, + { + "layer": "layer.45.self_attn.v_proj", + "ppl": 4.703403949737549 + }, + { + "layer": "layer.45.self_attn.o_proj", + "ppl": 4.703192234039307 + }, + { + "layer": "layer.45.mlp.gate_proj", + "ppl": 4.702317237854004 + }, + { + "layer": "layer.45.mlp.up_proj", + "ppl": 4.702488422393799 + }, + { + "layer": "layer.45.mlp.down_proj", + "ppl": 4.7017388343811035 + }, + { + "layer": "layer.46.self_attn.q_proj", + "ppl": 4.702981472015381 + }, + { + "layer": "layer.46.self_attn.k_proj", + "ppl": 4.703325271606445 + }, + { + "layer": "layer.46.self_attn.v_proj", + "ppl": 4.7026848793029785 + }, + { + "layer": "layer.46.self_attn.o_proj", + "ppl": 4.703094959259033 + }, + { + "layer": "layer.46.mlp.gate_proj", + "ppl": 4.702292442321777 + }, + { + "layer": "layer.46.mlp.up_proj", + "ppl": 4.7024922370910645 + }, + { + "layer": "layer.46.mlp.down_proj", + "ppl": 4.701921463012695 + }, + { + "layer": "layer.47.self_attn.q_proj", + "ppl": 4.703390598297119 + }, + { + "layer": "layer.47.self_attn.k_proj", + "ppl": 4.703287601470947 + }, + { + "layer": "layer.47.self_attn.v_proj", + "ppl": 4.703284740447998 + }, + { + "layer": "layer.47.self_attn.o_proj", + "ppl": 4.702694892883301 + }, + { + "layer": "layer.47.mlp.gate_proj", + "ppl": 4.702643871307373 + }, + { + "layer": "layer.47.mlp.up_proj", + "ppl": 4.704004287719727 + }, + { + "layer": "layer.47.mlp.down_proj", + "ppl": 4.702408790588379 + }, + { + "layer": "layer.48.self_attn.q_proj", + "ppl": 4.703620433807373 + }, + { + "layer": "layer.48.self_attn.k_proj", + "ppl": 4.70320463180542 + }, + { + "layer": "layer.48.self_attn.v_proj", + "ppl": 4.703140735626221 + }, + { + "layer": "layer.48.self_attn.o_proj", + "ppl": 4.703004837036133 + }, + { + "layer": "layer.48.mlp.gate_proj", + "ppl": 4.702014446258545 + }, + { + "layer": "layer.48.mlp.up_proj", + "ppl": 4.702700138092041 + }, + { + "layer": "layer.48.mlp.down_proj", + "ppl": 4.701652526855469 + }, + { + "layer": "layer.49.self_attn.q_proj", + "ppl": 4.7030134201049805 + }, + { + "layer": "layer.49.self_attn.k_proj", + "ppl": 4.703118801116943 + }, + { + "layer": "layer.49.self_attn.v_proj", + "ppl": 4.703254699707031 + }, + { + "layer": "layer.49.self_attn.o_proj", + "ppl": 4.703144550323486 + }, + { + "layer": "layer.49.mlp.gate_proj", + "ppl": 4.702024936676025 + }, + { + "layer": "layer.49.mlp.up_proj", + "ppl": 4.703512191772461 + }, + { + "layer": "layer.49.mlp.down_proj", + "ppl": 4.702323913574219 + }, + { + "layer": "layer.50.self_attn.q_proj", + "ppl": 4.703512191772461 + }, + { + "layer": "layer.50.self_attn.k_proj", + "ppl": 4.702815055847168 + }, + { + "layer": "layer.50.self_attn.v_proj", + "ppl": 4.703616619110107 + }, + { + "layer": "layer.50.self_attn.o_proj", + "ppl": 4.7030558586120605 + }, + { + "layer": "layer.50.mlp.gate_proj", + "ppl": 4.702375888824463 + }, + { + "layer": "layer.50.mlp.up_proj", + "ppl": 4.702380657196045 + }, + { + "layer": "layer.50.mlp.down_proj", + "ppl": 4.701871871948242 + }, + { + "layer": "layer.51.self_attn.q_proj", + "ppl": 4.703871726989746 + }, + { + "layer": "layer.51.self_attn.k_proj", + "ppl": 4.7031354904174805 + }, + { + "layer": "layer.51.self_attn.v_proj", + "ppl": 4.7030487060546875 + }, + { + "layer": "layer.51.self_attn.o_proj", + "ppl": 4.70342493057251 + }, + { + "layer": "layer.51.mlp.gate_proj", + "ppl": 4.703195095062256 + }, + { + "layer": "layer.51.mlp.up_proj", + "ppl": 4.70297908782959 + }, + { + "layer": "layer.51.mlp.down_proj", + "ppl": 4.702330112457275 + }, + { + "layer": "layer.52.self_attn.q_proj", + "ppl": 4.7033305168151855 + }, + { + "layer": "layer.52.self_attn.k_proj", + "ppl": 4.702695369720459 + }, + { + "layer": "layer.52.self_attn.v_proj", + "ppl": 4.703861713409424 + }, + { + "layer": "layer.52.self_attn.o_proj", + "ppl": 4.703660488128662 + }, + { + "layer": "layer.52.mlp.gate_proj", + "ppl": 4.701773166656494 + }, + { + "layer": "layer.52.mlp.up_proj", + "ppl": 4.702525615692139 + }, + { + "layer": "layer.52.mlp.down_proj", + "ppl": 4.702807903289795 + }, + { + "layer": "layer.53.self_attn.q_proj", + "ppl": 4.702857494354248 + }, + { + "layer": "layer.53.self_attn.k_proj", + "ppl": 4.703433513641357 + }, + { + "layer": "layer.53.self_attn.v_proj", + "ppl": 4.703125476837158 + }, + { + "layer": "layer.53.self_attn.o_proj", + "ppl": 4.70264196395874 + }, + { + "layer": "layer.53.mlp.gate_proj", + "ppl": 4.701794624328613 + }, + { + "layer": "layer.53.mlp.up_proj", + "ppl": 4.70221471786499 + }, + { + "layer": "layer.53.mlp.down_proj", + "ppl": 4.702579498291016 + }, + { + "layer": "layer.54.self_attn.q_proj", + "ppl": 4.7038726806640625 + }, + { + "layer": "layer.54.self_attn.k_proj", + "ppl": 4.702638149261475 + }, + { + "layer": "layer.54.self_attn.v_proj", + "ppl": 4.70264196395874 + }, + { + "layer": "layer.54.self_attn.o_proj", + "ppl": 4.70283842086792 + }, + { + "layer": "layer.54.mlp.gate_proj", + "ppl": 4.703349590301514 + }, + { + "layer": "layer.54.mlp.up_proj", + "ppl": 4.7029337882995605 + }, + { + "layer": "layer.54.mlp.down_proj", + "ppl": 4.702788352966309 + }, + { + "layer": "layer.55.self_attn.q_proj", + "ppl": 4.70347261428833 + }, + { + "layer": "layer.55.self_attn.k_proj", + "ppl": 4.703734397888184 + }, + { + "layer": "layer.55.self_attn.v_proj", + "ppl": 4.7029547691345215 + }, + { + "layer": "layer.55.self_attn.o_proj", + "ppl": 4.703006744384766 + }, + { + "layer": "layer.55.mlp.gate_proj", + "ppl": 4.701948165893555 + }, + { + "layer": "layer.55.mlp.up_proj", + "ppl": 4.701875686645508 + }, + { + "layer": "layer.55.mlp.down_proj", + "ppl": 4.702054500579834 + }, + { + "layer": "layer.56.self_attn.q_proj", + "ppl": 4.702834606170654 + }, + { + "layer": "layer.56.self_attn.k_proj", + "ppl": 4.702661037445068 + }, + { + "layer": "layer.56.self_attn.v_proj", + "ppl": 4.702756881713867 + }, + { + "layer": "layer.56.self_attn.o_proj", + "ppl": 4.703113079071045 + }, + { + "layer": "layer.56.mlp.gate_proj", + "ppl": 4.701828479766846 + }, + { + "layer": "layer.56.mlp.up_proj", + "ppl": 4.703291416168213 + }, + { + "layer": "layer.56.mlp.down_proj", + "ppl": 4.702169418334961 + }, + { + "layer": "layer.57.self_attn.q_proj", + "ppl": 4.703401565551758 + }, + { + "layer": "layer.57.self_attn.k_proj", + "ppl": 4.703444004058838 + }, + { + "layer": "layer.57.self_attn.v_proj", + "ppl": 4.703201770782471 + }, + { + "layer": "layer.57.self_attn.o_proj", + "ppl": 4.702711582183838 + }, + { + "layer": "layer.57.mlp.gate_proj", + "ppl": 4.7019758224487305 + }, + { + "layer": "layer.57.mlp.up_proj", + "ppl": 4.702703952789307 + }, + { + "layer": "layer.57.mlp.down_proj", + "ppl": 4.702531337738037 + }, + { + "layer": "layer.58.self_attn.q_proj", + "ppl": 4.703091144561768 + }, + { + "layer": "layer.58.self_attn.k_proj", + "ppl": 4.703314304351807 + }, + { + "layer": "layer.58.self_attn.v_proj", + "ppl": 4.703861713409424 + }, + { + "layer": "layer.58.self_attn.o_proj", + "ppl": 4.703454971313477 + }, + { + "layer": "layer.58.mlp.gate_proj", + "ppl": 4.701364994049072 + }, + { + "layer": "layer.58.mlp.up_proj", + "ppl": 4.703061580657959 + }, + { + "layer": "layer.58.mlp.down_proj", + "ppl": 4.70422887802124 + }, + { + "layer": "layer.59.self_attn.q_proj", + "ppl": 4.7025580406188965 + }, + { + "layer": "layer.59.self_attn.k_proj", + "ppl": 4.703043460845947 + }, + { + "layer": "layer.59.self_attn.v_proj", + "ppl": 4.7027668952941895 + }, + { + "layer": "layer.59.self_attn.o_proj", + "ppl": 4.702225208282471 + }, + { + "layer": "layer.59.mlp.gate_proj", + "ppl": 4.70100212097168 + }, + { + "layer": "layer.59.mlp.up_proj", + "ppl": 4.701580047607422 + }, + { + "layer": "layer.59.mlp.down_proj", + "ppl": 4.701686859130859 + }, + { + "layer": "layer.60.self_attn.q_proj", + "ppl": 4.70307731628418 + }, + { + "layer": "layer.60.self_attn.k_proj", + "ppl": 4.703437328338623 + }, + { + "layer": "layer.60.self_attn.v_proj", + "ppl": 4.703333377838135 + }, + { + "layer": "layer.60.self_attn.o_proj", + "ppl": 4.702215671539307 + }, + { + "layer": "layer.60.mlp.gate_proj", + "ppl": 4.702242851257324 + }, + { + "layer": "layer.60.mlp.up_proj", + "ppl": 4.702499866485596 + }, + { + "layer": "layer.60.mlp.down_proj", + "ppl": 4.702337741851807 + }, + { + "layer": "layer.61.self_attn.q_proj", + "ppl": 4.703242301940918 + }, + { + "layer": "layer.61.self_attn.k_proj", + "ppl": 4.703224182128906 + }, + { + "layer": "layer.61.self_attn.v_proj", + "ppl": 4.702940464019775 + }, + { + "layer": "layer.61.self_attn.o_proj", + "ppl": 4.702701091766357 + }, + { + "layer": "layer.61.mlp.gate_proj", + "ppl": 4.700225830078125 + }, + { + "layer": "layer.61.mlp.up_proj", + "ppl": 4.702769756317139 + }, + { + "layer": "layer.61.mlp.down_proj", + "ppl": 4.701820373535156 + }, + { + "layer": "layer.62.self_attn.q_proj", + "ppl": 4.702284336090088 + }, + { + "layer": "layer.62.self_attn.k_proj", + "ppl": 4.702784061431885 + }, + { + "layer": "layer.62.self_attn.v_proj", + "ppl": 4.70287561416626 + }, + { + "layer": "layer.62.self_attn.o_proj", + "ppl": 4.702466011047363 + }, + { + "layer": "layer.62.mlp.gate_proj", + "ppl": 4.7015767097473145 + }, + { + "layer": "layer.62.mlp.up_proj", + "ppl": 4.701701641082764 + }, + { + "layer": "layer.62.mlp.down_proj", + "ppl": 4.701198101043701 + }, + { + "layer": "layer.63.self_attn.q_proj", + "ppl": 4.703149795532227 + }, + { + "layer": "layer.63.self_attn.k_proj", + "ppl": 4.7034406661987305 + }, + { + "layer": "layer.63.self_attn.v_proj", + "ppl": 4.703200340270996 + }, + { + "layer": "layer.63.self_attn.o_proj", + "ppl": 4.70326566696167 + }, + { + "layer": "layer.63.mlp.gate_proj", + "ppl": 4.700595378875732 + }, + { + "layer": "layer.63.mlp.up_proj", + "ppl": 4.703243732452393 + }, + { + "layer": "layer.63.mlp.down_proj", + "ppl": 4.701714038848877 + }, + { + "layer": "layer.64.self_attn.q_proj", + "ppl": 4.703149318695068 + }, + { + "layer": "layer.64.self_attn.k_proj", + "ppl": 4.703354835510254 + }, + { + "layer": "layer.64.self_attn.v_proj", + "ppl": 4.702543258666992 + }, + { + "layer": "layer.64.self_attn.o_proj", + "ppl": 4.702445983886719 + }, + { + "layer": "layer.64.mlp.gate_proj", + "ppl": 4.700601100921631 + }, + { + "layer": "layer.64.mlp.up_proj", + "ppl": 4.7024078369140625 + }, + { + "layer": "layer.64.mlp.down_proj", + "ppl": 4.701562881469727 + }, + { + "layer": "layer.65.self_attn.q_proj", + "ppl": 4.7031378746032715 + }, + { + "layer": "layer.65.self_attn.k_proj", + "ppl": 4.701893329620361 + }, + { + "layer": "layer.65.self_attn.v_proj", + "ppl": 4.702304840087891 + }, + { + "layer": "layer.65.self_attn.o_proj", + "ppl": 4.7028608322143555 + }, + { + "layer": "layer.65.mlp.gate_proj", + "ppl": 4.700300693511963 + }, + { + "layer": "layer.65.mlp.up_proj", + "ppl": 4.702119827270508 + }, + { + "layer": "layer.65.mlp.down_proj", + "ppl": 4.702775001525879 + }, + { + "layer": "layer.66.self_attn.q_proj", + "ppl": 4.70318603515625 + }, + { + "layer": "layer.66.self_attn.k_proj", + "ppl": 4.703761100769043 + }, + { + "layer": "layer.66.self_attn.v_proj", + "ppl": 4.703184127807617 + }, + { + "layer": "layer.66.self_attn.o_proj", + "ppl": 4.702908515930176 + }, + { + "layer": "layer.66.mlp.gate_proj", + "ppl": 4.701323986053467 + }, + { + "layer": "layer.66.mlp.up_proj", + "ppl": 4.702622890472412 + }, + { + "layer": "layer.66.mlp.down_proj", + "ppl": 4.701777935028076 + }, + { + "layer": "layer.67.self_attn.q_proj", + "ppl": 4.703455924987793 + }, + { + "layer": "layer.67.self_attn.k_proj", + "ppl": 4.703369617462158 + }, + { + "layer": "layer.67.self_attn.v_proj", + "ppl": 4.703048229217529 + }, + { + "layer": "layer.67.self_attn.o_proj", + "ppl": 4.703493118286133 + }, + { + "layer": "layer.67.mlp.gate_proj", + "ppl": 4.700704574584961 + }, + { + "layer": "layer.67.mlp.up_proj", + "ppl": 4.702339172363281 + }, + { + "layer": "layer.67.mlp.down_proj", + "ppl": 4.701903343200684 + }, + { + "layer": "layer.68.self_attn.q_proj", + "ppl": 4.702698230743408 + }, + { + "layer": "layer.68.self_attn.k_proj", + "ppl": 4.702816009521484 + }, + { + "layer": "layer.68.self_attn.v_proj", + "ppl": 4.70266056060791 + }, + { + "layer": "layer.68.self_attn.o_proj", + "ppl": 4.702650547027588 + }, + { + "layer": "layer.68.mlp.gate_proj", + "ppl": 4.699793338775635 + }, + { + "layer": "layer.68.mlp.up_proj", + "ppl": 4.7021894454956055 + }, + { + "layer": "layer.68.mlp.down_proj", + "ppl": 4.702121257781982 + }, + { + "layer": "layer.69.self_attn.q_proj", + "ppl": 4.703423023223877 + }, + { + "layer": "layer.69.self_attn.k_proj", + "ppl": 4.703505516052246 + }, + { + "layer": "layer.69.self_attn.v_proj", + "ppl": 4.703286170959473 + }, + { + "layer": "layer.69.self_attn.o_proj", + "ppl": 4.703364849090576 + }, + { + "layer": "layer.69.mlp.gate_proj", + "ppl": 4.699944972991943 + }, + { + "layer": "layer.69.mlp.up_proj", + "ppl": 4.702324867248535 + }, + { + "layer": "layer.69.mlp.down_proj", + "ppl": 4.702025413513184 + }, + { + "layer": "layer.70.self_attn.q_proj", + "ppl": 4.703212738037109 + }, + { + "layer": "layer.70.self_attn.k_proj", + "ppl": 4.703370094299316 + }, + { + "layer": "layer.70.self_attn.v_proj", + "ppl": 4.702754497528076 + }, + { + "layer": "layer.70.self_attn.o_proj", + "ppl": 4.703029155731201 + }, + { + "layer": "layer.70.mlp.gate_proj", + "ppl": 4.6996283531188965 + }, + { + "layer": "layer.70.mlp.up_proj", + "ppl": 4.7017822265625 + }, + { + "layer": "layer.70.mlp.down_proj", + "ppl": 4.701957702636719 + }, + { + "layer": "layer.71.self_attn.q_proj", + "ppl": 4.704036235809326 + }, + { + "layer": "layer.71.self_attn.k_proj", + "ppl": 4.703591823577881 + }, + { + "layer": "layer.71.self_attn.v_proj", + "ppl": 4.703819274902344 + }, + { + "layer": "layer.71.self_attn.o_proj", + "ppl": 4.7032647132873535 + }, + { + "layer": "layer.71.mlp.gate_proj", + "ppl": 4.7004008293151855 + }, + { + "layer": "layer.71.mlp.up_proj", + "ppl": 4.701968193054199 + }, + { + "layer": "layer.71.mlp.down_proj", + "ppl": 4.702554702758789 + }, + { + "layer": "layer.72.self_attn.q_proj", + "ppl": 4.702734470367432 + }, + { + "layer": "layer.72.self_attn.k_proj", + "ppl": 4.703309059143066 + }, + { + "layer": "layer.72.self_attn.v_proj", + "ppl": 4.702672481536865 + }, + { + "layer": "layer.72.self_attn.o_proj", + "ppl": 4.70313024520874 + }, + { + "layer": "layer.72.mlp.gate_proj", + "ppl": 4.699913501739502 + }, + { + "layer": "layer.72.mlp.up_proj", + "ppl": 4.701173782348633 + }, + { + "layer": "layer.72.mlp.down_proj", + "ppl": 4.7007975578308105 + }, + { + "layer": "layer.73.self_attn.q_proj", + "ppl": 4.7029876708984375 + }, + { + "layer": "layer.73.self_attn.k_proj", + "ppl": 4.703234672546387 + }, + { + "layer": "layer.73.self_attn.v_proj", + "ppl": 4.703858852386475 + }, + { + "layer": "layer.73.self_attn.o_proj", + "ppl": 4.703519344329834 + }, + { + "layer": "layer.73.mlp.gate_proj", + "ppl": 4.701331615447998 + }, + { + "layer": "layer.73.mlp.up_proj", + "ppl": 4.701567649841309 + }, + { + "layer": "layer.73.mlp.down_proj", + "ppl": 4.70255184173584 + }, + { + "layer": "layer.74.self_attn.q_proj", + "ppl": 4.703307151794434 + }, + { + "layer": "layer.74.self_attn.k_proj", + "ppl": 4.7032365798950195 + }, + { + "layer": "layer.74.self_attn.v_proj", + "ppl": 4.702394485473633 + }, + { + "layer": "layer.74.self_attn.o_proj", + "ppl": 4.703299045562744 + }, + { + "layer": "layer.74.mlp.gate_proj", + "ppl": 4.7004899978637695 + }, + { + "layer": "layer.74.mlp.up_proj", + "ppl": 4.701378345489502 + }, + { + "layer": "layer.74.mlp.down_proj", + "ppl": 4.702141761779785 + }, + { + "layer": "layer.75.self_attn.q_proj", + "ppl": 4.7042059898376465 + }, + { + "layer": "layer.75.self_attn.k_proj", + "ppl": 4.703385353088379 + }, + { + "layer": "layer.75.self_attn.v_proj", + "ppl": 4.7026591300964355 + }, + { + "layer": "layer.75.self_attn.o_proj", + "ppl": 4.703097820281982 + }, + { + "layer": "layer.75.mlp.gate_proj", + "ppl": 4.69966459274292 + }, + { + "layer": "layer.75.mlp.up_proj", + "ppl": 4.69358491897583 + }, + { + "layer": "layer.75.mlp.down_proj", + "ppl": 4.700589656829834 + }, + { + "layer": "layer.76.self_attn.q_proj", + "ppl": 4.703497409820557 + }, + { + "layer": "layer.76.self_attn.k_proj", + "ppl": 4.703453063964844 + }, + { + "layer": "layer.76.self_attn.v_proj", + "ppl": 4.7044172286987305 + }, + { + "layer": "layer.76.self_attn.o_proj", + "ppl": 4.703649997711182 + }, + { + "layer": "layer.76.mlp.gate_proj", + "ppl": 4.699149131774902 + }, + { + "layer": "layer.76.mlp.up_proj", + "ppl": 4.694709777832031 + }, + { + "layer": "layer.76.mlp.down_proj", + "ppl": 4.70059061050415 + }, + { + "layer": "layer.77.self_attn.q_proj", + "ppl": 4.704280853271484 + }, + { + "layer": "layer.77.self_attn.k_proj", + "ppl": 4.703935146331787 + }, + { + "layer": "layer.77.self_attn.v_proj", + "ppl": 4.703520774841309 + }, + { + "layer": "layer.77.self_attn.o_proj", + "ppl": 4.70346736907959 + }, + { + "layer": "layer.77.mlp.gate_proj", + "ppl": 4.698714256286621 + }, + { + "layer": "layer.77.mlp.up_proj", + "ppl": 4.700464248657227 + }, + { + "layer": "layer.77.mlp.down_proj", + "ppl": 4.700931549072266 + }, + { + "layer": "layer.78.self_attn.q_proj", + "ppl": 4.7030816078186035 + }, + { + "layer": "layer.78.self_attn.k_proj", + "ppl": 4.703239917755127 + }, + { + "layer": "layer.78.self_attn.v_proj", + "ppl": 4.703450679779053 + }, + { + "layer": "layer.78.self_attn.o_proj", + "ppl": 4.70493221282959 + }, + { + "layer": "layer.78.mlp.gate_proj", + "ppl": 4.698056221008301 + }, + { + "layer": "layer.78.mlp.up_proj", + "ppl": 4.697591781616211 + }, + { + "layer": "layer.78.mlp.down_proj", + "ppl": 4.699810981750488 + }, + { + "layer": "layer.79.self_attn.q_proj", + "ppl": 4.702572822570801 + }, + { + "layer": "layer.79.self_attn.k_proj", + "ppl": 4.703275203704834 + }, + { + "layer": "layer.79.self_attn.v_proj", + "ppl": 4.704037666320801 + }, + { + "layer": "layer.79.self_attn.o_proj", + "ppl": 4.704761028289795 + }, + { + "layer": "layer.79.mlp.gate_proj", + "ppl": 4.693142890930176 + }, + { + "layer": "layer.79.mlp.up_proj", + "ppl": 4.690267086029053 + }, + { + "layer": "layer.79.mlp.down_proj", + "ppl": 4.683718681335449 + } + ] + }, + { + "bit": 5, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 4.400390625 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 4.40055513381958 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 4.400437355041504 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 4.400333881378174 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 4.400465965270996 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 4.4006547927856445 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 4.400594711303711 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 4.400410175323486 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 4.400452136993408 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 4.4005279541015625 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 4.400238037109375 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 4.400423049926758 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 4.400608062744141 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 4.400378704071045 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 4.400341987609863 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 4.4004902839660645 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 4.400392532348633 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 4.40032958984375 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 4.400373458862305 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 4.400312900543213 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 4.40040922164917 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 4.400397777557373 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 4.400390625 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 4.4004316329956055 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 4.400476455688477 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 4.400415420532227 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 4.400458335876465 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 4.400333881378174 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 4.400344371795654 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 4.400398254394531 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 4.4002203941345215 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 4.400353908538818 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 4.400375843048096 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 4.400396823883057 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 4.400330543518066 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 4.400407791137695 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 4.400483131408691 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 4.40031623840332 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 4.400507926940918 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 4.400445938110352 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 4.400388240814209 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 4.400382041931152 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 4.40026330947876 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 4.400371074676514 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 4.400404930114746 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 4.400446891784668 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 4.400341987609863 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 4.400282382965088 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 4.4001030921936035 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 4.400406360626221 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 4.4003801345825195 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 4.400650501251221 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 4.400394439697266 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 4.4003801345825195 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 4.400292873382568 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 4.400472640991211 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 4.40036153793335 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 4.400323390960693 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 4.400320053100586 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 4.400410175323486 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 4.400475025177002 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 4.400300979614258 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 4.400426387786865 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 4.4003777503967285 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 4.400312900543213 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 4.400453567504883 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 4.400453567504883 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 4.400465965270996 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 4.400355815887451 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 4.400450706481934 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 4.400443077087402 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 4.400437355041504 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 4.400363445281982 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 4.400357723236084 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 4.4005560874938965 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 4.400339126586914 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 4.400308132171631 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 4.400321960449219 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 4.400395393371582 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 4.4005303382873535 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 4.400197505950928 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 4.400353908538818 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 4.400364875793457 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 4.400373458862305 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 4.400428771972656 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 4.4003586769104 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 4.40047025680542 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 4.400611400604248 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 4.400455474853516 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 4.400524616241455 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 4.400384902954102 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 4.400432586669922 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 4.4004716873168945 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 4.400397777557373 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 4.400181293487549 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 4.40041971206665 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 4.4003424644470215 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 4.400404930114746 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 4.400513172149658 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 4.400387287139893 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 4.40040922164917 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 4.400340557098389 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 4.400434494018555 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 4.400352954864502 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 4.400454521179199 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 4.400374412536621 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 4.400437355041504 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 4.400423049926758 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 4.40037202835083 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 4.400390625 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 4.400409698486328 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 4.400277137756348 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 4.40036678314209 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 4.400401592254639 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 4.400384902954102 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 4.400390625 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 4.4003400802612305 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 4.400335311889648 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 4.400554180145264 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 4.400336742401123 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 4.400256156921387 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 4.400384902954102 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 4.400430202484131 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 4.400476455688477 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 4.400179862976074 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 4.40006160736084 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 4.40019416809082 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 4.400287628173828 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 4.400457382202148 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 4.400310039520264 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 4.400264263153076 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 4.400424003601074 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 4.400069713592529 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 4.4005446434021 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 4.40047025680542 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 4.400341987609863 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 4.400407314300537 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 4.400193214416504 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 4.400727272033691 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 4.400063991546631 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 4.400357723236084 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 4.400351524353027 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 4.400506973266602 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 4.400518894195557 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 4.400162220001221 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 4.4003987312316895 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 4.400366306304932 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 4.400439739227295 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 4.400447845458984 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 4.4004807472229 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 4.4004292488098145 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 4.400267601013184 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 4.400115489959717 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 4.400081634521484 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 4.400266170501709 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 4.400534629821777 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 4.400016784667969 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 4.400341987609863 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 4.4004597663879395 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 4.400429725646973 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 4.400396823883057 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 4.400400638580322 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 4.400345325469971 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 4.400286674499512 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 4.400158405303955 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 4.40045690536499 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 4.400325298309326 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 4.400112152099609 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 4.400511741638184 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 4.400367259979248 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 4.400387287139893 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 4.40031623840332 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 4.400051593780518 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 4.400306701660156 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 4.400041103363037 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 4.400290489196777 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 4.400443077087402 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 4.40032434463501 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 4.400467872619629 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 4.4004740715026855 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 4.400299072265625 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 4.400023460388184 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 4.400328159332275 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 4.400271415710449 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 4.400241374969482 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 4.40017032623291 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 4.400514125823975 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 4.400231838226318 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 4.400531768798828 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 4.40050745010376 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 4.400296211242676 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 4.400424957275391 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 4.400335788726807 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 4.400115489959717 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 4.400215148925781 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 4.399913311004639 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 4.400550842285156 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 4.400333881378174 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 4.400460720062256 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 4.40015172958374 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 4.399961948394775 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 4.400338649749756 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 4.399946689605713 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 4.4002580642700195 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 4.400136470794678 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 4.400175094604492 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 4.400381088256836 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 4.4002532958984375 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 4.400417804718018 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 4.400146961212158 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 4.40041446685791 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 4.400355815887451 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 4.400173187255859 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 4.400341987609863 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 4.40019416809082 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 4.40018892288208 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 4.399850368499756 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 4.4002299308776855 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 4.400254726409912 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 4.400088310241699 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 4.400145530700684 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 4.400420665740967 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 4.4002885818481445 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 4.400421619415283 + }, + { + "layer": "layer.32.self_attn.q_proj", + "ppl": 4.400523662567139 + }, + { + "layer": "layer.32.self_attn.k_proj", + "ppl": 4.400323390960693 + }, + { + "layer": "layer.32.self_attn.v_proj", + "ppl": 4.400354385375977 + }, + { + "layer": "layer.32.self_attn.o_proj", + "ppl": 4.400407314300537 + }, + { + "layer": "layer.32.mlp.gate_proj", + "ppl": 4.40018367767334 + }, + { + "layer": "layer.32.mlp.up_proj", + "ppl": 4.400731086730957 + }, + { + "layer": "layer.32.mlp.down_proj", + "ppl": 4.399999141693115 + }, + { + "layer": "layer.33.self_attn.q_proj", + "ppl": 4.400435924530029 + }, + { + "layer": "layer.33.self_attn.k_proj", + "ppl": 4.400431156158447 + }, + { + "layer": "layer.33.self_attn.v_proj", + "ppl": 4.400283336639404 + }, + { + "layer": "layer.33.self_attn.o_proj", + "ppl": 4.400202751159668 + }, + { + "layer": "layer.33.mlp.gate_proj", + "ppl": 4.399627685546875 + }, + { + "layer": "layer.33.mlp.up_proj", + "ppl": 4.400363445281982 + }, + { + "layer": "layer.33.mlp.down_proj", + "ppl": 4.399801731109619 + }, + { + "layer": "layer.34.self_attn.q_proj", + "ppl": 4.400497913360596 + }, + { + "layer": "layer.34.self_attn.k_proj", + "ppl": 4.400301933288574 + }, + { + "layer": "layer.34.self_attn.v_proj", + "ppl": 4.400141716003418 + }, + { + "layer": "layer.34.self_attn.o_proj", + "ppl": 4.40055513381958 + }, + { + "layer": "layer.34.mlp.gate_proj", + "ppl": 4.399469375610352 + }, + { + "layer": "layer.34.mlp.up_proj", + "ppl": 4.400435447692871 + }, + { + "layer": "layer.34.mlp.down_proj", + "ppl": 4.4004807472229 + }, + { + "layer": "layer.35.self_attn.q_proj", + "ppl": 4.400201320648193 + }, + { + "layer": "layer.35.self_attn.k_proj", + "ppl": 4.400416851043701 + }, + { + "layer": "layer.35.self_attn.v_proj", + "ppl": 4.400318622589111 + }, + { + "layer": "layer.35.self_attn.o_proj", + "ppl": 4.400071144104004 + }, + { + "layer": "layer.35.mlp.gate_proj", + "ppl": 4.39984130859375 + }, + { + "layer": "layer.35.mlp.up_proj", + "ppl": 4.400118350982666 + }, + { + "layer": "layer.35.mlp.down_proj", + "ppl": 4.399642467498779 + }, + { + "layer": "layer.36.self_attn.q_proj", + "ppl": 4.400259494781494 + }, + { + "layer": "layer.36.self_attn.k_proj", + "ppl": 4.400277137756348 + }, + { + "layer": "layer.36.self_attn.v_proj", + "ppl": 4.400401592254639 + }, + { + "layer": "layer.36.self_attn.o_proj", + "ppl": 4.400241851806641 + }, + { + "layer": "layer.36.mlp.gate_proj", + "ppl": 4.4001383781433105 + }, + { + "layer": "layer.36.mlp.up_proj", + "ppl": 4.399731159210205 + }, + { + "layer": "layer.36.mlp.down_proj", + "ppl": 4.400991916656494 + }, + { + "layer": "layer.37.self_attn.q_proj", + "ppl": 4.400244235992432 + }, + { + "layer": "layer.37.self_attn.k_proj", + "ppl": 4.400452613830566 + }, + { + "layer": "layer.37.self_attn.v_proj", + "ppl": 4.400294780731201 + }, + { + "layer": "layer.37.self_attn.o_proj", + "ppl": 4.400594711303711 + }, + { + "layer": "layer.37.mlp.gate_proj", + "ppl": 4.399487018585205 + }, + { + "layer": "layer.37.mlp.up_proj", + "ppl": 4.4002180099487305 + }, + { + "layer": "layer.37.mlp.down_proj", + "ppl": 4.399896621704102 + }, + { + "layer": "layer.38.self_attn.q_proj", + "ppl": 4.4004669189453125 + }, + { + "layer": "layer.38.self_attn.k_proj", + "ppl": 4.400420188903809 + }, + { + "layer": "layer.38.self_attn.v_proj", + "ppl": 4.400384902954102 + }, + { + "layer": "layer.38.self_attn.o_proj", + "ppl": 4.4001946449279785 + }, + { + "layer": "layer.38.mlp.gate_proj", + "ppl": 4.4000244140625 + }, + { + "layer": "layer.38.mlp.up_proj", + "ppl": 4.399868965148926 + }, + { + "layer": "layer.38.mlp.down_proj", + "ppl": 4.3997907638549805 + }, + { + "layer": "layer.39.self_attn.q_proj", + "ppl": 4.400249004364014 + }, + { + "layer": "layer.39.self_attn.k_proj", + "ppl": 4.400387287139893 + }, + { + "layer": "layer.39.self_attn.v_proj", + "ppl": 4.400209903717041 + }, + { + "layer": "layer.39.self_attn.o_proj", + "ppl": 4.400175094604492 + }, + { + "layer": "layer.39.mlp.gate_proj", + "ppl": 4.399832248687744 + }, + { + "layer": "layer.39.mlp.up_proj", + "ppl": 4.400378704071045 + }, + { + "layer": "layer.39.mlp.down_proj", + "ppl": 4.40001106262207 + }, + { + "layer": "layer.40.self_attn.q_proj", + "ppl": 4.400210857391357 + }, + { + "layer": "layer.40.self_attn.k_proj", + "ppl": 4.400156497955322 + }, + { + "layer": "layer.40.self_attn.v_proj", + "ppl": 4.400087356567383 + }, + { + "layer": "layer.40.self_attn.o_proj", + "ppl": 4.400285243988037 + }, + { + "layer": "layer.40.mlp.gate_proj", + "ppl": 4.400151252746582 + }, + { + "layer": "layer.40.mlp.up_proj", + "ppl": 4.400268077850342 + }, + { + "layer": "layer.40.mlp.down_proj", + "ppl": 4.399960994720459 + }, + { + "layer": "layer.41.self_attn.q_proj", + "ppl": 4.400315284729004 + }, + { + "layer": "layer.41.self_attn.k_proj", + "ppl": 4.400419235229492 + }, + { + "layer": "layer.41.self_attn.v_proj", + "ppl": 4.40065860748291 + }, + { + "layer": "layer.41.self_attn.o_proj", + "ppl": 4.400251865386963 + }, + { + "layer": "layer.41.mlp.gate_proj", + "ppl": 4.399377822875977 + }, + { + "layer": "layer.41.mlp.up_proj", + "ppl": 4.399932861328125 + }, + { + "layer": "layer.41.mlp.down_proj", + "ppl": 4.400606632232666 + }, + { + "layer": "layer.42.self_attn.q_proj", + "ppl": 4.400357723236084 + }, + { + "layer": "layer.42.self_attn.k_proj", + "ppl": 4.400369644165039 + }, + { + "layer": "layer.42.self_attn.v_proj", + "ppl": 4.400436878204346 + }, + { + "layer": "layer.42.self_attn.o_proj", + "ppl": 4.400356769561768 + }, + { + "layer": "layer.42.mlp.gate_proj", + "ppl": 4.399829387664795 + }, + { + "layer": "layer.42.mlp.up_proj", + "ppl": 4.399642467498779 + }, + { + "layer": "layer.42.mlp.down_proj", + "ppl": 4.399199962615967 + }, + { + "layer": "layer.43.self_attn.q_proj", + "ppl": 4.400346279144287 + }, + { + "layer": "layer.43.self_attn.k_proj", + "ppl": 4.400445938110352 + }, + { + "layer": "layer.43.self_attn.v_proj", + "ppl": 4.400411128997803 + }, + { + "layer": "layer.43.self_attn.o_proj", + "ppl": 4.400651454925537 + }, + { + "layer": "layer.43.mlp.gate_proj", + "ppl": 4.400179862976074 + }, + { + "layer": "layer.43.mlp.up_proj", + "ppl": 4.400649547576904 + }, + { + "layer": "layer.43.mlp.down_proj", + "ppl": 4.399278163909912 + }, + { + "layer": "layer.44.self_attn.q_proj", + "ppl": 4.400617599487305 + }, + { + "layer": "layer.44.self_attn.k_proj", + "ppl": 4.400423049926758 + }, + { + "layer": "layer.44.self_attn.v_proj", + "ppl": 4.400688648223877 + }, + { + "layer": "layer.44.self_attn.o_proj", + "ppl": 4.400416851043701 + }, + { + "layer": "layer.44.mlp.gate_proj", + "ppl": 4.400016784667969 + }, + { + "layer": "layer.44.mlp.up_proj", + "ppl": 4.3998589515686035 + }, + { + "layer": "layer.44.mlp.down_proj", + "ppl": 4.399917125701904 + }, + { + "layer": "layer.45.self_attn.q_proj", + "ppl": 4.400599479675293 + }, + { + "layer": "layer.45.self_attn.k_proj", + "ppl": 4.400425434112549 + }, + { + "layer": "layer.45.self_attn.v_proj", + "ppl": 4.400477409362793 + }, + { + "layer": "layer.45.self_attn.o_proj", + "ppl": 4.400374412536621 + }, + { + "layer": "layer.45.mlp.gate_proj", + "ppl": 4.399704933166504 + }, + { + "layer": "layer.45.mlp.up_proj", + "ppl": 4.400033473968506 + }, + { + "layer": "layer.45.mlp.down_proj", + "ppl": 4.400336742401123 + }, + { + "layer": "layer.46.self_attn.q_proj", + "ppl": 4.400257110595703 + }, + { + "layer": "layer.46.self_attn.k_proj", + "ppl": 4.400391101837158 + }, + { + "layer": "layer.46.self_attn.v_proj", + "ppl": 4.400492191314697 + }, + { + "layer": "layer.46.self_attn.o_proj", + "ppl": 4.4001569747924805 + }, + { + "layer": "layer.46.mlp.gate_proj", + "ppl": 4.399605751037598 + }, + { + "layer": "layer.46.mlp.up_proj", + "ppl": 4.399730682373047 + }, + { + "layer": "layer.46.mlp.down_proj", + "ppl": 4.3998613357543945 + }, + { + "layer": "layer.47.self_attn.q_proj", + "ppl": 4.400389671325684 + }, + { + "layer": "layer.47.self_attn.k_proj", + "ppl": 4.4003777503967285 + }, + { + "layer": "layer.47.self_attn.v_proj", + "ppl": 4.400640487670898 + }, + { + "layer": "layer.47.self_attn.o_proj", + "ppl": 4.400361061096191 + }, + { + "layer": "layer.47.mlp.gate_proj", + "ppl": 4.400395393371582 + }, + { + "layer": "layer.47.mlp.up_proj", + "ppl": 4.399893283843994 + }, + { + "layer": "layer.47.mlp.down_proj", + "ppl": 4.40023946762085 + }, + { + "layer": "layer.48.self_attn.q_proj", + "ppl": 4.400772571563721 + }, + { + "layer": "layer.48.self_attn.k_proj", + "ppl": 4.400274753570557 + }, + { + "layer": "layer.48.self_attn.v_proj", + "ppl": 4.400537967681885 + }, + { + "layer": "layer.48.self_attn.o_proj", + "ppl": 4.399847507476807 + }, + { + "layer": "layer.48.mlp.gate_proj", + "ppl": 4.399853229522705 + }, + { + "layer": "layer.48.mlp.up_proj", + "ppl": 4.399958610534668 + }, + { + "layer": "layer.48.mlp.down_proj", + "ppl": 4.400249004364014 + }, + { + "layer": "layer.49.self_attn.q_proj", + "ppl": 4.400669574737549 + }, + { + "layer": "layer.49.self_attn.k_proj", + "ppl": 4.400195121765137 + }, + { + "layer": "layer.49.self_attn.v_proj", + "ppl": 4.400571823120117 + }, + { + "layer": "layer.49.self_attn.o_proj", + "ppl": 4.400205612182617 + }, + { + "layer": "layer.49.mlp.gate_proj", + "ppl": 4.399924278259277 + }, + { + "layer": "layer.49.mlp.up_proj", + "ppl": 4.400060176849365 + }, + { + "layer": "layer.49.mlp.down_proj", + "ppl": 4.399766445159912 + }, + { + "layer": "layer.50.self_attn.q_proj", + "ppl": 4.400313854217529 + }, + { + "layer": "layer.50.self_attn.k_proj", + "ppl": 4.400323390960693 + }, + { + "layer": "layer.50.self_attn.v_proj", + "ppl": 4.4003496170043945 + }, + { + "layer": "layer.50.self_attn.o_proj", + "ppl": 4.400964736938477 + }, + { + "layer": "layer.50.mlp.gate_proj", + "ppl": 4.400355815887451 + }, + { + "layer": "layer.50.mlp.up_proj", + "ppl": 4.399549961090088 + }, + { + "layer": "layer.50.mlp.down_proj", + "ppl": 4.400534152984619 + }, + { + "layer": "layer.51.self_attn.q_proj", + "ppl": 4.400497913360596 + }, + { + "layer": "layer.51.self_attn.k_proj", + "ppl": 4.400521278381348 + }, + { + "layer": "layer.51.self_attn.v_proj", + "ppl": 4.400188446044922 + }, + { + "layer": "layer.51.self_attn.o_proj", + "ppl": 4.40049409866333 + }, + { + "layer": "layer.51.mlp.gate_proj", + "ppl": 4.399435997009277 + }, + { + "layer": "layer.51.mlp.up_proj", + "ppl": 4.400214195251465 + }, + { + "layer": "layer.51.mlp.down_proj", + "ppl": 4.399816989898682 + }, + { + "layer": "layer.52.self_attn.q_proj", + "ppl": 4.400211811065674 + }, + { + "layer": "layer.52.self_attn.k_proj", + "ppl": 4.400047302246094 + }, + { + "layer": "layer.52.self_attn.v_proj", + "ppl": 4.400199890136719 + }, + { + "layer": "layer.52.self_attn.o_proj", + "ppl": 4.4000091552734375 + }, + { + "layer": "layer.52.mlp.gate_proj", + "ppl": 4.399576187133789 + }, + { + "layer": "layer.52.mlp.up_proj", + "ppl": 4.3995771408081055 + }, + { + "layer": "layer.52.mlp.down_proj", + "ppl": 4.399689674377441 + }, + { + "layer": "layer.53.self_attn.q_proj", + "ppl": 4.40056037902832 + }, + { + "layer": "layer.53.self_attn.k_proj", + "ppl": 4.400350570678711 + }, + { + "layer": "layer.53.self_attn.v_proj", + "ppl": 4.400546550750732 + }, + { + "layer": "layer.53.self_attn.o_proj", + "ppl": 4.400061130523682 + }, + { + "layer": "layer.53.mlp.gate_proj", + "ppl": 4.399815559387207 + }, + { + "layer": "layer.53.mlp.up_proj", + "ppl": 4.40047025680542 + }, + { + "layer": "layer.53.mlp.down_proj", + "ppl": 4.400304317474365 + }, + { + "layer": "layer.54.self_attn.q_proj", + "ppl": 4.400269985198975 + }, + { + "layer": "layer.54.self_attn.k_proj", + "ppl": 4.400017738342285 + }, + { + "layer": "layer.54.self_attn.v_proj", + "ppl": 4.400537014007568 + }, + { + "layer": "layer.54.self_attn.o_proj", + "ppl": 4.400290012359619 + }, + { + "layer": "layer.54.mlp.gate_proj", + "ppl": 4.400187015533447 + }, + { + "layer": "layer.54.mlp.up_proj", + "ppl": 4.399503707885742 + }, + { + "layer": "layer.54.mlp.down_proj", + "ppl": 4.4002790451049805 + }, + { + "layer": "layer.55.self_attn.q_proj", + "ppl": 4.400158882141113 + }, + { + "layer": "layer.55.self_attn.k_proj", + "ppl": 4.400456428527832 + }, + { + "layer": "layer.55.self_attn.v_proj", + "ppl": 4.400277137756348 + }, + { + "layer": "layer.55.self_attn.o_proj", + "ppl": 4.400026798248291 + }, + { + "layer": "layer.55.mlp.gate_proj", + "ppl": 4.399533271789551 + }, + { + "layer": "layer.55.mlp.up_proj", + "ppl": 4.400207042694092 + }, + { + "layer": "layer.55.mlp.down_proj", + "ppl": 4.399667263031006 + }, + { + "layer": "layer.56.self_attn.q_proj", + "ppl": 4.4002509117126465 + }, + { + "layer": "layer.56.self_attn.k_proj", + "ppl": 4.400256633758545 + }, + { + "layer": "layer.56.self_attn.v_proj", + "ppl": 4.400455951690674 + }, + { + "layer": "layer.56.self_attn.o_proj", + "ppl": 4.399827003479004 + }, + { + "layer": "layer.56.mlp.gate_proj", + "ppl": 4.399483680725098 + }, + { + "layer": "layer.56.mlp.up_proj", + "ppl": 4.399956226348877 + }, + { + "layer": "layer.56.mlp.down_proj", + "ppl": 4.400164604187012 + }, + { + "layer": "layer.57.self_attn.q_proj", + "ppl": 4.400632381439209 + }, + { + "layer": "layer.57.self_attn.k_proj", + "ppl": 4.400166034698486 + }, + { + "layer": "layer.57.self_attn.v_proj", + "ppl": 4.400021076202393 + }, + { + "layer": "layer.57.self_attn.o_proj", + "ppl": 4.400156497955322 + }, + { + "layer": "layer.57.mlp.gate_proj", + "ppl": 4.399770259857178 + }, + { + "layer": "layer.57.mlp.up_proj", + "ppl": 4.40004825592041 + }, + { + "layer": "layer.57.mlp.down_proj", + "ppl": 4.400211811065674 + }, + { + "layer": "layer.58.self_attn.q_proj", + "ppl": 4.400482177734375 + }, + { + "layer": "layer.58.self_attn.k_proj", + "ppl": 4.400343418121338 + }, + { + "layer": "layer.58.self_attn.v_proj", + "ppl": 4.400331497192383 + }, + { + "layer": "layer.58.self_attn.o_proj", + "ppl": 4.399943828582764 + }, + { + "layer": "layer.58.mlp.gate_proj", + "ppl": 4.399641036987305 + }, + { + "layer": "layer.58.mlp.up_proj", + "ppl": 4.400486946105957 + }, + { + "layer": "layer.58.mlp.down_proj", + "ppl": 4.399755954742432 + }, + { + "layer": "layer.59.self_attn.q_proj", + "ppl": 4.400491237640381 + }, + { + "layer": "layer.59.self_attn.k_proj", + "ppl": 4.4001264572143555 + }, + { + "layer": "layer.59.self_attn.v_proj", + "ppl": 4.400088310241699 + }, + { + "layer": "layer.59.self_attn.o_proj", + "ppl": 4.4003987312316895 + }, + { + "layer": "layer.59.mlp.gate_proj", + "ppl": 4.399747848510742 + }, + { + "layer": "layer.59.mlp.up_proj", + "ppl": 4.399668216705322 + }, + { + "layer": "layer.59.mlp.down_proj", + "ppl": 4.399681568145752 + }, + { + "layer": "layer.60.self_attn.q_proj", + "ppl": 4.400518417358398 + }, + { + "layer": "layer.60.self_attn.k_proj", + "ppl": 4.400404930114746 + }, + { + "layer": "layer.60.self_attn.v_proj", + "ppl": 4.4003705978393555 + }, + { + "layer": "layer.60.self_attn.o_proj", + "ppl": 4.400097370147705 + }, + { + "layer": "layer.60.mlp.gate_proj", + "ppl": 4.399293899536133 + }, + { + "layer": "layer.60.mlp.up_proj", + "ppl": 4.399867057800293 + }, + { + "layer": "layer.60.mlp.down_proj", + "ppl": 4.399807929992676 + }, + { + "layer": "layer.61.self_attn.q_proj", + "ppl": 4.400530815124512 + }, + { + "layer": "layer.61.self_attn.k_proj", + "ppl": 4.4004597663879395 + }, + { + "layer": "layer.61.self_attn.v_proj", + "ppl": 4.400328159332275 + }, + { + "layer": "layer.61.self_attn.o_proj", + "ppl": 4.40022087097168 + }, + { + "layer": "layer.61.mlp.gate_proj", + "ppl": 4.3991217613220215 + }, + { + "layer": "layer.61.mlp.up_proj", + "ppl": 4.399914264678955 + }, + { + "layer": "layer.61.mlp.down_proj", + "ppl": 4.39975118637085 + }, + { + "layer": "layer.62.self_attn.q_proj", + "ppl": 4.399970054626465 + }, + { + "layer": "layer.62.self_attn.k_proj", + "ppl": 4.399919509887695 + }, + { + "layer": "layer.62.self_attn.v_proj", + "ppl": 4.399950981140137 + }, + { + "layer": "layer.62.self_attn.o_proj", + "ppl": 4.399641036987305 + }, + { + "layer": "layer.62.mlp.gate_proj", + "ppl": 4.398950099945068 + }, + { + "layer": "layer.62.mlp.up_proj", + "ppl": 4.399483680725098 + }, + { + "layer": "layer.62.mlp.down_proj", + "ppl": 4.399820804595947 + }, + { + "layer": "layer.63.self_attn.q_proj", + "ppl": 4.400251865386963 + }, + { + "layer": "layer.63.self_attn.k_proj", + "ppl": 4.4002580642700195 + }, + { + "layer": "layer.63.self_attn.v_proj", + "ppl": 4.399962425231934 + }, + { + "layer": "layer.63.self_attn.o_proj", + "ppl": 4.400351524353027 + }, + { + "layer": "layer.63.mlp.gate_proj", + "ppl": 4.398874282836914 + }, + { + "layer": "layer.63.mlp.up_proj", + "ppl": 4.400057315826416 + }, + { + "layer": "layer.63.mlp.down_proj", + "ppl": 4.399930953979492 + }, + { + "layer": "layer.64.self_attn.q_proj", + "ppl": 4.4003705978393555 + }, + { + "layer": "layer.64.self_attn.k_proj", + "ppl": 4.400292873382568 + }, + { + "layer": "layer.64.self_attn.v_proj", + "ppl": 4.399926662445068 + }, + { + "layer": "layer.64.self_attn.o_proj", + "ppl": 4.400074481964111 + }, + { + "layer": "layer.64.mlp.gate_proj", + "ppl": 4.398674488067627 + }, + { + "layer": "layer.64.mlp.up_proj", + "ppl": 4.3999762535095215 + }, + { + "layer": "layer.64.mlp.down_proj", + "ppl": 4.399697780609131 + }, + { + "layer": "layer.65.self_attn.q_proj", + "ppl": 4.399686336517334 + }, + { + "layer": "layer.65.self_attn.k_proj", + "ppl": 4.399963855743408 + }, + { + "layer": "layer.65.self_attn.v_proj", + "ppl": 4.400456428527832 + }, + { + "layer": "layer.65.self_attn.o_proj", + "ppl": 4.400234699249268 + }, + { + "layer": "layer.65.mlp.gate_proj", + "ppl": 4.398816108703613 + }, + { + "layer": "layer.65.mlp.up_proj", + "ppl": 4.399857044219971 + }, + { + "layer": "layer.65.mlp.down_proj", + "ppl": 4.39975118637085 + }, + { + "layer": "layer.66.self_attn.q_proj", + "ppl": 4.400450706481934 + }, + { + "layer": "layer.66.self_attn.k_proj", + "ppl": 4.399932384490967 + }, + { + "layer": "layer.66.self_attn.v_proj", + "ppl": 4.400202751159668 + }, + { + "layer": "layer.66.self_attn.o_proj", + "ppl": 4.4004902839660645 + }, + { + "layer": "layer.66.mlp.gate_proj", + "ppl": 4.3984270095825195 + }, + { + "layer": "layer.66.mlp.up_proj", + "ppl": 4.399379253387451 + }, + { + "layer": "layer.66.mlp.down_proj", + "ppl": 4.399839878082275 + }, + { + "layer": "layer.67.self_attn.q_proj", + "ppl": 4.400287628173828 + }, + { + "layer": "layer.67.self_attn.k_proj", + "ppl": 4.400371551513672 + }, + { + "layer": "layer.67.self_attn.v_proj", + "ppl": 4.400486469268799 + }, + { + "layer": "layer.67.self_attn.o_proj", + "ppl": 4.400363445281982 + }, + { + "layer": "layer.67.mlp.gate_proj", + "ppl": 4.398804187774658 + }, + { + "layer": "layer.67.mlp.up_proj", + "ppl": 4.399876594543457 + }, + { + "layer": "layer.67.mlp.down_proj", + "ppl": 4.400657653808594 + }, + { + "layer": "layer.68.self_attn.q_proj", + "ppl": 4.400091171264648 + }, + { + "layer": "layer.68.self_attn.k_proj", + "ppl": 4.4002580642700195 + }, + { + "layer": "layer.68.self_attn.v_proj", + "ppl": 4.400302410125732 + }, + { + "layer": "layer.68.self_attn.o_proj", + "ppl": 4.400269985198975 + }, + { + "layer": "layer.68.mlp.gate_proj", + "ppl": 4.398923873901367 + }, + { + "layer": "layer.68.mlp.up_proj", + "ppl": 4.399153709411621 + }, + { + "layer": "layer.68.mlp.down_proj", + "ppl": 4.4001617431640625 + }, + { + "layer": "layer.69.self_attn.q_proj", + "ppl": 4.400332927703857 + }, + { + "layer": "layer.69.self_attn.k_proj", + "ppl": 4.400181293487549 + }, + { + "layer": "layer.69.self_attn.v_proj", + "ppl": 4.400294780731201 + }, + { + "layer": "layer.69.self_attn.o_proj", + "ppl": 4.400482654571533 + }, + { + "layer": "layer.69.mlp.gate_proj", + "ppl": 4.399112701416016 + }, + { + "layer": "layer.69.mlp.up_proj", + "ppl": 4.399920463562012 + }, + { + "layer": "layer.69.mlp.down_proj", + "ppl": 4.399411678314209 + }, + { + "layer": "layer.70.self_attn.q_proj", + "ppl": 4.400401592254639 + }, + { + "layer": "layer.70.self_attn.k_proj", + "ppl": 4.4003400802612305 + }, + { + "layer": "layer.70.self_attn.v_proj", + "ppl": 4.40032434463501 + }, + { + "layer": "layer.70.self_attn.o_proj", + "ppl": 4.400277137756348 + }, + { + "layer": "layer.70.mlp.gate_proj", + "ppl": 4.398960590362549 + }, + { + "layer": "layer.70.mlp.up_proj", + "ppl": 4.400259494781494 + }, + { + "layer": "layer.70.mlp.down_proj", + "ppl": 4.399885654449463 + }, + { + "layer": "layer.71.self_attn.q_proj", + "ppl": 4.400590896606445 + }, + { + "layer": "layer.71.self_attn.k_proj", + "ppl": 4.400413513183594 + }, + { + "layer": "layer.71.self_attn.v_proj", + "ppl": 4.400137424468994 + }, + { + "layer": "layer.71.self_attn.o_proj", + "ppl": 4.400343418121338 + }, + { + "layer": "layer.71.mlp.gate_proj", + "ppl": 4.3995184898376465 + }, + { + "layer": "layer.71.mlp.up_proj", + "ppl": 4.4001264572143555 + }, + { + "layer": "layer.71.mlp.down_proj", + "ppl": 4.399285793304443 + }, + { + "layer": "layer.72.self_attn.q_proj", + "ppl": 4.400227069854736 + }, + { + "layer": "layer.72.self_attn.k_proj", + "ppl": 4.400356292724609 + }, + { + "layer": "layer.72.self_attn.v_proj", + "ppl": 4.400152206420898 + }, + { + "layer": "layer.72.self_attn.o_proj", + "ppl": 4.400258541107178 + }, + { + "layer": "layer.72.mlp.gate_proj", + "ppl": 4.398385524749756 + }, + { + "layer": "layer.72.mlp.up_proj", + "ppl": 4.398835182189941 + }, + { + "layer": "layer.72.mlp.down_proj", + "ppl": 4.399263858795166 + }, + { + "layer": "layer.73.self_attn.q_proj", + "ppl": 4.3999762535095215 + }, + { + "layer": "layer.73.self_attn.k_proj", + "ppl": 4.400176525115967 + }, + { + "layer": "layer.73.self_attn.v_proj", + "ppl": 4.4005126953125 + }, + { + "layer": "layer.73.self_attn.o_proj", + "ppl": 4.400450229644775 + }, + { + "layer": "layer.73.mlp.gate_proj", + "ppl": 4.398819923400879 + }, + { + "layer": "layer.73.mlp.up_proj", + "ppl": 4.399275302886963 + }, + { + "layer": "layer.73.mlp.down_proj", + "ppl": 4.399411201477051 + }, + { + "layer": "layer.74.self_attn.q_proj", + "ppl": 4.400368690490723 + }, + { + "layer": "layer.74.self_attn.k_proj", + "ppl": 4.400517463684082 + }, + { + "layer": "layer.74.self_attn.v_proj", + "ppl": 4.400325775146484 + }, + { + "layer": "layer.74.self_attn.o_proj", + "ppl": 4.40026330947876 + }, + { + "layer": "layer.74.mlp.gate_proj", + "ppl": 4.399063587188721 + }, + { + "layer": "layer.74.mlp.up_proj", + "ppl": 4.399440765380859 + }, + { + "layer": "layer.74.mlp.down_proj", + "ppl": 4.398950576782227 + }, + { + "layer": "layer.75.self_attn.q_proj", + "ppl": 4.400228500366211 + }, + { + "layer": "layer.75.self_attn.k_proj", + "ppl": 4.400293350219727 + }, + { + "layer": "layer.75.self_attn.v_proj", + "ppl": 4.399792671203613 + }, + { + "layer": "layer.75.self_attn.o_proj", + "ppl": 4.400076866149902 + }, + { + "layer": "layer.75.mlp.gate_proj", + "ppl": 4.398985862731934 + }, + { + "layer": "layer.75.mlp.up_proj", + "ppl": 4.399979114532471 + }, + { + "layer": "layer.75.mlp.down_proj", + "ppl": 4.399557590484619 + }, + { + "layer": "layer.76.self_attn.q_proj", + "ppl": 4.400311470031738 + }, + { + "layer": "layer.76.self_attn.k_proj", + "ppl": 4.400600910186768 + }, + { + "layer": "layer.76.self_attn.v_proj", + "ppl": 4.401032447814941 + }, + { + "layer": "layer.76.self_attn.o_proj", + "ppl": 4.4005913734436035 + }, + { + "layer": "layer.76.mlp.gate_proj", + "ppl": 4.398752212524414 + }, + { + "layer": "layer.76.mlp.up_proj", + "ppl": 4.3985819816589355 + }, + { + "layer": "layer.76.mlp.down_proj", + "ppl": 4.399528503417969 + }, + { + "layer": "layer.77.self_attn.q_proj", + "ppl": 4.4002580642700195 + }, + { + "layer": "layer.77.self_attn.k_proj", + "ppl": 4.400800704956055 + }, + { + "layer": "layer.77.self_attn.v_proj", + "ppl": 4.400167942047119 + }, + { + "layer": "layer.77.self_attn.o_proj", + "ppl": 4.400726318359375 + }, + { + "layer": "layer.77.mlp.gate_proj", + "ppl": 4.398130416870117 + }, + { + "layer": "layer.77.mlp.up_proj", + "ppl": 4.399073600769043 + }, + { + "layer": "layer.77.mlp.down_proj", + "ppl": 4.398903846740723 + }, + { + "layer": "layer.78.self_attn.q_proj", + "ppl": 4.400352954864502 + }, + { + "layer": "layer.78.self_attn.k_proj", + "ppl": 4.400294780731201 + }, + { + "layer": "layer.78.self_attn.v_proj", + "ppl": 4.400938034057617 + }, + { + "layer": "layer.78.self_attn.o_proj", + "ppl": 4.401062488555908 + }, + { + "layer": "layer.78.mlp.gate_proj", + "ppl": 4.397465705871582 + }, + { + "layer": "layer.78.mlp.up_proj", + "ppl": 4.398340702056885 + }, + { + "layer": "layer.78.mlp.down_proj", + "ppl": 4.398187160491943 + }, + { + "layer": "layer.79.self_attn.q_proj", + "ppl": 4.399982452392578 + }, + { + "layer": "layer.79.self_attn.k_proj", + "ppl": 4.400362014770508 + }, + { + "layer": "layer.79.self_attn.v_proj", + "ppl": 4.400581359863281 + }, + { + "layer": "layer.79.self_attn.o_proj", + "ppl": 4.400692939758301 + }, + { + "layer": "layer.79.mlp.gate_proj", + "ppl": 4.395840644836426 + }, + { + "layer": "layer.79.mlp.up_proj", + "ppl": 4.39369535446167 + }, + { + "layer": "layer.79.mlp.down_proj", + "ppl": 4.391618251800537 + } + ] + }, + { + "bit": 6, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 4.261491298675537 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 4.261433124542236 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 4.261356830596924 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 4.26146936416626 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 4.261435031890869 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 4.261556148529053 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 4.261502265930176 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 4.261560916900635 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 4.2613325119018555 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 4.261439323425293 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 4.261478900909424 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 4.261399269104004 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 4.261441707611084 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 4.261491298675537 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 4.261389255523682 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 4.26130485534668 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 4.261427879333496 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 4.2612481117248535 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 4.261348724365234 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 4.261478900909424 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 4.261381149291992 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 4.26134729385376 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 4.2614874839782715 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 4.261519432067871 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 4.261498928070068 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 4.261380195617676 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 4.2614593505859375 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 4.261515140533447 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 4.261406421661377 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 4.261326313018799 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 4.261326789855957 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 4.261440277099609 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 4.261341571807861 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 4.2614922523498535 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 4.261648178100586 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 4.261411666870117 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 4.261538505554199 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 4.261328220367432 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 4.2613019943237305 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 4.261303901672363 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 4.261392593383789 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 4.261444091796875 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 4.2614312171936035 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 4.261373043060303 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 4.261468410491943 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 4.261441230773926 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 4.261430263519287 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 4.261637210845947 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 4.261456489562988 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 4.261476516723633 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 4.261490821838379 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 4.261476039886475 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 4.261396884918213 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 4.2613911628723145 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 4.261564254760742 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 4.261252403259277 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 4.261577129364014 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 4.261361122131348 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 4.261573314666748 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 4.261364459991455 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 4.261311054229736 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 4.261512756347656 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 4.261275291442871 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 4.261531352996826 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 4.261383533477783 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 4.261334419250488 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 4.26142692565918 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 4.2613935470581055 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 4.261378765106201 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 4.26129150390625 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 4.26148796081543 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 4.261555194854736 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 4.26147985458374 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 4.26146125793457 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 4.261162757873535 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 4.261197090148926 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 4.261533737182617 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 4.261392116546631 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 4.261556625366211 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 4.2614054679870605 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 4.2614665031433105 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 4.261358261108398 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 4.26143217086792 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 4.261538505554199 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 4.261439323425293 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 4.2615485191345215 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 4.261477470397949 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 4.261388778686523 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 4.261287689208984 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 4.261427879333496 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 4.261333465576172 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 4.26145601272583 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 4.261395454406738 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 4.2614359855651855 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 4.2615509033203125 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 4.261260509490967 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 4.261369228363037 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 4.261383056640625 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 4.2614006996154785 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 4.261293888092041 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 4.261329174041748 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 4.261442184448242 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 4.261415481567383 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 4.261436939239502 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 4.261321067810059 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 4.261355876922607 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 4.261490821838379 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 4.261115074157715 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 4.261276721954346 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 4.261287212371826 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 4.261334419250488 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 4.261390209197998 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 4.261524200439453 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 4.261387348175049 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 4.2614288330078125 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 4.261319160461426 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 4.261379241943359 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 4.261500358581543 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 4.261374473571777 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 4.2613935470581055 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 4.261338710784912 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 4.261358261108398 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 4.261178970336914 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 4.2614312171936035 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 4.261317253112793 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 4.261529922485352 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 4.261460304260254 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 4.261517524719238 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 4.261255741119385 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 4.261575222015381 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 4.2617011070251465 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 4.2613725662231445 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 4.261512279510498 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 4.261590003967285 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 4.261476039886475 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 4.261422157287598 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 4.261556625366211 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 4.261511325836182 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 4.261435031890869 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 4.261658191680908 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 4.261471271514893 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 4.2614359855651855 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 4.261578559875488 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 4.26120138168335 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 4.261427879333496 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 4.261383533477783 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 4.261261463165283 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 4.261392116546631 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 4.2614593505859375 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 4.261377334594727 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 4.261423110961914 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 4.2612833976745605 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 4.261459827423096 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 4.261349201202393 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 4.261361598968506 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 4.26137113571167 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 4.261314392089844 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 4.261192798614502 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 4.261334419250488 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 4.261268615722656 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 4.261614799499512 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 4.261377334594727 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 4.261324405670166 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 4.261414051055908 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 4.261383056640625 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 4.2612409591674805 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 4.261348247528076 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 4.261610507965088 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 4.26146125793457 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 4.261518955230713 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 4.261394023895264 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 4.261237144470215 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 4.2612457275390625 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 4.261473178863525 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 4.261347770690918 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 4.261389255523682 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 4.2613525390625 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 4.2613983154296875 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 4.261441230773926 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 4.260903358459473 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 4.261325836181641 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 4.260951519012451 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 4.261617183685303 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 4.2613844871521 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 4.26155424118042 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 4.261460304260254 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 4.261422157287598 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 4.261279106140137 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 4.260895729064941 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 4.261390209197998 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 4.261442184448242 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 4.261302947998047 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 4.261254787445068 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 4.261525630950928 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 4.261386394500732 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 4.261009216308594 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 4.261507034301758 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 4.261470794677734 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 4.261526584625244 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 4.261341571807861 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 4.261269569396973 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 4.261197090148926 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 4.261068344116211 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 4.261368274688721 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 4.261372089385986 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 4.261276721954346 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 4.261050701141357 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 4.2613983154296875 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 4.261468410491943 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 4.261340618133545 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 4.2614569664001465 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 4.261569023132324 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 4.261364936828613 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 4.261306285858154 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 4.261214733123779 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 4.2613630294799805 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 4.260740280151367 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 4.261359214782715 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 4.261363506317139 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 4.261462688446045 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 4.261194229125977 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 4.261826038360596 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 4.261404991149902 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 4.260871887207031 + }, + { + "layer": "layer.32.self_attn.q_proj", + "ppl": 4.261457443237305 + }, + { + "layer": "layer.32.self_attn.k_proj", + "ppl": 4.2615203857421875 + }, + { + "layer": "layer.32.self_attn.v_proj", + "ppl": 4.2613844871521 + }, + { + "layer": "layer.32.self_attn.o_proj", + "ppl": 4.261441230773926 + }, + { + "layer": "layer.32.mlp.gate_proj", + "ppl": 4.261440277099609 + }, + { + "layer": "layer.32.mlp.up_proj", + "ppl": 4.261013031005859 + }, + { + "layer": "layer.32.mlp.down_proj", + "ppl": 4.261062145233154 + }, + { + "layer": "layer.33.self_attn.q_proj", + "ppl": 4.261378288269043 + }, + { + "layer": "layer.33.self_attn.k_proj", + "ppl": 4.2614641189575195 + }, + { + "layer": "layer.33.self_attn.v_proj", + "ppl": 4.261360168457031 + }, + { + "layer": "layer.33.self_attn.o_proj", + "ppl": 4.261133193969727 + }, + { + "layer": "layer.33.mlp.gate_proj", + "ppl": 4.26125955581665 + }, + { + "layer": "layer.33.mlp.up_proj", + "ppl": 4.261049270629883 + }, + { + "layer": "layer.33.mlp.down_proj", + "ppl": 4.260688304901123 + }, + { + "layer": "layer.34.self_attn.q_proj", + "ppl": 4.261348247528076 + }, + { + "layer": "layer.34.self_attn.k_proj", + "ppl": 4.261341571807861 + }, + { + "layer": "layer.34.self_attn.v_proj", + "ppl": 4.261324882507324 + }, + { + "layer": "layer.34.self_attn.o_proj", + "ppl": 4.261171817779541 + }, + { + "layer": "layer.34.mlp.gate_proj", + "ppl": 4.261027812957764 + }, + { + "layer": "layer.34.mlp.up_proj", + "ppl": 4.261177062988281 + }, + { + "layer": "layer.34.mlp.down_proj", + "ppl": 4.261342525482178 + }, + { + "layer": "layer.35.self_attn.q_proj", + "ppl": 4.2614593505859375 + }, + { + "layer": "layer.35.self_attn.k_proj", + "ppl": 4.2613935470581055 + }, + { + "layer": "layer.35.self_attn.v_proj", + "ppl": 4.261310577392578 + }, + { + "layer": "layer.35.self_attn.o_proj", + "ppl": 4.261436939239502 + }, + { + "layer": "layer.35.mlp.gate_proj", + "ppl": 4.2613325119018555 + }, + { + "layer": "layer.35.mlp.up_proj", + "ppl": 4.261221408843994 + }, + { + "layer": "layer.35.mlp.down_proj", + "ppl": 4.261804580688477 + }, + { + "layer": "layer.36.self_attn.q_proj", + "ppl": 4.261366367340088 + }, + { + "layer": "layer.36.self_attn.k_proj", + "ppl": 4.261199474334717 + }, + { + "layer": "layer.36.self_attn.v_proj", + "ppl": 4.2612481117248535 + }, + { + "layer": "layer.36.self_attn.o_proj", + "ppl": 4.2610273361206055 + }, + { + "layer": "layer.36.mlp.gate_proj", + "ppl": 4.2612080574035645 + }, + { + "layer": "layer.36.mlp.up_proj", + "ppl": 4.26186990737915 + }, + { + "layer": "layer.36.mlp.down_proj", + "ppl": 4.2606201171875 + }, + { + "layer": "layer.37.self_attn.q_proj", + "ppl": 4.2614312171936035 + }, + { + "layer": "layer.37.self_attn.k_proj", + "ppl": 4.261361598968506 + }, + { + "layer": "layer.37.self_attn.v_proj", + "ppl": 4.261363983154297 + }, + { + "layer": "layer.37.self_attn.o_proj", + "ppl": 4.261274337768555 + }, + { + "layer": "layer.37.mlp.gate_proj", + "ppl": 4.261212348937988 + }, + { + "layer": "layer.37.mlp.up_proj", + "ppl": 4.261478424072266 + }, + { + "layer": "layer.37.mlp.down_proj", + "ppl": 4.261256217956543 + }, + { + "layer": "layer.38.self_attn.q_proj", + "ppl": 4.2614545822143555 + }, + { + "layer": "layer.38.self_attn.k_proj", + "ppl": 4.261317253112793 + }, + { + "layer": "layer.38.self_attn.v_proj", + "ppl": 4.261434078216553 + }, + { + "layer": "layer.38.self_attn.o_proj", + "ppl": 4.261425018310547 + }, + { + "layer": "layer.38.mlp.gate_proj", + "ppl": 4.260735034942627 + }, + { + "layer": "layer.38.mlp.up_proj", + "ppl": 4.261392593383789 + }, + { + "layer": "layer.38.mlp.down_proj", + "ppl": 4.261174201965332 + }, + { + "layer": "layer.39.self_attn.q_proj", + "ppl": 4.261210918426514 + }, + { + "layer": "layer.39.self_attn.k_proj", + "ppl": 4.2613444328308105 + }, + { + "layer": "layer.39.self_attn.v_proj", + "ppl": 4.261595726013184 + }, + { + "layer": "layer.39.self_attn.o_proj", + "ppl": 4.261213302612305 + }, + { + "layer": "layer.39.mlp.gate_proj", + "ppl": 4.261372089385986 + }, + { + "layer": "layer.39.mlp.up_proj", + "ppl": 4.261353492736816 + }, + { + "layer": "layer.39.mlp.down_proj", + "ppl": 4.260984897613525 + }, + { + "layer": "layer.40.self_attn.q_proj", + "ppl": 4.261108875274658 + }, + { + "layer": "layer.40.self_attn.k_proj", + "ppl": 4.261340618133545 + }, + { + "layer": "layer.40.self_attn.v_proj", + "ppl": 4.261330604553223 + }, + { + "layer": "layer.40.self_attn.o_proj", + "ppl": 4.261334419250488 + }, + { + "layer": "layer.40.mlp.gate_proj", + "ppl": 4.261143684387207 + }, + { + "layer": "layer.40.mlp.up_proj", + "ppl": 4.260762691497803 + }, + { + "layer": "layer.40.mlp.down_proj", + "ppl": 4.260998249053955 + }, + { + "layer": "layer.41.self_attn.q_proj", + "ppl": 4.261464595794678 + }, + { + "layer": "layer.41.self_attn.k_proj", + "ppl": 4.261448383331299 + }, + { + "layer": "layer.41.self_attn.v_proj", + "ppl": 4.2612786293029785 + }, + { + "layer": "layer.41.self_attn.o_proj", + "ppl": 4.261342525482178 + }, + { + "layer": "layer.41.mlp.gate_proj", + "ppl": 4.261401653289795 + }, + { + "layer": "layer.41.mlp.up_proj", + "ppl": 4.261349678039551 + }, + { + "layer": "layer.41.mlp.down_proj", + "ppl": 4.260735988616943 + }, + { + "layer": "layer.42.self_attn.q_proj", + "ppl": 4.261363983154297 + }, + { + "layer": "layer.42.self_attn.k_proj", + "ppl": 4.2614312171936035 + }, + { + "layer": "layer.42.self_attn.v_proj", + "ppl": 4.261093616485596 + }, + { + "layer": "layer.42.self_attn.o_proj", + "ppl": 4.261445045471191 + }, + { + "layer": "layer.42.mlp.gate_proj", + "ppl": 4.261415481567383 + }, + { + "layer": "layer.42.mlp.up_proj", + "ppl": 4.261874198913574 + }, + { + "layer": "layer.42.mlp.down_proj", + "ppl": 4.2610979080200195 + }, + { + "layer": "layer.43.self_attn.q_proj", + "ppl": 4.261485576629639 + }, + { + "layer": "layer.43.self_attn.k_proj", + "ppl": 4.261411666870117 + }, + { + "layer": "layer.43.self_attn.v_proj", + "ppl": 4.261502742767334 + }, + { + "layer": "layer.43.self_attn.o_proj", + "ppl": 4.261262893676758 + }, + { + "layer": "layer.43.mlp.gate_proj", + "ppl": 4.261104583740234 + }, + { + "layer": "layer.43.mlp.up_proj", + "ppl": 4.260831356048584 + }, + { + "layer": "layer.43.mlp.down_proj", + "ppl": 4.261900901794434 + }, + { + "layer": "layer.44.self_attn.q_proj", + "ppl": 4.261427879333496 + }, + { + "layer": "layer.44.self_attn.k_proj", + "ppl": 4.261219501495361 + }, + { + "layer": "layer.44.self_attn.v_proj", + "ppl": 4.261264801025391 + }, + { + "layer": "layer.44.self_attn.o_proj", + "ppl": 4.261312961578369 + }, + { + "layer": "layer.44.mlp.gate_proj", + "ppl": 4.26125431060791 + }, + { + "layer": "layer.44.mlp.up_proj", + "ppl": 4.261342525482178 + }, + { + "layer": "layer.44.mlp.down_proj", + "ppl": 4.261070728302002 + }, + { + "layer": "layer.45.self_attn.q_proj", + "ppl": 4.261441230773926 + }, + { + "layer": "layer.45.self_attn.k_proj", + "ppl": 4.261264324188232 + }, + { + "layer": "layer.45.self_attn.v_proj", + "ppl": 4.261598587036133 + }, + { + "layer": "layer.45.self_attn.o_proj", + "ppl": 4.261240005493164 + }, + { + "layer": "layer.45.mlp.gate_proj", + "ppl": 4.260793209075928 + }, + { + "layer": "layer.45.mlp.up_proj", + "ppl": 4.261603832244873 + }, + { + "layer": "layer.45.mlp.down_proj", + "ppl": 4.261293411254883 + }, + { + "layer": "layer.46.self_attn.q_proj", + "ppl": 4.261423110961914 + }, + { + "layer": "layer.46.self_attn.k_proj", + "ppl": 4.261338710784912 + }, + { + "layer": "layer.46.self_attn.v_proj", + "ppl": 4.261540412902832 + }, + { + "layer": "layer.46.self_attn.o_proj", + "ppl": 4.2610297203063965 + }, + { + "layer": "layer.46.mlp.gate_proj", + "ppl": 4.260982036590576 + }, + { + "layer": "layer.46.mlp.up_proj", + "ppl": 4.261136531829834 + }, + { + "layer": "layer.46.mlp.down_proj", + "ppl": 4.26174783706665 + }, + { + "layer": "layer.47.self_attn.q_proj", + "ppl": 4.261592388153076 + }, + { + "layer": "layer.47.self_attn.k_proj", + "ppl": 4.261486530303955 + }, + { + "layer": "layer.47.self_attn.v_proj", + "ppl": 4.2615580558776855 + }, + { + "layer": "layer.47.self_attn.o_proj", + "ppl": 4.261170864105225 + }, + { + "layer": "layer.47.mlp.gate_proj", + "ppl": 4.261251449584961 + }, + { + "layer": "layer.47.mlp.up_proj", + "ppl": 4.261381149291992 + }, + { + "layer": "layer.47.mlp.down_proj", + "ppl": 4.2610673904418945 + }, + { + "layer": "layer.48.self_attn.q_proj", + "ppl": 4.261324882507324 + }, + { + "layer": "layer.48.self_attn.k_proj", + "ppl": 4.261222839355469 + }, + { + "layer": "layer.48.self_attn.v_proj", + "ppl": 4.26146936416626 + }, + { + "layer": "layer.48.self_attn.o_proj", + "ppl": 4.261441707611084 + }, + { + "layer": "layer.48.mlp.gate_proj", + "ppl": 4.260858535766602 + }, + { + "layer": "layer.48.mlp.up_proj", + "ppl": 4.2612810134887695 + }, + { + "layer": "layer.48.mlp.down_proj", + "ppl": 4.260841369628906 + }, + { + "layer": "layer.49.self_attn.q_proj", + "ppl": 4.261162757873535 + }, + { + "layer": "layer.49.self_attn.k_proj", + "ppl": 4.261320114135742 + }, + { + "layer": "layer.49.self_attn.v_proj", + "ppl": 4.261353492736816 + }, + { + "layer": "layer.49.self_attn.o_proj", + "ppl": 4.2615580558776855 + }, + { + "layer": "layer.49.mlp.gate_proj", + "ppl": 4.261143684387207 + }, + { + "layer": "layer.49.mlp.up_proj", + "ppl": 4.261531352996826 + }, + { + "layer": "layer.49.mlp.down_proj", + "ppl": 4.261404514312744 + }, + { + "layer": "layer.50.self_attn.q_proj", + "ppl": 4.261443614959717 + }, + { + "layer": "layer.50.self_attn.k_proj", + "ppl": 4.261373043060303 + }, + { + "layer": "layer.50.self_attn.v_proj", + "ppl": 4.261250972747803 + }, + { + "layer": "layer.50.self_attn.o_proj", + "ppl": 4.261213302612305 + }, + { + "layer": "layer.50.mlp.gate_proj", + "ppl": 4.2609543800354 + }, + { + "layer": "layer.50.mlp.up_proj", + "ppl": 4.26107931137085 + }, + { + "layer": "layer.50.mlp.down_proj", + "ppl": 4.260898590087891 + }, + { + "layer": "layer.51.self_attn.q_proj", + "ppl": 4.261456489562988 + }, + { + "layer": "layer.51.self_attn.k_proj", + "ppl": 4.261133193969727 + }, + { + "layer": "layer.51.self_attn.v_proj", + "ppl": 4.2612433433532715 + }, + { + "layer": "layer.51.self_attn.o_proj", + "ppl": 4.261326313018799 + }, + { + "layer": "layer.51.mlp.gate_proj", + "ppl": 4.261225700378418 + }, + { + "layer": "layer.51.mlp.up_proj", + "ppl": 4.261485576629639 + }, + { + "layer": "layer.51.mlp.down_proj", + "ppl": 4.261440277099609 + }, + { + "layer": "layer.52.self_attn.q_proj", + "ppl": 4.261498928070068 + }, + { + "layer": "layer.52.self_attn.k_proj", + "ppl": 4.261123180389404 + }, + { + "layer": "layer.52.self_attn.v_proj", + "ppl": 4.261325359344482 + }, + { + "layer": "layer.52.self_attn.o_proj", + "ppl": 4.2612738609313965 + }, + { + "layer": "layer.52.mlp.gate_proj", + "ppl": 4.261361122131348 + }, + { + "layer": "layer.52.mlp.up_proj", + "ppl": 4.261629581451416 + }, + { + "layer": "layer.52.mlp.down_proj", + "ppl": 4.260699272155762 + }, + { + "layer": "layer.53.self_attn.q_proj", + "ppl": 4.261587619781494 + }, + { + "layer": "layer.53.self_attn.k_proj", + "ppl": 4.261168956756592 + }, + { + "layer": "layer.53.self_attn.v_proj", + "ppl": 4.261116981506348 + }, + { + "layer": "layer.53.self_attn.o_proj", + "ppl": 4.261306285858154 + }, + { + "layer": "layer.53.mlp.gate_proj", + "ppl": 4.2611165046691895 + }, + { + "layer": "layer.53.mlp.up_proj", + "ppl": 4.260991096496582 + }, + { + "layer": "layer.53.mlp.down_proj", + "ppl": 4.261147975921631 + }, + { + "layer": "layer.54.self_attn.q_proj", + "ppl": 4.261417388916016 + }, + { + "layer": "layer.54.self_attn.k_proj", + "ppl": 4.2612199783325195 + }, + { + "layer": "layer.54.self_attn.v_proj", + "ppl": 4.2614359855651855 + }, + { + "layer": "layer.54.self_attn.o_proj", + "ppl": 4.261235237121582 + }, + { + "layer": "layer.54.mlp.gate_proj", + "ppl": 4.2613115310668945 + }, + { + "layer": "layer.54.mlp.up_proj", + "ppl": 4.261419773101807 + }, + { + "layer": "layer.54.mlp.down_proj", + "ppl": 4.260567665100098 + }, + { + "layer": "layer.55.self_attn.q_proj", + "ppl": 4.261387348175049 + }, + { + "layer": "layer.55.self_attn.k_proj", + "ppl": 4.261423587799072 + }, + { + "layer": "layer.55.self_attn.v_proj", + "ppl": 4.261693000793457 + }, + { + "layer": "layer.55.self_attn.o_proj", + "ppl": 4.261345386505127 + }, + { + "layer": "layer.55.mlp.gate_proj", + "ppl": 4.261111259460449 + }, + { + "layer": "layer.55.mlp.up_proj", + "ppl": 4.26132345199585 + }, + { + "layer": "layer.55.mlp.down_proj", + "ppl": 4.260955333709717 + }, + { + "layer": "layer.56.self_attn.q_proj", + "ppl": 4.261237621307373 + }, + { + "layer": "layer.56.self_attn.k_proj", + "ppl": 4.2614970207214355 + }, + { + "layer": "layer.56.self_attn.v_proj", + "ppl": 4.261307716369629 + }, + { + "layer": "layer.56.self_attn.o_proj", + "ppl": 4.26136589050293 + }, + { + "layer": "layer.56.mlp.gate_proj", + "ppl": 4.260746479034424 + }, + { + "layer": "layer.56.mlp.up_proj", + "ppl": 4.261294841766357 + }, + { + "layer": "layer.56.mlp.down_proj", + "ppl": 4.261047840118408 + }, + { + "layer": "layer.57.self_attn.q_proj", + "ppl": 4.261258125305176 + }, + { + "layer": "layer.57.self_attn.k_proj", + "ppl": 4.261516094207764 + }, + { + "layer": "layer.57.self_attn.v_proj", + "ppl": 4.261605739593506 + }, + { + "layer": "layer.57.self_attn.o_proj", + "ppl": 4.261536598205566 + }, + { + "layer": "layer.57.mlp.gate_proj", + "ppl": 4.261176586151123 + }, + { + "layer": "layer.57.mlp.up_proj", + "ppl": 4.261300563812256 + }, + { + "layer": "layer.57.mlp.down_proj", + "ppl": 4.260849952697754 + }, + { + "layer": "layer.58.self_attn.q_proj", + "ppl": 4.261424541473389 + }, + { + "layer": "layer.58.self_attn.k_proj", + "ppl": 4.261312007904053 + }, + { + "layer": "layer.58.self_attn.v_proj", + "ppl": 4.261425018310547 + }, + { + "layer": "layer.58.self_attn.o_proj", + "ppl": 4.26137638092041 + }, + { + "layer": "layer.58.mlp.gate_proj", + "ppl": 4.261129379272461 + }, + { + "layer": "layer.58.mlp.up_proj", + "ppl": 4.2615203857421875 + }, + { + "layer": "layer.58.mlp.down_proj", + "ppl": 4.261245250701904 + }, + { + "layer": "layer.59.self_attn.q_proj", + "ppl": 4.261199474334717 + }, + { + "layer": "layer.59.self_attn.k_proj", + "ppl": 4.261142253875732 + }, + { + "layer": "layer.59.self_attn.v_proj", + "ppl": 4.261059284210205 + }, + { + "layer": "layer.59.self_attn.o_proj", + "ppl": 4.261230945587158 + }, + { + "layer": "layer.59.mlp.gate_proj", + "ppl": 4.260870456695557 + }, + { + "layer": "layer.59.mlp.up_proj", + "ppl": 4.261746883392334 + }, + { + "layer": "layer.59.mlp.down_proj", + "ppl": 4.260766983032227 + }, + { + "layer": "layer.60.self_attn.q_proj", + "ppl": 4.261340618133545 + }, + { + "layer": "layer.60.self_attn.k_proj", + "ppl": 4.261388301849365 + }, + { + "layer": "layer.60.self_attn.v_proj", + "ppl": 4.261139392852783 + }, + { + "layer": "layer.60.self_attn.o_proj", + "ppl": 4.261450290679932 + }, + { + "layer": "layer.60.mlp.gate_proj", + "ppl": 4.261023044586182 + }, + { + "layer": "layer.60.mlp.up_proj", + "ppl": 4.261114120483398 + }, + { + "layer": "layer.60.mlp.down_proj", + "ppl": 4.260880947113037 + }, + { + "layer": "layer.61.self_attn.q_proj", + "ppl": 4.261227130889893 + }, + { + "layer": "layer.61.self_attn.k_proj", + "ppl": 4.261353492736816 + }, + { + "layer": "layer.61.self_attn.v_proj", + "ppl": 4.261290550231934 + }, + { + "layer": "layer.61.self_attn.o_proj", + "ppl": 4.261127471923828 + }, + { + "layer": "layer.61.mlp.gate_proj", + "ppl": 4.260707378387451 + }, + { + "layer": "layer.61.mlp.up_proj", + "ppl": 4.261281490325928 + }, + { + "layer": "layer.61.mlp.down_proj", + "ppl": 4.261073589324951 + }, + { + "layer": "layer.62.self_attn.q_proj", + "ppl": 4.261146545410156 + }, + { + "layer": "layer.62.self_attn.k_proj", + "ppl": 4.26129674911499 + }, + { + "layer": "layer.62.self_attn.v_proj", + "ppl": 4.261267185211182 + }, + { + "layer": "layer.62.self_attn.o_proj", + "ppl": 4.2611985206604 + }, + { + "layer": "layer.62.mlp.gate_proj", + "ppl": 4.2608418464660645 + }, + { + "layer": "layer.62.mlp.up_proj", + "ppl": 4.261137962341309 + }, + { + "layer": "layer.62.mlp.down_proj", + "ppl": 4.260821342468262 + }, + { + "layer": "layer.63.self_attn.q_proj", + "ppl": 4.261254787445068 + }, + { + "layer": "layer.63.self_attn.k_proj", + "ppl": 4.2613325119018555 + }, + { + "layer": "layer.63.self_attn.v_proj", + "ppl": 4.261369705200195 + }, + { + "layer": "layer.63.self_attn.o_proj", + "ppl": 4.261196136474609 + }, + { + "layer": "layer.63.mlp.gate_proj", + "ppl": 4.26039981842041 + }, + { + "layer": "layer.63.mlp.up_proj", + "ppl": 4.261033058166504 + }, + { + "layer": "layer.63.mlp.down_proj", + "ppl": 4.260614395141602 + }, + { + "layer": "layer.64.self_attn.q_proj", + "ppl": 4.261280059814453 + }, + { + "layer": "layer.64.self_attn.k_proj", + "ppl": 4.2612714767456055 + }, + { + "layer": "layer.64.self_attn.v_proj", + "ppl": 4.261096477508545 + }, + { + "layer": "layer.64.self_attn.o_proj", + "ppl": 4.2609782218933105 + }, + { + "layer": "layer.64.mlp.gate_proj", + "ppl": 4.2604169845581055 + }, + { + "layer": "layer.64.mlp.up_proj", + "ppl": 4.2609148025512695 + }, + { + "layer": "layer.64.mlp.down_proj", + "ppl": 4.261212348937988 + }, + { + "layer": "layer.65.self_attn.q_proj", + "ppl": 4.261209964752197 + }, + { + "layer": "layer.65.self_attn.k_proj", + "ppl": 4.261105537414551 + }, + { + "layer": "layer.65.self_attn.v_proj", + "ppl": 4.261157035827637 + }, + { + "layer": "layer.65.self_attn.o_proj", + "ppl": 4.261292457580566 + }, + { + "layer": "layer.65.mlp.gate_proj", + "ppl": 4.260663032531738 + }, + { + "layer": "layer.65.mlp.up_proj", + "ppl": 4.26128625869751 + }, + { + "layer": "layer.65.mlp.down_proj", + "ppl": 4.260838031768799 + }, + { + "layer": "layer.66.self_attn.q_proj", + "ppl": 4.261345386505127 + }, + { + "layer": "layer.66.self_attn.k_proj", + "ppl": 4.2615437507629395 + }, + { + "layer": "layer.66.self_attn.v_proj", + "ppl": 4.261446475982666 + }, + { + "layer": "layer.66.self_attn.o_proj", + "ppl": 4.261228084564209 + }, + { + "layer": "layer.66.mlp.gate_proj", + "ppl": 4.261127471923828 + }, + { + "layer": "layer.66.mlp.up_proj", + "ppl": 4.260624408721924 + }, + { + "layer": "layer.66.mlp.down_proj", + "ppl": 4.261096477508545 + }, + { + "layer": "layer.67.self_attn.q_proj", + "ppl": 4.261358737945557 + }, + { + "layer": "layer.67.self_attn.k_proj", + "ppl": 4.261395454406738 + }, + { + "layer": "layer.67.self_attn.v_proj", + "ppl": 4.261358737945557 + }, + { + "layer": "layer.67.self_attn.o_proj", + "ppl": 4.261532306671143 + }, + { + "layer": "layer.67.mlp.gate_proj", + "ppl": 4.260128021240234 + }, + { + "layer": "layer.67.mlp.up_proj", + "ppl": 4.261195182800293 + }, + { + "layer": "layer.67.mlp.down_proj", + "ppl": 4.26057243347168 + }, + { + "layer": "layer.68.self_attn.q_proj", + "ppl": 4.261139869689941 + }, + { + "layer": "layer.68.self_attn.k_proj", + "ppl": 4.261247158050537 + }, + { + "layer": "layer.68.self_attn.v_proj", + "ppl": 4.261360168457031 + }, + { + "layer": "layer.68.self_attn.o_proj", + "ppl": 4.261212348937988 + }, + { + "layer": "layer.68.mlp.gate_proj", + "ppl": 4.260603427886963 + }, + { + "layer": "layer.68.mlp.up_proj", + "ppl": 4.261584281921387 + }, + { + "layer": "layer.68.mlp.down_proj", + "ppl": 4.261096477508545 + }, + { + "layer": "layer.69.self_attn.q_proj", + "ppl": 4.261402606964111 + }, + { + "layer": "layer.69.self_attn.k_proj", + "ppl": 4.261423110961914 + }, + { + "layer": "layer.69.self_attn.v_proj", + "ppl": 4.261227130889893 + }, + { + "layer": "layer.69.self_attn.o_proj", + "ppl": 4.261199951171875 + }, + { + "layer": "layer.69.mlp.gate_proj", + "ppl": 4.260489463806152 + }, + { + "layer": "layer.69.mlp.up_proj", + "ppl": 4.261216640472412 + }, + { + "layer": "layer.69.mlp.down_proj", + "ppl": 4.260995388031006 + }, + { + "layer": "layer.70.self_attn.q_proj", + "ppl": 4.26140022277832 + }, + { + "layer": "layer.70.self_attn.k_proj", + "ppl": 4.261453151702881 + }, + { + "layer": "layer.70.self_attn.v_proj", + "ppl": 4.261239051818848 + }, + { + "layer": "layer.70.self_attn.o_proj", + "ppl": 4.26119327545166 + }, + { + "layer": "layer.70.mlp.gate_proj", + "ppl": 4.260231018066406 + }, + { + "layer": "layer.70.mlp.up_proj", + "ppl": 4.260605335235596 + }, + { + "layer": "layer.70.mlp.down_proj", + "ppl": 4.260857105255127 + }, + { + "layer": "layer.71.self_attn.q_proj", + "ppl": 4.261422157287598 + }, + { + "layer": "layer.71.self_attn.k_proj", + "ppl": 4.261348247528076 + }, + { + "layer": "layer.71.self_attn.v_proj", + "ppl": 4.261312484741211 + }, + { + "layer": "layer.71.self_attn.o_proj", + "ppl": 4.261366367340088 + }, + { + "layer": "layer.71.mlp.gate_proj", + "ppl": 4.2603888511657715 + }, + { + "layer": "layer.71.mlp.up_proj", + "ppl": 4.260647773742676 + }, + { + "layer": "layer.71.mlp.down_proj", + "ppl": 4.261778354644775 + }, + { + "layer": "layer.72.self_attn.q_proj", + "ppl": 4.261290550231934 + }, + { + "layer": "layer.72.self_attn.k_proj", + "ppl": 4.2614030838012695 + }, + { + "layer": "layer.72.self_attn.v_proj", + "ppl": 4.2613019943237305 + }, + { + "layer": "layer.72.self_attn.o_proj", + "ppl": 4.261241912841797 + }, + { + "layer": "layer.72.mlp.gate_proj", + "ppl": 4.2603654861450195 + }, + { + "layer": "layer.72.mlp.up_proj", + "ppl": 4.261109828948975 + }, + { + "layer": "layer.72.mlp.down_proj", + "ppl": 4.261168003082275 + }, + { + "layer": "layer.73.self_attn.q_proj", + "ppl": 4.261013984680176 + }, + { + "layer": "layer.73.self_attn.k_proj", + "ppl": 4.261250019073486 + }, + { + "layer": "layer.73.self_attn.v_proj", + "ppl": 4.261373996734619 + }, + { + "layer": "layer.73.self_attn.o_proj", + "ppl": 4.261590480804443 + }, + { + "layer": "layer.73.mlp.gate_proj", + "ppl": 4.26090145111084 + }, + { + "layer": "layer.73.mlp.up_proj", + "ppl": 4.2613725662231445 + }, + { + "layer": "layer.73.mlp.down_proj", + "ppl": 4.260936260223389 + }, + { + "layer": "layer.74.self_attn.q_proj", + "ppl": 4.261415481567383 + }, + { + "layer": "layer.74.self_attn.k_proj", + "ppl": 4.261397361755371 + }, + { + "layer": "layer.74.self_attn.v_proj", + "ppl": 4.261354923248291 + }, + { + "layer": "layer.74.self_attn.o_proj", + "ppl": 4.26130485534668 + }, + { + "layer": "layer.74.mlp.gate_proj", + "ppl": 4.260402202606201 + }, + { + "layer": "layer.74.mlp.up_proj", + "ppl": 4.260822296142578 + }, + { + "layer": "layer.74.mlp.down_proj", + "ppl": 4.2614264488220215 + }, + { + "layer": "layer.75.self_attn.q_proj", + "ppl": 4.261445045471191 + }, + { + "layer": "layer.75.self_attn.k_proj", + "ppl": 4.261274337768555 + }, + { + "layer": "layer.75.self_attn.v_proj", + "ppl": 4.261397361755371 + }, + { + "layer": "layer.75.self_attn.o_proj", + "ppl": 4.261356830596924 + }, + { + "layer": "layer.75.mlp.gate_proj", + "ppl": 4.259439468383789 + }, + { + "layer": "layer.75.mlp.up_proj", + "ppl": 4.258745193481445 + }, + { + "layer": "layer.75.mlp.down_proj", + "ppl": 4.260847568511963 + }, + { + "layer": "layer.76.self_attn.q_proj", + "ppl": 4.261329174041748 + }, + { + "layer": "layer.76.self_attn.k_proj", + "ppl": 4.261320114135742 + }, + { + "layer": "layer.76.self_attn.v_proj", + "ppl": 4.261723518371582 + }, + { + "layer": "layer.76.self_attn.o_proj", + "ppl": 4.261177062988281 + }, + { + "layer": "layer.76.mlp.gate_proj", + "ppl": 4.260202884674072 + }, + { + "layer": "layer.76.mlp.up_proj", + "ppl": 4.260838508605957 + }, + { + "layer": "layer.76.mlp.down_proj", + "ppl": 4.260563850402832 + }, + { + "layer": "layer.77.self_attn.q_proj", + "ppl": 4.261816501617432 + }, + { + "layer": "layer.77.self_attn.k_proj", + "ppl": 4.261425971984863 + }, + { + "layer": "layer.77.self_attn.v_proj", + "ppl": 4.261273384094238 + }, + { + "layer": "layer.77.self_attn.o_proj", + "ppl": 4.261237621307373 + }, + { + "layer": "layer.77.mlp.gate_proj", + "ppl": 4.260368347167969 + }, + { + "layer": "layer.77.mlp.up_proj", + "ppl": 4.2604241371154785 + }, + { + "layer": "layer.77.mlp.down_proj", + "ppl": 4.260264873504639 + }, + { + "layer": "layer.78.self_attn.q_proj", + "ppl": 4.261312007904053 + }, + { + "layer": "layer.78.self_attn.k_proj", + "ppl": 4.261512756347656 + }, + { + "layer": "layer.78.self_attn.v_proj", + "ppl": 4.2612624168396 + }, + { + "layer": "layer.78.self_attn.o_proj", + "ppl": 4.2618536949157715 + }, + { + "layer": "layer.78.mlp.gate_proj", + "ppl": 4.259432792663574 + }, + { + "layer": "layer.78.mlp.up_proj", + "ppl": 4.259665012359619 + }, + { + "layer": "layer.78.mlp.down_proj", + "ppl": 4.260185718536377 + }, + { + "layer": "layer.79.self_attn.q_proj", + "ppl": 4.261204242706299 + }, + { + "layer": "layer.79.self_attn.k_proj", + "ppl": 4.26129150390625 + }, + { + "layer": "layer.79.self_attn.v_proj", + "ppl": 4.261369705200195 + }, + { + "layer": "layer.79.self_attn.o_proj", + "ppl": 4.261690139770508 + }, + { + "layer": "layer.79.mlp.gate_proj", + "ppl": 4.258448123931885 + }, + { + "layer": "layer.79.mlp.up_proj", + "ppl": 4.258193016052246 + }, + { + "layer": "layer.79.mlp.down_proj", + "ppl": 4.257203578948975 + } + ] + }, + { + "bit": 7, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 4.186372756958008 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 4.1863884925842285 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 4.186199188232422 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 4.186277389526367 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 4.186336040496826 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 4.186314582824707 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 4.186440944671631 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 4.186336994171143 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 4.186275005340576 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 4.186288356781006 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 4.18624210357666 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 4.186311721801758 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 4.186207294464111 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 4.186440944671631 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 4.186225414276123 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 4.186223983764648 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 4.186328887939453 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 4.186161041259766 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 4.186176776885986 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 4.1862077713012695 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 4.186196804046631 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 4.186308860778809 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 4.18618631362915 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 4.186229228973389 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 4.186213970184326 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 4.186230182647705 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 4.186188220977783 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 4.186152458190918 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 4.186173439025879 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 4.186283588409424 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 4.186316013336182 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 4.186194896697998 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 4.186404705047607 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 4.186246871948242 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 4.186268329620361 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 4.186337947845459 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 4.186323165893555 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 4.186389923095703 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 4.186186790466309 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 4.18631649017334 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 4.18625545501709 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 4.186236381530762 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 4.186186790466309 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 4.1863484382629395 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 4.186227321624756 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 4.186341285705566 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 4.186215877532959 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 4.1862568855285645 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 4.18623161315918 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 4.1863508224487305 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 4.186409950256348 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 4.186135292053223 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 4.186237812042236 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 4.186295032501221 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 4.186219692230225 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 4.186336040496826 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 4.186298847198486 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 4.186341285705566 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 4.186172008514404 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 4.186389923095703 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 4.186378002166748 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 4.186152935028076 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 4.186192512512207 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 4.186145782470703 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 4.186294078826904 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 4.186206817626953 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 4.186352729797363 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 4.1862616539001465 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 4.186254501342773 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 4.18626594543457 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 4.186275959014893 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 4.186337947845459 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 4.186315059661865 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 4.186333179473877 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 4.1862030029296875 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 4.186242580413818 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 4.186213970184326 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 4.186261177062988 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 4.186338424682617 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 4.186300754547119 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 4.186383247375488 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 4.186288356781006 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 4.186200141906738 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 4.186295032501221 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 4.186347007751465 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 4.186266899108887 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 4.186342239379883 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 4.18638277053833 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 4.186420440673828 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 4.1864752769470215 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 4.186418056488037 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 4.186220169067383 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 4.186411380767822 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 4.186223030090332 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 4.186162948608398 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 4.186255931854248 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 4.186245441436768 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 4.18626070022583 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 4.186339378356934 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 4.186419486999512 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 4.186261177062988 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 4.186308860778809 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 4.186252117156982 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 4.186289310455322 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 4.186079025268555 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 4.186309814453125 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 4.186169147491455 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 4.186496734619141 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 4.186392784118652 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 4.186478614807129 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 4.186278343200684 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 4.1863250732421875 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 4.186348915100098 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 4.186376571655273 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 4.186352729797363 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 4.186208248138428 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 4.186305046081543 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 4.18625545501709 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 4.186156272888184 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 4.186252117156982 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 4.186333179473877 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 4.186321258544922 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 4.186326026916504 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 4.1863322257995605 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 4.186356544494629 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 4.186150550842285 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 4.186230659484863 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 4.186328887939453 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 4.186305522918701 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 4.18630313873291 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 4.186347007751465 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 4.186338424682617 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 4.186319828033447 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 4.186415672302246 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 4.1862897872924805 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 4.186262607574463 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 4.186175346374512 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 4.186262607574463 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 4.186227321624756 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 4.186254501342773 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 4.186247825622559 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 4.186254501342773 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 4.186230659484863 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 4.186277389526367 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 4.186257839202881 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 4.18640661239624 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 4.18630313873291 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 4.18635368347168 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 4.186347961425781 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 4.186366081237793 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 4.186184406280518 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 4.186351299285889 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 4.186233997344971 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 4.186196327209473 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 4.186313152313232 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 4.186316967010498 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 4.186245441436768 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 4.186338901519775 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 4.186161041259766 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 4.186366081237793 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 4.186156272888184 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 4.186276912689209 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 4.1860761642456055 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 4.186240196228027 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 4.186252117156982 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 4.186263561248779 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 4.186175346374512 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 4.186039447784424 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 4.186307907104492 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 4.186230659484863 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 4.186367511749268 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 4.186429023742676 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 4.186038494110107 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 4.186166763305664 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 4.186366081237793 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 4.186270236968994 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 4.186284065246582 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 4.186219692230225 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 4.186209201812744 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 4.186341285705566 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 4.186347961425781 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 4.186280727386475 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 4.186286926269531 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 4.186348915100098 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 4.186185359954834 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 4.186404228210449 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 4.186336517333984 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 4.186197757720947 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 4.1860833168029785 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 4.186309814453125 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 4.186431407928467 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 4.186275005340576 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 4.186203956604004 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 4.186202526092529 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 4.186319828033447 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 4.186280727386475 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 4.186206340789795 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 4.18626594543457 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 4.1862077713012695 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 4.186298370361328 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 4.186089992523193 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 4.186368942260742 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 4.18624210357666 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 4.186167240142822 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 4.186208248138428 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 4.18612813949585 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 4.1862640380859375 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 4.186027526855469 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 4.186188220977783 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 4.18637752532959 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 4.186247825622559 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 4.186347961425781 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 4.1860857009887695 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 4.186157703399658 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 4.186164379119873 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 4.186399936676025 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 4.186458587646484 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 4.186284065246582 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 4.186282634735107 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 4.186187744140625 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 4.186407566070557 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 4.186215877532959 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 4.18631649017334 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 4.186326026916504 + }, + { + "layer": "layer.32.self_attn.q_proj", + "ppl": 4.186253070831299 + }, + { + "layer": "layer.32.self_attn.k_proj", + "ppl": 4.186363697052002 + }, + { + "layer": "layer.32.self_attn.v_proj", + "ppl": 4.186366558074951 + }, + { + "layer": "layer.32.self_attn.o_proj", + "ppl": 4.186305046081543 + }, + { + "layer": "layer.32.mlp.gate_proj", + "ppl": 4.185941696166992 + }, + { + "layer": "layer.32.mlp.up_proj", + "ppl": 4.186021327972412 + }, + { + "layer": "layer.32.mlp.down_proj", + "ppl": 4.186036586761475 + }, + { + "layer": "layer.33.self_attn.q_proj", + "ppl": 4.186221599578857 + }, + { + "layer": "layer.33.self_attn.k_proj", + "ppl": 4.186294078826904 + }, + { + "layer": "layer.33.self_attn.v_proj", + "ppl": 4.186260223388672 + }, + { + "layer": "layer.33.self_attn.o_proj", + "ppl": 4.186424732208252 + }, + { + "layer": "layer.33.mlp.gate_proj", + "ppl": 4.186076641082764 + }, + { + "layer": "layer.33.mlp.up_proj", + "ppl": 4.186182498931885 + }, + { + "layer": "layer.33.mlp.down_proj", + "ppl": 4.1856207847595215 + }, + { + "layer": "layer.34.self_attn.q_proj", + "ppl": 4.186272144317627 + }, + { + "layer": "layer.34.self_attn.k_proj", + "ppl": 4.186305046081543 + }, + { + "layer": "layer.34.self_attn.v_proj", + "ppl": 4.186331272125244 + }, + { + "layer": "layer.34.self_attn.o_proj", + "ppl": 4.186303615570068 + }, + { + "layer": "layer.34.mlp.gate_proj", + "ppl": 4.18605375289917 + }, + { + "layer": "layer.34.mlp.up_proj", + "ppl": 4.186615467071533 + }, + { + "layer": "layer.34.mlp.down_proj", + "ppl": 4.186039447784424 + }, + { + "layer": "layer.35.self_attn.q_proj", + "ppl": 4.186359405517578 + }, + { + "layer": "layer.35.self_attn.k_proj", + "ppl": 4.186418056488037 + }, + { + "layer": "layer.35.self_attn.v_proj", + "ppl": 4.186468601226807 + }, + { + "layer": "layer.35.self_attn.o_proj", + "ppl": 4.186190128326416 + }, + { + "layer": "layer.35.mlp.gate_proj", + "ppl": 4.186075210571289 + }, + { + "layer": "layer.35.mlp.up_proj", + "ppl": 4.186532974243164 + }, + { + "layer": "layer.35.mlp.down_proj", + "ppl": 4.185781955718994 + }, + { + "layer": "layer.36.self_attn.q_proj", + "ppl": 4.186260223388672 + }, + { + "layer": "layer.36.self_attn.k_proj", + "ppl": 4.186240196228027 + }, + { + "layer": "layer.36.self_attn.v_proj", + "ppl": 4.186298847198486 + }, + { + "layer": "layer.36.self_attn.o_proj", + "ppl": 4.18617582321167 + }, + { + "layer": "layer.36.mlp.gate_proj", + "ppl": 4.18604850769043 + }, + { + "layer": "layer.36.mlp.up_proj", + "ppl": 4.1857428550720215 + }, + { + "layer": "layer.36.mlp.down_proj", + "ppl": 4.186122417449951 + }, + { + "layer": "layer.37.self_attn.q_proj", + "ppl": 4.186306476593018 + }, + { + "layer": "layer.37.self_attn.k_proj", + "ppl": 4.186308860778809 + }, + { + "layer": "layer.37.self_attn.v_proj", + "ppl": 4.186268329620361 + }, + { + "layer": "layer.37.self_attn.o_proj", + "ppl": 4.18621301651001 + }, + { + "layer": "layer.37.mlp.gate_proj", + "ppl": 4.1862263679504395 + }, + { + "layer": "layer.37.mlp.up_proj", + "ppl": 4.186184406280518 + }, + { + "layer": "layer.37.mlp.down_proj", + "ppl": 4.185766696929932 + }, + { + "layer": "layer.38.self_attn.q_proj", + "ppl": 4.18621301651001 + }, + { + "layer": "layer.38.self_attn.k_proj", + "ppl": 4.186398506164551 + }, + { + "layer": "layer.38.self_attn.v_proj", + "ppl": 4.186360836029053 + }, + { + "layer": "layer.38.self_attn.o_proj", + "ppl": 4.186176300048828 + }, + { + "layer": "layer.38.mlp.gate_proj", + "ppl": 4.1862640380859375 + }, + { + "layer": "layer.38.mlp.up_proj", + "ppl": 4.186015605926514 + }, + { + "layer": "layer.38.mlp.down_proj", + "ppl": 4.186309814453125 + }, + { + "layer": "layer.39.self_attn.q_proj", + "ppl": 4.186390399932861 + }, + { + "layer": "layer.39.self_attn.k_proj", + "ppl": 4.18624210357666 + }, + { + "layer": "layer.39.self_attn.v_proj", + "ppl": 4.186305999755859 + }, + { + "layer": "layer.39.self_attn.o_proj", + "ppl": 4.18632698059082 + }, + { + "layer": "layer.39.mlp.gate_proj", + "ppl": 4.185667514801025 + }, + { + "layer": "layer.39.mlp.up_proj", + "ppl": 4.186285018920898 + }, + { + "layer": "layer.39.mlp.down_proj", + "ppl": 4.186146259307861 + }, + { + "layer": "layer.40.self_attn.q_proj", + "ppl": 4.186246871948242 + }, + { + "layer": "layer.40.self_attn.k_proj", + "ppl": 4.186425685882568 + }, + { + "layer": "layer.40.self_attn.v_proj", + "ppl": 4.186245441436768 + }, + { + "layer": "layer.40.self_attn.o_proj", + "ppl": 4.186233043670654 + }, + { + "layer": "layer.40.mlp.gate_proj", + "ppl": 4.185969829559326 + }, + { + "layer": "layer.40.mlp.up_proj", + "ppl": 4.186387538909912 + }, + { + "layer": "layer.40.mlp.down_proj", + "ppl": 4.186020374298096 + }, + { + "layer": "layer.41.self_attn.q_proj", + "ppl": 4.186247825622559 + }, + { + "layer": "layer.41.self_attn.k_proj", + "ppl": 4.18637752532959 + }, + { + "layer": "layer.41.self_attn.v_proj", + "ppl": 4.186497688293457 + }, + { + "layer": "layer.41.self_attn.o_proj", + "ppl": 4.186433792114258 + }, + { + "layer": "layer.41.mlp.gate_proj", + "ppl": 4.186013221740723 + }, + { + "layer": "layer.41.mlp.up_proj", + "ppl": 4.186525344848633 + }, + { + "layer": "layer.41.mlp.down_proj", + "ppl": 4.186285495758057 + }, + { + "layer": "layer.42.self_attn.q_proj", + "ppl": 4.186367034912109 + }, + { + "layer": "layer.42.self_attn.k_proj", + "ppl": 4.186261177062988 + }, + { + "layer": "layer.42.self_attn.v_proj", + "ppl": 4.186344623565674 + }, + { + "layer": "layer.42.self_attn.o_proj", + "ppl": 4.18621826171875 + }, + { + "layer": "layer.42.mlp.gate_proj", + "ppl": 4.186509609222412 + }, + { + "layer": "layer.42.mlp.up_proj", + "ppl": 4.1860833168029785 + }, + { + "layer": "layer.42.mlp.down_proj", + "ppl": 4.186442852020264 + }, + { + "layer": "layer.43.self_attn.q_proj", + "ppl": 4.186308860778809 + }, + { + "layer": "layer.43.self_attn.k_proj", + "ppl": 4.186277389526367 + }, + { + "layer": "layer.43.self_attn.v_proj", + "ppl": 4.186215877532959 + }, + { + "layer": "layer.43.self_attn.o_proj", + "ppl": 4.186264991760254 + }, + { + "layer": "layer.43.mlp.gate_proj", + "ppl": 4.186488151550293 + }, + { + "layer": "layer.43.mlp.up_proj", + "ppl": 4.18623685836792 + }, + { + "layer": "layer.43.mlp.down_proj", + "ppl": 4.1863179206848145 + }, + { + "layer": "layer.44.self_attn.q_proj", + "ppl": 4.18626070022583 + }, + { + "layer": "layer.44.self_attn.k_proj", + "ppl": 4.186341285705566 + }, + { + "layer": "layer.44.self_attn.v_proj", + "ppl": 4.186372756958008 + }, + { + "layer": "layer.44.self_attn.o_proj", + "ppl": 4.186197280883789 + }, + { + "layer": "layer.44.mlp.gate_proj", + "ppl": 4.186285495758057 + }, + { + "layer": "layer.44.mlp.up_proj", + "ppl": 4.186179161071777 + }, + { + "layer": "layer.44.mlp.down_proj", + "ppl": 4.186688423156738 + }, + { + "layer": "layer.45.self_attn.q_proj", + "ppl": 4.186424732208252 + }, + { + "layer": "layer.45.self_attn.k_proj", + "ppl": 4.186485290527344 + }, + { + "layer": "layer.45.self_attn.v_proj", + "ppl": 4.186123371124268 + }, + { + "layer": "layer.45.self_attn.o_proj", + "ppl": 4.186136245727539 + }, + { + "layer": "layer.45.mlp.gate_proj", + "ppl": 4.186146259307861 + }, + { + "layer": "layer.45.mlp.up_proj", + "ppl": 4.186200141906738 + }, + { + "layer": "layer.45.mlp.down_proj", + "ppl": 4.186044692993164 + }, + { + "layer": "layer.46.self_attn.q_proj", + "ppl": 4.1860761642456055 + }, + { + "layer": "layer.46.self_attn.k_proj", + "ppl": 4.186342239379883 + }, + { + "layer": "layer.46.self_attn.v_proj", + "ppl": 4.186184406280518 + }, + { + "layer": "layer.46.self_attn.o_proj", + "ppl": 4.186531066894531 + }, + { + "layer": "layer.46.mlp.gate_proj", + "ppl": 4.186290740966797 + }, + { + "layer": "layer.46.mlp.up_proj", + "ppl": 4.1861772537231445 + }, + { + "layer": "layer.46.mlp.down_proj", + "ppl": 4.186206340789795 + }, + { + "layer": "layer.47.self_attn.q_proj", + "ppl": 4.186355113983154 + }, + { + "layer": "layer.47.self_attn.k_proj", + "ppl": 4.186410427093506 + }, + { + "layer": "layer.47.self_attn.v_proj", + "ppl": 4.186341285705566 + }, + { + "layer": "layer.47.self_attn.o_proj", + "ppl": 4.186351776123047 + }, + { + "layer": "layer.47.mlp.gate_proj", + "ppl": 4.1860785484313965 + }, + { + "layer": "layer.47.mlp.up_proj", + "ppl": 4.1862101554870605 + }, + { + "layer": "layer.47.mlp.down_proj", + "ppl": 4.1864094734191895 + }, + { + "layer": "layer.48.self_attn.q_proj", + "ppl": 4.186445236206055 + }, + { + "layer": "layer.48.self_attn.k_proj", + "ppl": 4.186313629150391 + }, + { + "layer": "layer.48.self_attn.v_proj", + "ppl": 4.186284065246582 + }, + { + "layer": "layer.48.self_attn.o_proj", + "ppl": 4.186237335205078 + }, + { + "layer": "layer.48.mlp.gate_proj", + "ppl": 4.186213970184326 + }, + { + "layer": "layer.48.mlp.up_proj", + "ppl": 4.186167240142822 + }, + { + "layer": "layer.48.mlp.down_proj", + "ppl": 4.186544418334961 + }, + { + "layer": "layer.49.self_attn.q_proj", + "ppl": 4.186445236206055 + }, + { + "layer": "layer.49.self_attn.k_proj", + "ppl": 4.186278343200684 + }, + { + "layer": "layer.49.self_attn.v_proj", + "ppl": 4.186316013336182 + }, + { + "layer": "layer.49.self_attn.o_proj", + "ppl": 4.186208248138428 + }, + { + "layer": "layer.49.mlp.gate_proj", + "ppl": 4.186262130737305 + }, + { + "layer": "layer.49.mlp.up_proj", + "ppl": 4.186176300048828 + }, + { + "layer": "layer.49.mlp.down_proj", + "ppl": 4.186154365539551 + }, + { + "layer": "layer.50.self_attn.q_proj", + "ppl": 4.18645715713501 + }, + { + "layer": "layer.50.self_attn.k_proj", + "ppl": 4.186262607574463 + }, + { + "layer": "layer.50.self_attn.v_proj", + "ppl": 4.186692237854004 + }, + { + "layer": "layer.50.self_attn.o_proj", + "ppl": 4.186476230621338 + }, + { + "layer": "layer.50.mlp.gate_proj", + "ppl": 4.186066627502441 + }, + { + "layer": "layer.50.mlp.up_proj", + "ppl": 4.186346054077148 + }, + { + "layer": "layer.50.mlp.down_proj", + "ppl": 4.1864800453186035 + }, + { + "layer": "layer.51.self_attn.q_proj", + "ppl": 4.186498641967773 + }, + { + "layer": "layer.51.self_attn.k_proj", + "ppl": 4.186522483825684 + }, + { + "layer": "layer.51.self_attn.v_proj", + "ppl": 4.1864013671875 + }, + { + "layer": "layer.51.self_attn.o_proj", + "ppl": 4.186280250549316 + }, + { + "layer": "layer.51.mlp.gate_proj", + "ppl": 4.186087608337402 + }, + { + "layer": "layer.51.mlp.up_proj", + "ppl": 4.186022758483887 + }, + { + "layer": "layer.51.mlp.down_proj", + "ppl": 4.185744285583496 + }, + { + "layer": "layer.52.self_attn.q_proj", + "ppl": 4.1862568855285645 + }, + { + "layer": "layer.52.self_attn.k_proj", + "ppl": 4.186323165893555 + }, + { + "layer": "layer.52.self_attn.v_proj", + "ppl": 4.18621826171875 + }, + { + "layer": "layer.52.self_attn.o_proj", + "ppl": 4.186223030090332 + }, + { + "layer": "layer.52.mlp.gate_proj", + "ppl": 4.185874938964844 + }, + { + "layer": "layer.52.mlp.up_proj", + "ppl": 4.186121940612793 + }, + { + "layer": "layer.52.mlp.down_proj", + "ppl": 4.186245441436768 + }, + { + "layer": "layer.53.self_attn.q_proj", + "ppl": 4.18630838394165 + }, + { + "layer": "layer.53.self_attn.k_proj", + "ppl": 4.186452865600586 + }, + { + "layer": "layer.53.self_attn.v_proj", + "ppl": 4.186462879180908 + }, + { + "layer": "layer.53.self_attn.o_proj", + "ppl": 4.186219215393066 + }, + { + "layer": "layer.53.mlp.gate_proj", + "ppl": 4.18600606918335 + }, + { + "layer": "layer.53.mlp.up_proj", + "ppl": 4.186634540557861 + }, + { + "layer": "layer.53.mlp.down_proj", + "ppl": 4.1862592697143555 + }, + { + "layer": "layer.54.self_attn.q_proj", + "ppl": 4.186376571655273 + }, + { + "layer": "layer.54.self_attn.k_proj", + "ppl": 4.186229228973389 + }, + { + "layer": "layer.54.self_attn.v_proj", + "ppl": 4.186298370361328 + }, + { + "layer": "layer.54.self_attn.o_proj", + "ppl": 4.1863884925842285 + }, + { + "layer": "layer.54.mlp.gate_proj", + "ppl": 4.186020374298096 + }, + { + "layer": "layer.54.mlp.up_proj", + "ppl": 4.186367034912109 + }, + { + "layer": "layer.54.mlp.down_proj", + "ppl": 4.186435699462891 + }, + { + "layer": "layer.55.self_attn.q_proj", + "ppl": 4.186319828033447 + }, + { + "layer": "layer.55.self_attn.k_proj", + "ppl": 4.186356067657471 + }, + { + "layer": "layer.55.self_attn.v_proj", + "ppl": 4.186228275299072 + }, + { + "layer": "layer.55.self_attn.o_proj", + "ppl": 4.186252117156982 + }, + { + "layer": "layer.55.mlp.gate_proj", + "ppl": 4.185614585876465 + }, + { + "layer": "layer.55.mlp.up_proj", + "ppl": 4.186408042907715 + }, + { + "layer": "layer.55.mlp.down_proj", + "ppl": 4.18599796295166 + }, + { + "layer": "layer.56.self_attn.q_proj", + "ppl": 4.186352729797363 + }, + { + "layer": "layer.56.self_attn.k_proj", + "ppl": 4.1862077713012695 + }, + { + "layer": "layer.56.self_attn.v_proj", + "ppl": 4.18612813949585 + }, + { + "layer": "layer.56.self_attn.o_proj", + "ppl": 4.186117172241211 + }, + { + "layer": "layer.56.mlp.gate_proj", + "ppl": 4.18593168258667 + }, + { + "layer": "layer.56.mlp.up_proj", + "ppl": 4.186459541320801 + }, + { + "layer": "layer.56.mlp.down_proj", + "ppl": 4.18595027923584 + }, + { + "layer": "layer.57.self_attn.q_proj", + "ppl": 4.186455726623535 + }, + { + "layer": "layer.57.self_attn.k_proj", + "ppl": 4.186263084411621 + }, + { + "layer": "layer.57.self_attn.v_proj", + "ppl": 4.1863508224487305 + }, + { + "layer": "layer.57.self_attn.o_proj", + "ppl": 4.186232089996338 + }, + { + "layer": "layer.57.mlp.gate_proj", + "ppl": 4.185940265655518 + }, + { + "layer": "layer.57.mlp.up_proj", + "ppl": 4.186091423034668 + }, + { + "layer": "layer.57.mlp.down_proj", + "ppl": 4.186230182647705 + }, + { + "layer": "layer.58.self_attn.q_proj", + "ppl": 4.186433792114258 + }, + { + "layer": "layer.58.self_attn.k_proj", + "ppl": 4.1863694190979 + }, + { + "layer": "layer.58.self_attn.v_proj", + "ppl": 4.186244010925293 + }, + { + "layer": "layer.58.self_attn.o_proj", + "ppl": 4.18631649017334 + }, + { + "layer": "layer.58.mlp.gate_proj", + "ppl": 4.185822010040283 + }, + { + "layer": "layer.58.mlp.up_proj", + "ppl": 4.186233043670654 + }, + { + "layer": "layer.58.mlp.down_proj", + "ppl": 4.186621189117432 + }, + { + "layer": "layer.59.self_attn.q_proj", + "ppl": 4.186411380767822 + }, + { + "layer": "layer.59.self_attn.k_proj", + "ppl": 4.186349868774414 + }, + { + "layer": "layer.59.self_attn.v_proj", + "ppl": 4.186073303222656 + }, + { + "layer": "layer.59.self_attn.o_proj", + "ppl": 4.186099529266357 + }, + { + "layer": "layer.59.mlp.gate_proj", + "ppl": 4.186074256896973 + }, + { + "layer": "layer.59.mlp.up_proj", + "ppl": 4.186097621917725 + }, + { + "layer": "layer.59.mlp.down_proj", + "ppl": 4.186094284057617 + }, + { + "layer": "layer.60.self_attn.q_proj", + "ppl": 4.186303615570068 + }, + { + "layer": "layer.60.self_attn.k_proj", + "ppl": 4.186412811279297 + }, + { + "layer": "layer.60.self_attn.v_proj", + "ppl": 4.1865010261535645 + }, + { + "layer": "layer.60.self_attn.o_proj", + "ppl": 4.186326503753662 + }, + { + "layer": "layer.60.mlp.gate_proj", + "ppl": 4.185867786407471 + }, + { + "layer": "layer.60.mlp.up_proj", + "ppl": 4.186615467071533 + }, + { + "layer": "layer.60.mlp.down_proj", + "ppl": 4.186367034912109 + }, + { + "layer": "layer.61.self_attn.q_proj", + "ppl": 4.186333179473877 + }, + { + "layer": "layer.61.self_attn.k_proj", + "ppl": 4.1864213943481445 + }, + { + "layer": "layer.61.self_attn.v_proj", + "ppl": 4.186137676239014 + }, + { + "layer": "layer.61.self_attn.o_proj", + "ppl": 4.186196804046631 + }, + { + "layer": "layer.61.mlp.gate_proj", + "ppl": 4.185671329498291 + }, + { + "layer": "layer.61.mlp.up_proj", + "ppl": 4.186089515686035 + }, + { + "layer": "layer.61.mlp.down_proj", + "ppl": 4.186068058013916 + }, + { + "layer": "layer.62.self_attn.q_proj", + "ppl": 4.1861252784729 + }, + { + "layer": "layer.62.self_attn.k_proj", + "ppl": 4.186322212219238 + }, + { + "layer": "layer.62.self_attn.v_proj", + "ppl": 4.186228275299072 + }, + { + "layer": "layer.62.self_attn.o_proj", + "ppl": 4.186114311218262 + }, + { + "layer": "layer.62.mlp.gate_proj", + "ppl": 4.185441017150879 + }, + { + "layer": "layer.62.mlp.up_proj", + "ppl": 4.1861162185668945 + }, + { + "layer": "layer.62.mlp.down_proj", + "ppl": 4.186342239379883 + }, + { + "layer": "layer.63.self_attn.q_proj", + "ppl": 4.186472415924072 + }, + { + "layer": "layer.63.self_attn.k_proj", + "ppl": 4.186347007751465 + }, + { + "layer": "layer.63.self_attn.v_proj", + "ppl": 4.186247825622559 + }, + { + "layer": "layer.63.self_attn.o_proj", + "ppl": 4.186073303222656 + }, + { + "layer": "layer.63.mlp.gate_proj", + "ppl": 4.185977458953857 + }, + { + "layer": "layer.63.mlp.up_proj", + "ppl": 4.186094760894775 + }, + { + "layer": "layer.63.mlp.down_proj", + "ppl": 4.186333656311035 + }, + { + "layer": "layer.64.self_attn.q_proj", + "ppl": 4.186273574829102 + }, + { + "layer": "layer.64.self_attn.k_proj", + "ppl": 4.1863627433776855 + }, + { + "layer": "layer.64.self_attn.v_proj", + "ppl": 4.186305999755859 + }, + { + "layer": "layer.64.self_attn.o_proj", + "ppl": 4.186153411865234 + }, + { + "layer": "layer.64.mlp.gate_proj", + "ppl": 4.185579299926758 + }, + { + "layer": "layer.64.mlp.up_proj", + "ppl": 4.186405658721924 + }, + { + "layer": "layer.64.mlp.down_proj", + "ppl": 4.186001300811768 + }, + { + "layer": "layer.65.self_attn.q_proj", + "ppl": 4.186164379119873 + }, + { + "layer": "layer.65.self_attn.k_proj", + "ppl": 4.186196804046631 + }, + { + "layer": "layer.65.self_attn.v_proj", + "ppl": 4.186297416687012 + }, + { + "layer": "layer.65.self_attn.o_proj", + "ppl": 4.1863627433776855 + }, + { + "layer": "layer.65.mlp.gate_proj", + "ppl": 4.185702323913574 + }, + { + "layer": "layer.65.mlp.up_proj", + "ppl": 4.185889720916748 + }, + { + "layer": "layer.65.mlp.down_proj", + "ppl": 4.186028480529785 + }, + { + "layer": "layer.66.self_attn.q_proj", + "ppl": 4.18640661239624 + }, + { + "layer": "layer.66.self_attn.k_proj", + "ppl": 4.186206340789795 + }, + { + "layer": "layer.66.self_attn.v_proj", + "ppl": 4.186344623565674 + }, + { + "layer": "layer.66.self_attn.o_proj", + "ppl": 4.186374187469482 + }, + { + "layer": "layer.66.mlp.gate_proj", + "ppl": 4.185770034790039 + }, + { + "layer": "layer.66.mlp.up_proj", + "ppl": 4.186100006103516 + }, + { + "layer": "layer.66.mlp.down_proj", + "ppl": 4.1860575675964355 + }, + { + "layer": "layer.67.self_attn.q_proj", + "ppl": 4.186339855194092 + }, + { + "layer": "layer.67.self_attn.k_proj", + "ppl": 4.186408996582031 + }, + { + "layer": "layer.67.self_attn.v_proj", + "ppl": 4.186203956604004 + }, + { + "layer": "layer.67.self_attn.o_proj", + "ppl": 4.1864237785339355 + }, + { + "layer": "layer.67.mlp.gate_proj", + "ppl": 4.185878753662109 + }, + { + "layer": "layer.67.mlp.up_proj", + "ppl": 4.186182498931885 + }, + { + "layer": "layer.67.mlp.down_proj", + "ppl": 4.18604040145874 + }, + { + "layer": "layer.68.self_attn.q_proj", + "ppl": 4.186262130737305 + }, + { + "layer": "layer.68.self_attn.k_proj", + "ppl": 4.186280250549316 + }, + { + "layer": "layer.68.self_attn.v_proj", + "ppl": 4.186185359954834 + }, + { + "layer": "layer.68.self_attn.o_proj", + "ppl": 4.186076641082764 + }, + { + "layer": "layer.68.mlp.gate_proj", + "ppl": 4.1857171058654785 + }, + { + "layer": "layer.68.mlp.up_proj", + "ppl": 4.186068534851074 + }, + { + "layer": "layer.68.mlp.down_proj", + "ppl": 4.185908794403076 + }, + { + "layer": "layer.69.self_attn.q_proj", + "ppl": 4.186415672302246 + }, + { + "layer": "layer.69.self_attn.k_proj", + "ppl": 4.186293125152588 + }, + { + "layer": "layer.69.self_attn.v_proj", + "ppl": 4.186225414276123 + }, + { + "layer": "layer.69.self_attn.o_proj", + "ppl": 4.1863579750061035 + }, + { + "layer": "layer.69.mlp.gate_proj", + "ppl": 4.185760974884033 + }, + { + "layer": "layer.69.mlp.up_proj", + "ppl": 4.186081409454346 + }, + { + "layer": "layer.69.mlp.down_proj", + "ppl": 4.185802459716797 + }, + { + "layer": "layer.70.self_attn.q_proj", + "ppl": 4.186384201049805 + }, + { + "layer": "layer.70.self_attn.k_proj", + "ppl": 4.186388969421387 + }, + { + "layer": "layer.70.self_attn.v_proj", + "ppl": 4.186374187469482 + }, + { + "layer": "layer.70.self_attn.o_proj", + "ppl": 4.18646764755249 + }, + { + "layer": "layer.70.mlp.gate_proj", + "ppl": 4.185793399810791 + }, + { + "layer": "layer.70.mlp.up_proj", + "ppl": 4.18590784072876 + }, + { + "layer": "layer.70.mlp.down_proj", + "ppl": 4.185648441314697 + }, + { + "layer": "layer.71.self_attn.q_proj", + "ppl": 4.186456203460693 + }, + { + "layer": "layer.71.self_attn.k_proj", + "ppl": 4.186388969421387 + }, + { + "layer": "layer.71.self_attn.v_proj", + "ppl": 4.186346530914307 + }, + { + "layer": "layer.71.self_attn.o_proj", + "ppl": 4.186396598815918 + }, + { + "layer": "layer.71.mlp.gate_proj", + "ppl": 4.185815811157227 + }, + { + "layer": "layer.71.mlp.up_proj", + "ppl": 4.186066627502441 + }, + { + "layer": "layer.71.mlp.down_proj", + "ppl": 4.186343669891357 + }, + { + "layer": "layer.72.self_attn.q_proj", + "ppl": 4.1863508224487305 + }, + { + "layer": "layer.72.self_attn.k_proj", + "ppl": 4.186289310455322 + }, + { + "layer": "layer.72.self_attn.v_proj", + "ppl": 4.186245441436768 + }, + { + "layer": "layer.72.self_attn.o_proj", + "ppl": 4.186279296875 + }, + { + "layer": "layer.72.mlp.gate_proj", + "ppl": 4.185880184173584 + }, + { + "layer": "layer.72.mlp.up_proj", + "ppl": 4.186227798461914 + }, + { + "layer": "layer.72.mlp.down_proj", + "ppl": 4.185957908630371 + }, + { + "layer": "layer.73.self_attn.q_proj", + "ppl": 4.186237812042236 + }, + { + "layer": "layer.73.self_attn.k_proj", + "ppl": 4.186268329620361 + }, + { + "layer": "layer.73.self_attn.v_proj", + "ppl": 4.186468601226807 + }, + { + "layer": "layer.73.self_attn.o_proj", + "ppl": 4.186286926269531 + }, + { + "layer": "layer.73.mlp.gate_proj", + "ppl": 4.185567378997803 + }, + { + "layer": "layer.73.mlp.up_proj", + "ppl": 4.185986518859863 + }, + { + "layer": "layer.73.mlp.down_proj", + "ppl": 4.185732364654541 + }, + { + "layer": "layer.74.self_attn.q_proj", + "ppl": 4.186366558074951 + }, + { + "layer": "layer.74.self_attn.k_proj", + "ppl": 4.186212062835693 + }, + { + "layer": "layer.74.self_attn.v_proj", + "ppl": 4.186121463775635 + }, + { + "layer": "layer.74.self_attn.o_proj", + "ppl": 4.186252117156982 + }, + { + "layer": "layer.74.mlp.gate_proj", + "ppl": 4.185802936553955 + }, + { + "layer": "layer.74.mlp.up_proj", + "ppl": 4.185910701751709 + }, + { + "layer": "layer.74.mlp.down_proj", + "ppl": 4.186167240142822 + }, + { + "layer": "layer.75.self_attn.q_proj", + "ppl": 4.186282157897949 + }, + { + "layer": "layer.75.self_attn.k_proj", + "ppl": 4.1864333152771 + }, + { + "layer": "layer.75.self_attn.v_proj", + "ppl": 4.186073303222656 + }, + { + "layer": "layer.75.self_attn.o_proj", + "ppl": 4.186263084411621 + }, + { + "layer": "layer.75.mlp.gate_proj", + "ppl": 4.1858086585998535 + }, + { + "layer": "layer.75.mlp.up_proj", + "ppl": 4.185954570770264 + }, + { + "layer": "layer.75.mlp.down_proj", + "ppl": 4.186131000518799 + }, + { + "layer": "layer.76.self_attn.q_proj", + "ppl": 4.186413288116455 + }, + { + "layer": "layer.76.self_attn.k_proj", + "ppl": 4.186288356781006 + }, + { + "layer": "layer.76.self_attn.v_proj", + "ppl": 4.186446666717529 + }, + { + "layer": "layer.76.self_attn.o_proj", + "ppl": 4.186370849609375 + }, + { + "layer": "layer.76.mlp.gate_proj", + "ppl": 4.185305595397949 + }, + { + "layer": "layer.76.mlp.up_proj", + "ppl": 4.186069965362549 + }, + { + "layer": "layer.76.mlp.down_proj", + "ppl": 4.185941696166992 + }, + { + "layer": "layer.77.self_attn.q_proj", + "ppl": 4.186185836791992 + }, + { + "layer": "layer.77.self_attn.k_proj", + "ppl": 4.186549186706543 + }, + { + "layer": "layer.77.self_attn.v_proj", + "ppl": 4.186212539672852 + }, + { + "layer": "layer.77.self_attn.o_proj", + "ppl": 4.186379909515381 + }, + { + "layer": "layer.77.mlp.gate_proj", + "ppl": 4.185476779937744 + }, + { + "layer": "layer.77.mlp.up_proj", + "ppl": 4.185921669006348 + }, + { + "layer": "layer.77.mlp.down_proj", + "ppl": 4.185542106628418 + }, + { + "layer": "layer.78.self_attn.q_proj", + "ppl": 4.186193466186523 + }, + { + "layer": "layer.78.self_attn.k_proj", + "ppl": 4.186174392700195 + }, + { + "layer": "layer.78.self_attn.v_proj", + "ppl": 4.186479091644287 + }, + { + "layer": "layer.78.self_attn.o_proj", + "ppl": 4.186371803283691 + }, + { + "layer": "layer.78.mlp.gate_proj", + "ppl": 4.185886383056641 + }, + { + "layer": "layer.78.mlp.up_proj", + "ppl": 4.18556022644043 + }, + { + "layer": "layer.78.mlp.down_proj", + "ppl": 4.1855363845825195 + }, + { + "layer": "layer.79.self_attn.q_proj", + "ppl": 4.186171054840088 + }, + { + "layer": "layer.79.self_attn.k_proj", + "ppl": 4.186254501342773 + }, + { + "layer": "layer.79.self_attn.v_proj", + "ppl": 4.186473369598389 + }, + { + "layer": "layer.79.self_attn.o_proj", + "ppl": 4.18660831451416 + }, + { + "layer": "layer.79.mlp.gate_proj", + "ppl": 4.184767246246338 + }, + { + "layer": "layer.79.mlp.up_proj", + "ppl": 4.184300422668457 + }, + { + "layer": "layer.79.mlp.down_proj", + "ppl": 4.183382034301758 + } + ] + }, + { + "bit": 8, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 4.1388020515441895 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 4.138944625854492 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 4.138950347900391 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 4.138954162597656 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 4.138841152191162 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 4.138920307159424 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 4.13893985748291 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 4.138988494873047 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 4.138920307159424 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 4.138862133026123 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 4.138869762420654 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 4.138866901397705 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 4.138877868652344 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 4.138900279998779 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 4.1389617919921875 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 4.138861179351807 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 4.1389360427856445 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 4.138869762420654 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 4.138935089111328 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 4.1388258934021 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 4.139078140258789 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 4.138914585113525 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 4.1389851570129395 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 4.138903617858887 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 4.138908863067627 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 4.1389851570129395 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 4.138926029205322 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 4.138903617858887 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 4.138824462890625 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 4.138908863067627 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 4.138974666595459 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 4.138759613037109 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 4.1387763023376465 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 4.13889741897583 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 4.138849258422852 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 4.138892650604248 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 4.138927936553955 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 4.1389241218566895 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 4.138911247253418 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 4.138976097106934 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 4.1388936042785645 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 4.138855457305908 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 4.138875484466553 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 4.138911247253418 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 4.13888692855835 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 4.13879919052124 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 4.1388421058654785 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 4.138922214508057 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 4.138905048370361 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 4.138994216918945 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 4.138956069946289 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 4.138921737670898 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 4.138819694519043 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 4.1388959884643555 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 4.138895511627197 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 4.138913631439209 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 4.138944625854492 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 4.138916969299316 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 4.1388654708862305 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 4.138926029205322 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 4.138823509216309 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 4.138879776000977 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 4.138901233673096 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 4.138900279998779 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 4.138955593109131 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 4.1388421058654785 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 4.1388068199157715 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 4.1388630867004395 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 4.138861179351807 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 4.138819694519043 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 4.138782978057861 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 4.13881778717041 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 4.1389689445495605 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 4.1389265060424805 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 4.138824462890625 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 4.138818740844727 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 4.138981819152832 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 4.138874530792236 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 4.138982772827148 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 4.138922214508057 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 4.1388936042785645 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 4.138855934143066 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 4.13900899887085 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 4.138876914978027 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 4.1389079093933105 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 4.1388092041015625 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 4.138907432556152 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 4.138957977294922 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 4.138881683349609 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 4.139011859893799 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 4.138845920562744 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 4.138880729675293 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 4.1388983726501465 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 4.138853073120117 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 4.1387152671813965 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 4.13875150680542 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 4.1388959884643555 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 4.138763904571533 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 4.1389479637146 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 4.1389265060424805 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 4.138926029205322 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 4.138904571533203 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 4.138837814331055 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 4.138822555541992 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 4.138946056365967 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 4.138922214508057 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 4.13891077041626 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 4.138836860656738 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 4.1388373374938965 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 4.138727188110352 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 4.138753414154053 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 4.138698101043701 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 4.138767242431641 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 4.138833522796631 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 4.138969421386719 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 4.138799667358398 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 4.138796806335449 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 4.1388373374938965 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 4.138922214508057 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 4.138792991638184 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 4.138949394226074 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 4.138813495635986 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 4.138705253601074 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 4.1389360427856445 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 4.138821125030518 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 4.138808727264404 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 4.138856887817383 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 4.138883590698242 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 4.138794898986816 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 4.138896465301514 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 4.138940811157227 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 4.138909339904785 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 4.138792037963867 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 4.138876914978027 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 4.138998985290527 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 4.138871192932129 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 4.138927936553955 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 4.138893127441406 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 4.138927936553955 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 4.139015197753906 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 4.138971328735352 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 4.13878870010376 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 4.138935565948486 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 4.138863563537598 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 4.138810157775879 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 4.138862609863281 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 4.138881683349609 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 4.138901710510254 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 4.138863563537598 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 4.138786315917969 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 4.1390061378479 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 4.138729572296143 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 4.1389617919921875 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 4.138830184936523 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 4.13888692855835 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 4.138916015625 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 4.139028549194336 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 4.138864994049072 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 4.138857841491699 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 4.138949394226074 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 4.138785362243652 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 4.1389336585998535 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 4.138838291168213 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 4.138920307159424 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 4.138919353485107 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 4.138907432556152 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 4.138772964477539 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 4.139005184173584 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 4.138901233673096 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 4.139012336730957 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 4.138907432556152 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 4.1388444900512695 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 4.138979911804199 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 4.138680934906006 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 4.138768672943115 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 4.138779163360596 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 4.1387858390808105 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 4.138981342315674 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 4.138796806335449 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 4.138563632965088 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 4.138683795928955 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 4.138712406158447 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 4.138866901397705 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 4.138825416564941 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 4.13900089263916 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 4.138634204864502 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 4.1387529373168945 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 4.1387619972229 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 4.138977527618408 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 4.138797283172607 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 4.138852596282959 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 4.138792037963867 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 4.13872766494751 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 4.138864994049072 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 4.138792991638184 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 4.138843059539795 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 4.138915061950684 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 4.138880729675293 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 4.138950824737549 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 4.1388044357299805 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 4.138808250427246 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 4.138869762420654 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 4.138794898986816 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 4.1388840675354 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 4.138781547546387 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 4.138838768005371 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 4.138906002044678 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 4.13887357711792 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 4.13886022567749 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 4.138722896575928 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 4.1388068199157715 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 4.138841152191162 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 4.138859272003174 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 4.138902187347412 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 4.138843536376953 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 4.138766288757324 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 4.13885498046875 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 4.138857841491699 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 4.138986110687256 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 4.138819694519043 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 4.138765335083008 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 4.139106750488281 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 4.13895845413208 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 4.138686180114746 + }, + { + "layer": "layer.32.self_attn.q_proj", + "ppl": 4.13881778717041 + }, + { + "layer": "layer.32.self_attn.k_proj", + "ppl": 4.1388630867004395 + }, + { + "layer": "layer.32.self_attn.v_proj", + "ppl": 4.138890266418457 + }, + { + "layer": "layer.32.self_attn.o_proj", + "ppl": 4.13881254196167 + }, + { + "layer": "layer.32.mlp.gate_proj", + "ppl": 4.138875961303711 + }, + { + "layer": "layer.32.mlp.up_proj", + "ppl": 4.139027118682861 + }, + { + "layer": "layer.32.mlp.down_proj", + "ppl": 4.138741970062256 + }, + { + "layer": "layer.33.self_attn.q_proj", + "ppl": 4.138981342315674 + }, + { + "layer": "layer.33.self_attn.k_proj", + "ppl": 4.13881778717041 + }, + { + "layer": "layer.33.self_attn.v_proj", + "ppl": 4.138792991638184 + }, + { + "layer": "layer.33.self_attn.o_proj", + "ppl": 4.138767719268799 + }, + { + "layer": "layer.33.mlp.gate_proj", + "ppl": 4.138542652130127 + }, + { + "layer": "layer.33.mlp.up_proj", + "ppl": 4.138920307159424 + }, + { + "layer": "layer.33.mlp.down_proj", + "ppl": 4.139156341552734 + }, + { + "layer": "layer.34.self_attn.q_proj", + "ppl": 4.138907432556152 + }, + { + "layer": "layer.34.self_attn.k_proj", + "ppl": 4.138885498046875 + }, + { + "layer": "layer.34.self_attn.v_proj", + "ppl": 4.138791084289551 + }, + { + "layer": "layer.34.self_attn.o_proj", + "ppl": 4.138677597045898 + }, + { + "layer": "layer.34.mlp.gate_proj", + "ppl": 4.138803482055664 + }, + { + "layer": "layer.34.mlp.up_proj", + "ppl": 4.1387176513671875 + }, + { + "layer": "layer.34.mlp.down_proj", + "ppl": 4.138847827911377 + }, + { + "layer": "layer.35.self_attn.q_proj", + "ppl": 4.138885498046875 + }, + { + "layer": "layer.35.self_attn.k_proj", + "ppl": 4.1389479637146 + }, + { + "layer": "layer.35.self_attn.v_proj", + "ppl": 4.138872146606445 + }, + { + "layer": "layer.35.self_attn.o_proj", + "ppl": 4.138760089874268 + }, + { + "layer": "layer.35.mlp.gate_proj", + "ppl": 4.138957977294922 + }, + { + "layer": "layer.35.mlp.up_proj", + "ppl": 4.138382434844971 + }, + { + "layer": "layer.35.mlp.down_proj", + "ppl": 4.139060974121094 + }, + { + "layer": "layer.36.self_attn.q_proj", + "ppl": 4.138975143432617 + }, + { + "layer": "layer.36.self_attn.k_proj", + "ppl": 4.139014720916748 + }, + { + "layer": "layer.36.self_attn.v_proj", + "ppl": 4.138912200927734 + }, + { + "layer": "layer.36.self_attn.o_proj", + "ppl": 4.138824939727783 + }, + { + "layer": "layer.36.mlp.gate_proj", + "ppl": 4.138722896575928 + }, + { + "layer": "layer.36.mlp.up_proj", + "ppl": 4.138752460479736 + }, + { + "layer": "layer.36.mlp.down_proj", + "ppl": 4.138591766357422 + }, + { + "layer": "layer.37.self_attn.q_proj", + "ppl": 4.138908386230469 + }, + { + "layer": "layer.37.self_attn.k_proj", + "ppl": 4.138890266418457 + }, + { + "layer": "layer.37.self_attn.v_proj", + "ppl": 4.138900279998779 + }, + { + "layer": "layer.37.self_attn.o_proj", + "ppl": 4.139028549194336 + }, + { + "layer": "layer.37.mlp.gate_proj", + "ppl": 4.138774394989014 + }, + { + "layer": "layer.37.mlp.up_proj", + "ppl": 4.139087677001953 + }, + { + "layer": "layer.37.mlp.down_proj", + "ppl": 4.139037609100342 + }, + { + "layer": "layer.38.self_attn.q_proj", + "ppl": 4.138771057128906 + }, + { + "layer": "layer.38.self_attn.k_proj", + "ppl": 4.1389617919921875 + }, + { + "layer": "layer.38.self_attn.v_proj", + "ppl": 4.138743877410889 + }, + { + "layer": "layer.38.self_attn.o_proj", + "ppl": 4.138861179351807 + }, + { + "layer": "layer.38.mlp.gate_proj", + "ppl": 4.1388115882873535 + }, + { + "layer": "layer.38.mlp.up_proj", + "ppl": 4.138787746429443 + }, + { + "layer": "layer.38.mlp.down_proj", + "ppl": 4.138792991638184 + }, + { + "layer": "layer.39.self_attn.q_proj", + "ppl": 4.13885498046875 + }, + { + "layer": "layer.39.self_attn.k_proj", + "ppl": 4.138808727264404 + }, + { + "layer": "layer.39.self_attn.v_proj", + "ppl": 4.138827323913574 + }, + { + "layer": "layer.39.self_attn.o_proj", + "ppl": 4.138607501983643 + }, + { + "layer": "layer.39.mlp.gate_proj", + "ppl": 4.138855934143066 + }, + { + "layer": "layer.39.mlp.up_proj", + "ppl": 4.138814449310303 + }, + { + "layer": "layer.39.mlp.down_proj", + "ppl": 4.138659477233887 + }, + { + "layer": "layer.40.self_attn.q_proj", + "ppl": 4.138813495635986 + }, + { + "layer": "layer.40.self_attn.k_proj", + "ppl": 4.138795375823975 + }, + { + "layer": "layer.40.self_attn.v_proj", + "ppl": 4.138998985290527 + }, + { + "layer": "layer.40.self_attn.o_proj", + "ppl": 4.138769149780273 + }, + { + "layer": "layer.40.mlp.gate_proj", + "ppl": 4.139030456542969 + }, + { + "layer": "layer.40.mlp.up_proj", + "ppl": 4.138894557952881 + }, + { + "layer": "layer.40.mlp.down_proj", + "ppl": 4.13908052444458 + }, + { + "layer": "layer.41.self_attn.q_proj", + "ppl": 4.138796806335449 + }, + { + "layer": "layer.41.self_attn.k_proj", + "ppl": 4.138944625854492 + }, + { + "layer": "layer.41.self_attn.v_proj", + "ppl": 4.13884973526001 + }, + { + "layer": "layer.41.self_attn.o_proj", + "ppl": 4.138967514038086 + }, + { + "layer": "layer.41.mlp.gate_proj", + "ppl": 4.138931751251221 + }, + { + "layer": "layer.41.mlp.up_proj", + "ppl": 4.138595104217529 + }, + { + "layer": "layer.41.mlp.down_proj", + "ppl": 4.1387128829956055 + }, + { + "layer": "layer.42.self_attn.q_proj", + "ppl": 4.138864517211914 + }, + { + "layer": "layer.42.self_attn.k_proj", + "ppl": 4.138880729675293 + }, + { + "layer": "layer.42.self_attn.v_proj", + "ppl": 4.138791084289551 + }, + { + "layer": "layer.42.self_attn.o_proj", + "ppl": 4.138916969299316 + }, + { + "layer": "layer.42.mlp.gate_proj", + "ppl": 4.1386003494262695 + }, + { + "layer": "layer.42.mlp.up_proj", + "ppl": 4.139017105102539 + }, + { + "layer": "layer.42.mlp.down_proj", + "ppl": 4.138936519622803 + }, + { + "layer": "layer.43.self_attn.q_proj", + "ppl": 4.13893985748291 + }, + { + "layer": "layer.43.self_attn.k_proj", + "ppl": 4.138952732086182 + }, + { + "layer": "layer.43.self_attn.v_proj", + "ppl": 4.138819217681885 + }, + { + "layer": "layer.43.self_attn.o_proj", + "ppl": 4.1388840675354 + }, + { + "layer": "layer.43.mlp.gate_proj", + "ppl": 4.1386308670043945 + }, + { + "layer": "layer.43.mlp.up_proj", + "ppl": 4.138803005218506 + }, + { + "layer": "layer.43.mlp.down_proj", + "ppl": 4.138983726501465 + }, + { + "layer": "layer.44.self_attn.q_proj", + "ppl": 4.138829708099365 + }, + { + "layer": "layer.44.self_attn.k_proj", + "ppl": 4.138787746429443 + }, + { + "layer": "layer.44.self_attn.v_proj", + "ppl": 4.138941764831543 + }, + { + "layer": "layer.44.self_attn.o_proj", + "ppl": 4.138759613037109 + }, + { + "layer": "layer.44.mlp.gate_proj", + "ppl": 4.138851165771484 + }, + { + "layer": "layer.44.mlp.up_proj", + "ppl": 4.138647079467773 + }, + { + "layer": "layer.44.mlp.down_proj", + "ppl": 4.1383161544799805 + }, + { + "layer": "layer.45.self_attn.q_proj", + "ppl": 4.138900279998779 + }, + { + "layer": "layer.45.self_attn.k_proj", + "ppl": 4.138834476470947 + }, + { + "layer": "layer.45.self_attn.v_proj", + "ppl": 4.138977527618408 + }, + { + "layer": "layer.45.self_attn.o_proj", + "ppl": 4.139042854309082 + }, + { + "layer": "layer.45.mlp.gate_proj", + "ppl": 4.138845920562744 + }, + { + "layer": "layer.45.mlp.up_proj", + "ppl": 4.138941764831543 + }, + { + "layer": "layer.45.mlp.down_proj", + "ppl": 4.138735294342041 + }, + { + "layer": "layer.46.self_attn.q_proj", + "ppl": 4.1388468742370605 + }, + { + "layer": "layer.46.self_attn.k_proj", + "ppl": 4.138787746429443 + }, + { + "layer": "layer.46.self_attn.v_proj", + "ppl": 4.139041900634766 + }, + { + "layer": "layer.46.self_attn.o_proj", + "ppl": 4.138681411743164 + }, + { + "layer": "layer.46.mlp.gate_proj", + "ppl": 4.138669013977051 + }, + { + "layer": "layer.46.mlp.up_proj", + "ppl": 4.139228820800781 + }, + { + "layer": "layer.46.mlp.down_proj", + "ppl": 4.138737678527832 + }, + { + "layer": "layer.47.self_attn.q_proj", + "ppl": 4.138937473297119 + }, + { + "layer": "layer.47.self_attn.k_proj", + "ppl": 4.138830184936523 + }, + { + "layer": "layer.47.self_attn.v_proj", + "ppl": 4.138909339904785 + }, + { + "layer": "layer.47.self_attn.o_proj", + "ppl": 4.138911247253418 + }, + { + "layer": "layer.47.mlp.gate_proj", + "ppl": 4.138874530792236 + }, + { + "layer": "layer.47.mlp.up_proj", + "ppl": 4.139069080352783 + }, + { + "layer": "layer.47.mlp.down_proj", + "ppl": 4.138696193695068 + }, + { + "layer": "layer.48.self_attn.q_proj", + "ppl": 4.138768196105957 + }, + { + "layer": "layer.48.self_attn.k_proj", + "ppl": 4.138834476470947 + }, + { + "layer": "layer.48.self_attn.v_proj", + "ppl": 4.139084339141846 + }, + { + "layer": "layer.48.self_attn.o_proj", + "ppl": 4.139013767242432 + }, + { + "layer": "layer.48.mlp.gate_proj", + "ppl": 4.138635158538818 + }, + { + "layer": "layer.48.mlp.up_proj", + "ppl": 4.138843536376953 + }, + { + "layer": "layer.48.mlp.down_proj", + "ppl": 4.138643741607666 + }, + { + "layer": "layer.49.self_attn.q_proj", + "ppl": 4.138862133026123 + }, + { + "layer": "layer.49.self_attn.k_proj", + "ppl": 4.138725757598877 + }, + { + "layer": "layer.49.self_attn.v_proj", + "ppl": 4.138822555541992 + }, + { + "layer": "layer.49.self_attn.o_proj", + "ppl": 4.138810634613037 + }, + { + "layer": "layer.49.mlp.gate_proj", + "ppl": 4.1390862464904785 + }, + { + "layer": "layer.49.mlp.up_proj", + "ppl": 4.139013767242432 + }, + { + "layer": "layer.49.mlp.down_proj", + "ppl": 4.138805866241455 + }, + { + "layer": "layer.50.self_attn.q_proj", + "ppl": 4.138857841491699 + }, + { + "layer": "layer.50.self_attn.k_proj", + "ppl": 4.139012813568115 + }, + { + "layer": "layer.50.self_attn.v_proj", + "ppl": 4.138779163360596 + }, + { + "layer": "layer.50.self_attn.o_proj", + "ppl": 4.138760089874268 + }, + { + "layer": "layer.50.mlp.gate_proj", + "ppl": 4.138713836669922 + }, + { + "layer": "layer.50.mlp.up_proj", + "ppl": 4.138981342315674 + }, + { + "layer": "layer.50.mlp.down_proj", + "ppl": 4.138406753540039 + }, + { + "layer": "layer.51.self_attn.q_proj", + "ppl": 4.138872146606445 + }, + { + "layer": "layer.51.self_attn.k_proj", + "ppl": 4.13888692855835 + }, + { + "layer": "layer.51.self_attn.v_proj", + "ppl": 4.138803482055664 + }, + { + "layer": "layer.51.self_attn.o_proj", + "ppl": 4.138862133026123 + }, + { + "layer": "layer.51.mlp.gate_proj", + "ppl": 4.138916015625 + }, + { + "layer": "layer.51.mlp.up_proj", + "ppl": 4.138911247253418 + }, + { + "layer": "layer.51.mlp.down_proj", + "ppl": 4.1390838623046875 + }, + { + "layer": "layer.52.self_attn.q_proj", + "ppl": 4.138953685760498 + }, + { + "layer": "layer.52.self_attn.k_proj", + "ppl": 4.138869285583496 + }, + { + "layer": "layer.52.self_attn.v_proj", + "ppl": 4.138866424560547 + }, + { + "layer": "layer.52.self_attn.o_proj", + "ppl": 4.138815402984619 + }, + { + "layer": "layer.52.mlp.gate_proj", + "ppl": 4.138911247253418 + }, + { + "layer": "layer.52.mlp.up_proj", + "ppl": 4.138925552368164 + }, + { + "layer": "layer.52.mlp.down_proj", + "ppl": 4.138701915740967 + }, + { + "layer": "layer.53.self_attn.q_proj", + "ppl": 4.138977527618408 + }, + { + "layer": "layer.53.self_attn.k_proj", + "ppl": 4.138900279998779 + }, + { + "layer": "layer.53.self_attn.v_proj", + "ppl": 4.138668537139893 + }, + { + "layer": "layer.53.self_attn.o_proj", + "ppl": 4.138796806335449 + }, + { + "layer": "layer.53.mlp.gate_proj", + "ppl": 4.13864803314209 + }, + { + "layer": "layer.53.mlp.up_proj", + "ppl": 4.138359546661377 + }, + { + "layer": "layer.53.mlp.down_proj", + "ppl": 4.138801097869873 + }, + { + "layer": "layer.54.self_attn.q_proj", + "ppl": 4.138899803161621 + }, + { + "layer": "layer.54.self_attn.k_proj", + "ppl": 4.1387939453125 + }, + { + "layer": "layer.54.self_attn.v_proj", + "ppl": 4.13887357711792 + }, + { + "layer": "layer.54.self_attn.o_proj", + "ppl": 4.1388163566589355 + }, + { + "layer": "layer.54.mlp.gate_proj", + "ppl": 4.138740539550781 + }, + { + "layer": "layer.54.mlp.up_proj", + "ppl": 4.138891220092773 + }, + { + "layer": "layer.54.mlp.down_proj", + "ppl": 4.1388068199157715 + }, + { + "layer": "layer.55.self_attn.q_proj", + "ppl": 4.138853073120117 + }, + { + "layer": "layer.55.self_attn.k_proj", + "ppl": 4.1389665603637695 + }, + { + "layer": "layer.55.self_attn.v_proj", + "ppl": 4.138843059539795 + }, + { + "layer": "layer.55.self_attn.o_proj", + "ppl": 4.138991355895996 + }, + { + "layer": "layer.55.mlp.gate_proj", + "ppl": 4.139035224914551 + }, + { + "layer": "layer.55.mlp.up_proj", + "ppl": 4.138700008392334 + }, + { + "layer": "layer.55.mlp.down_proj", + "ppl": 4.1388115882873535 + }, + { + "layer": "layer.56.self_attn.q_proj", + "ppl": 4.138907432556152 + }, + { + "layer": "layer.56.self_attn.k_proj", + "ppl": 4.1389241218566895 + }, + { + "layer": "layer.56.self_attn.v_proj", + "ppl": 4.138977527618408 + }, + { + "layer": "layer.56.self_attn.o_proj", + "ppl": 4.138978481292725 + }, + { + "layer": "layer.56.mlp.gate_proj", + "ppl": 4.13867712020874 + }, + { + "layer": "layer.56.mlp.up_proj", + "ppl": 4.138546943664551 + }, + { + "layer": "layer.56.mlp.down_proj", + "ppl": 4.139009475708008 + }, + { + "layer": "layer.57.self_attn.q_proj", + "ppl": 4.138882637023926 + }, + { + "layer": "layer.57.self_attn.k_proj", + "ppl": 4.138935565948486 + }, + { + "layer": "layer.57.self_attn.v_proj", + "ppl": 4.138876914978027 + }, + { + "layer": "layer.57.self_attn.o_proj", + "ppl": 4.138970851898193 + }, + { + "layer": "layer.57.mlp.gate_proj", + "ppl": 4.138642311096191 + }, + { + "layer": "layer.57.mlp.up_proj", + "ppl": 4.138744354248047 + }, + { + "layer": "layer.57.mlp.down_proj", + "ppl": 4.138648509979248 + }, + { + "layer": "layer.58.self_attn.q_proj", + "ppl": 4.138878345489502 + }, + { + "layer": "layer.58.self_attn.k_proj", + "ppl": 4.138909816741943 + }, + { + "layer": "layer.58.self_attn.v_proj", + "ppl": 4.138916969299316 + }, + { + "layer": "layer.58.self_attn.o_proj", + "ppl": 4.1390700340271 + }, + { + "layer": "layer.58.mlp.gate_proj", + "ppl": 4.138798713684082 + }, + { + "layer": "layer.58.mlp.up_proj", + "ppl": 4.139223098754883 + }, + { + "layer": "layer.58.mlp.down_proj", + "ppl": 4.138378620147705 + }, + { + "layer": "layer.59.self_attn.q_proj", + "ppl": 4.138896465301514 + }, + { + "layer": "layer.59.self_attn.k_proj", + "ppl": 4.138882160186768 + }, + { + "layer": "layer.59.self_attn.v_proj", + "ppl": 4.138906002044678 + }, + { + "layer": "layer.59.self_attn.o_proj", + "ppl": 4.138851165771484 + }, + { + "layer": "layer.59.mlp.gate_proj", + "ppl": 4.138702869415283 + }, + { + "layer": "layer.59.mlp.up_proj", + "ppl": 4.139027118682861 + }, + { + "layer": "layer.59.mlp.down_proj", + "ppl": 4.138606071472168 + }, + { + "layer": "layer.60.self_attn.q_proj", + "ppl": 4.138786792755127 + }, + { + "layer": "layer.60.self_attn.k_proj", + "ppl": 4.138777732849121 + }, + { + "layer": "layer.60.self_attn.v_proj", + "ppl": 4.1388163566589355 + }, + { + "layer": "layer.60.self_attn.o_proj", + "ppl": 4.138874530792236 + }, + { + "layer": "layer.60.mlp.gate_proj", + "ppl": 4.138412952423096 + }, + { + "layer": "layer.60.mlp.up_proj", + "ppl": 4.138530254364014 + }, + { + "layer": "layer.60.mlp.down_proj", + "ppl": 4.138898849487305 + }, + { + "layer": "layer.61.self_attn.q_proj", + "ppl": 4.138815402984619 + }, + { + "layer": "layer.61.self_attn.k_proj", + "ppl": 4.138892650604248 + }, + { + "layer": "layer.61.self_attn.v_proj", + "ppl": 4.138908386230469 + }, + { + "layer": "layer.61.self_attn.o_proj", + "ppl": 4.138768196105957 + }, + { + "layer": "layer.61.mlp.gate_proj", + "ppl": 4.138535499572754 + }, + { + "layer": "layer.61.mlp.up_proj", + "ppl": 4.138702869415283 + }, + { + "layer": "layer.61.mlp.down_proj", + "ppl": 4.138827323913574 + }, + { + "layer": "layer.62.self_attn.q_proj", + "ppl": 4.13877534866333 + }, + { + "layer": "layer.62.self_attn.k_proj", + "ppl": 4.138815402984619 + }, + { + "layer": "layer.62.self_attn.v_proj", + "ppl": 4.138784408569336 + }, + { + "layer": "layer.62.self_attn.o_proj", + "ppl": 4.1388678550720215 + }, + { + "layer": "layer.62.mlp.gate_proj", + "ppl": 4.1391215324401855 + }, + { + "layer": "layer.62.mlp.up_proj", + "ppl": 4.138636112213135 + }, + { + "layer": "layer.62.mlp.down_proj", + "ppl": 4.138587474822998 + }, + { + "layer": "layer.63.self_attn.q_proj", + "ppl": 4.138784885406494 + }, + { + "layer": "layer.63.self_attn.k_proj", + "ppl": 4.138843059539795 + }, + { + "layer": "layer.63.self_attn.v_proj", + "ppl": 4.138821125030518 + }, + { + "layer": "layer.63.self_attn.o_proj", + "ppl": 4.138868808746338 + }, + { + "layer": "layer.63.mlp.gate_proj", + "ppl": 4.138531684875488 + }, + { + "layer": "layer.63.mlp.up_proj", + "ppl": 4.138786792755127 + }, + { + "layer": "layer.63.mlp.down_proj", + "ppl": 4.13858699798584 + }, + { + "layer": "layer.64.self_attn.q_proj", + "ppl": 4.138861179351807 + }, + { + "layer": "layer.64.self_attn.k_proj", + "ppl": 4.138936996459961 + }, + { + "layer": "layer.64.self_attn.v_proj", + "ppl": 4.138660430908203 + }, + { + "layer": "layer.64.self_attn.o_proj", + "ppl": 4.138678073883057 + }, + { + "layer": "layer.64.mlp.gate_proj", + "ppl": 4.138594627380371 + }, + { + "layer": "layer.64.mlp.up_proj", + "ppl": 4.138642311096191 + }, + { + "layer": "layer.64.mlp.down_proj", + "ppl": 4.1388726234436035 + }, + { + "layer": "layer.65.self_attn.q_proj", + "ppl": 4.138826370239258 + }, + { + "layer": "layer.65.self_attn.k_proj", + "ppl": 4.138745307922363 + }, + { + "layer": "layer.65.self_attn.v_proj", + "ppl": 4.138815879821777 + }, + { + "layer": "layer.65.self_attn.o_proj", + "ppl": 4.138877868652344 + }, + { + "layer": "layer.65.mlp.gate_proj", + "ppl": 4.138718605041504 + }, + { + "layer": "layer.65.mlp.up_proj", + "ppl": 4.1388373374938965 + }, + { + "layer": "layer.65.mlp.down_proj", + "ppl": 4.138707160949707 + }, + { + "layer": "layer.66.self_attn.q_proj", + "ppl": 4.138716697692871 + }, + { + "layer": "layer.66.self_attn.k_proj", + "ppl": 4.1389546394348145 + }, + { + "layer": "layer.66.self_attn.v_proj", + "ppl": 4.138955593109131 + }, + { + "layer": "layer.66.self_attn.o_proj", + "ppl": 4.138845443725586 + }, + { + "layer": "layer.66.mlp.gate_proj", + "ppl": 4.138693332672119 + }, + { + "layer": "layer.66.mlp.up_proj", + "ppl": 4.138601303100586 + }, + { + "layer": "layer.66.mlp.down_proj", + "ppl": 4.138653755187988 + }, + { + "layer": "layer.67.self_attn.q_proj", + "ppl": 4.138894557952881 + }, + { + "layer": "layer.67.self_attn.k_proj", + "ppl": 4.138911247253418 + }, + { + "layer": "layer.67.self_attn.v_proj", + "ppl": 4.13886022567749 + }, + { + "layer": "layer.67.self_attn.o_proj", + "ppl": 4.13878059387207 + }, + { + "layer": "layer.67.mlp.gate_proj", + "ppl": 4.138492584228516 + }, + { + "layer": "layer.67.mlp.up_proj", + "ppl": 4.138686656951904 + }, + { + "layer": "layer.67.mlp.down_proj", + "ppl": 4.1385087966918945 + }, + { + "layer": "layer.68.self_attn.q_proj", + "ppl": 4.138832092285156 + }, + { + "layer": "layer.68.self_attn.k_proj", + "ppl": 4.138821125030518 + }, + { + "layer": "layer.68.self_attn.v_proj", + "ppl": 4.1387619972229 + }, + { + "layer": "layer.68.self_attn.o_proj", + "ppl": 4.1388936042785645 + }, + { + "layer": "layer.68.mlp.gate_proj", + "ppl": 4.1387553215026855 + }, + { + "layer": "layer.68.mlp.up_proj", + "ppl": 4.13870906829834 + }, + { + "layer": "layer.68.mlp.down_proj", + "ppl": 4.138865947723389 + }, + { + "layer": "layer.69.self_attn.q_proj", + "ppl": 4.138794898986816 + }, + { + "layer": "layer.69.self_attn.k_proj", + "ppl": 4.138942718505859 + }, + { + "layer": "layer.69.self_attn.v_proj", + "ppl": 4.138739109039307 + }, + { + "layer": "layer.69.self_attn.o_proj", + "ppl": 4.138895511627197 + }, + { + "layer": "layer.69.mlp.gate_proj", + "ppl": 4.138301372528076 + }, + { + "layer": "layer.69.mlp.up_proj", + "ppl": 4.138761043548584 + }, + { + "layer": "layer.69.mlp.down_proj", + "ppl": 4.138911724090576 + }, + { + "layer": "layer.70.self_attn.q_proj", + "ppl": 4.1388421058654785 + }, + { + "layer": "layer.70.self_attn.k_proj", + "ppl": 4.138930320739746 + }, + { + "layer": "layer.70.self_attn.v_proj", + "ppl": 4.1388068199157715 + }, + { + "layer": "layer.70.self_attn.o_proj", + "ppl": 4.13889741897583 + }, + { + "layer": "layer.70.mlp.gate_proj", + "ppl": 4.138184070587158 + }, + { + "layer": "layer.70.mlp.up_proj", + "ppl": 4.138825416564941 + }, + { + "layer": "layer.70.mlp.down_proj", + "ppl": 4.138597011566162 + }, + { + "layer": "layer.71.self_attn.q_proj", + "ppl": 4.138991355895996 + }, + { + "layer": "layer.71.self_attn.k_proj", + "ppl": 4.138916969299316 + }, + { + "layer": "layer.71.self_attn.v_proj", + "ppl": 4.138845920562744 + }, + { + "layer": "layer.71.self_attn.o_proj", + "ppl": 4.13881254196167 + }, + { + "layer": "layer.71.mlp.gate_proj", + "ppl": 4.138483047485352 + }, + { + "layer": "layer.71.mlp.up_proj", + "ppl": 4.138665199279785 + }, + { + "layer": "layer.71.mlp.down_proj", + "ppl": 4.13837194442749 + }, + { + "layer": "layer.72.self_attn.q_proj", + "ppl": 4.138808250427246 + }, + { + "layer": "layer.72.self_attn.k_proj", + "ppl": 4.138891696929932 + }, + { + "layer": "layer.72.self_attn.v_proj", + "ppl": 4.13877010345459 + }, + { + "layer": "layer.72.self_attn.o_proj", + "ppl": 4.138799667358398 + }, + { + "layer": "layer.72.mlp.gate_proj", + "ppl": 4.138271331787109 + }, + { + "layer": "layer.72.mlp.up_proj", + "ppl": 4.13838529586792 + }, + { + "layer": "layer.72.mlp.down_proj", + "ppl": 4.1384968757629395 + }, + { + "layer": "layer.73.self_attn.q_proj", + "ppl": 4.138916969299316 + }, + { + "layer": "layer.73.self_attn.k_proj", + "ppl": 4.138901233673096 + }, + { + "layer": "layer.73.self_attn.v_proj", + "ppl": 4.138899803161621 + }, + { + "layer": "layer.73.self_attn.o_proj", + "ppl": 4.138847827911377 + }, + { + "layer": "layer.73.mlp.gate_proj", + "ppl": 4.138495922088623 + }, + { + "layer": "layer.73.mlp.up_proj", + "ppl": 4.138620376586914 + }, + { + "layer": "layer.73.mlp.down_proj", + "ppl": 4.138813495635986 + }, + { + "layer": "layer.74.self_attn.q_proj", + "ppl": 4.1388444900512695 + }, + { + "layer": "layer.74.self_attn.k_proj", + "ppl": 4.138907432556152 + }, + { + "layer": "layer.74.self_attn.v_proj", + "ppl": 4.13884973526001 + }, + { + "layer": "layer.74.self_attn.o_proj", + "ppl": 4.138861179351807 + }, + { + "layer": "layer.74.mlp.gate_proj", + "ppl": 4.138286113739014 + }, + { + "layer": "layer.74.mlp.up_proj", + "ppl": 4.138773441314697 + }, + { + "layer": "layer.74.mlp.down_proj", + "ppl": 4.138653755187988 + }, + { + "layer": "layer.75.self_attn.q_proj", + "ppl": 4.13889741897583 + }, + { + "layer": "layer.75.self_attn.k_proj", + "ppl": 4.138868808746338 + }, + { + "layer": "layer.75.self_attn.v_proj", + "ppl": 4.138878345489502 + }, + { + "layer": "layer.75.self_attn.o_proj", + "ppl": 4.138765335083008 + }, + { + "layer": "layer.75.mlp.gate_proj", + "ppl": 4.138195991516113 + }, + { + "layer": "layer.75.mlp.up_proj", + "ppl": 4.138639450073242 + }, + { + "layer": "layer.75.mlp.down_proj", + "ppl": 4.138511657714844 + }, + { + "layer": "layer.76.self_attn.q_proj", + "ppl": 4.138862133026123 + }, + { + "layer": "layer.76.self_attn.k_proj", + "ppl": 4.138830184936523 + }, + { + "layer": "layer.76.self_attn.v_proj", + "ppl": 4.138866424560547 + }, + { + "layer": "layer.76.self_attn.o_proj", + "ppl": 4.138937473297119 + }, + { + "layer": "layer.76.mlp.gate_proj", + "ppl": 4.138706684112549 + }, + { + "layer": "layer.76.mlp.up_proj", + "ppl": 4.1386518478393555 + }, + { + "layer": "layer.76.mlp.down_proj", + "ppl": 4.138493537902832 + }, + { + "layer": "layer.77.self_attn.q_proj", + "ppl": 4.139087677001953 + }, + { + "layer": "layer.77.self_attn.k_proj", + "ppl": 4.139103889465332 + }, + { + "layer": "layer.77.self_attn.v_proj", + "ppl": 4.138926029205322 + }, + { + "layer": "layer.77.self_attn.o_proj", + "ppl": 4.138856887817383 + }, + { + "layer": "layer.77.mlp.gate_proj", + "ppl": 4.13828182220459 + }, + { + "layer": "layer.77.mlp.up_proj", + "ppl": 4.138495922088623 + }, + { + "layer": "layer.77.mlp.down_proj", + "ppl": 4.1384663581848145 + }, + { + "layer": "layer.78.self_attn.q_proj", + "ppl": 4.138900279998779 + }, + { + "layer": "layer.78.self_attn.k_proj", + "ppl": 4.138948917388916 + }, + { + "layer": "layer.78.self_attn.v_proj", + "ppl": 4.138953685760498 + }, + { + "layer": "layer.78.self_attn.o_proj", + "ppl": 4.138957977294922 + }, + { + "layer": "layer.78.mlp.gate_proj", + "ppl": 4.13812255859375 + }, + { + "layer": "layer.78.mlp.up_proj", + "ppl": 4.138654708862305 + }, + { + "layer": "layer.78.mlp.down_proj", + "ppl": 4.138125896453857 + }, + { + "layer": "layer.79.self_attn.q_proj", + "ppl": 4.138777732849121 + }, + { + "layer": "layer.79.self_attn.k_proj", + "ppl": 4.1388936042785645 + }, + { + "layer": "layer.79.self_attn.v_proj", + "ppl": 4.138744354248047 + }, + { + "layer": "layer.79.self_attn.o_proj", + "ppl": 4.138971328735352 + }, + { + "layer": "layer.79.mlp.gate_proj", + "ppl": 4.137265205383301 + }, + { + "layer": "layer.79.mlp.up_proj", + "ppl": 4.137465953826904 + }, + { + "layer": "layer.79.mlp.down_proj", + "ppl": 4.137305736541748 + } + ] + }, + { + "bit": 9, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 4.1068596839904785 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 4.1069254875183105 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 4.106873512268066 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 4.106855392456055 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 4.106867790222168 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 4.106881141662598 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 4.106968402862549 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 4.106753826141357 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 4.106857776641846 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 4.106924057006836 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 4.106952667236328 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 4.106990337371826 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 4.1069016456604 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 4.106743335723877 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 4.106814384460449 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 4.106893539428711 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 4.106926441192627 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 4.106846332550049 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 4.106924057006836 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 4.106790065765381 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 4.106875419616699 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 4.106861591339111 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 4.1068878173828125 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 4.106925010681152 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 4.106936931610107 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 4.106849670410156 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 4.1068925857543945 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 4.106867790222168 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 4.106843948364258 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 4.106900691986084 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 4.106961250305176 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 4.1070098876953125 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 4.106945037841797 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 4.106904029846191 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 4.106966495513916 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 4.106926441192627 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 4.106863975524902 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 4.106872081756592 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 4.106906414031982 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 4.106822490692139 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 4.106778621673584 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 4.1068620681762695 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 4.106778144836426 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 4.106841087341309 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 4.106799602508545 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 4.106759548187256 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 4.106940269470215 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 4.106875419616699 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 4.106993675231934 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 4.106822490692139 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 4.106810092926025 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 4.106904029846191 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 4.106860637664795 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 4.106973648071289 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 4.106791019439697 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 4.106839179992676 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 4.10683012008667 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 4.106894016265869 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 4.106837272644043 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 4.107000350952148 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 4.10683012008667 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 4.106822967529297 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 4.106923580169678 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 4.106814861297607 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 4.106802463531494 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 4.106889247894287 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 4.106832981109619 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 4.106837749481201 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 4.106934070587158 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 4.1069769859313965 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 4.106935024261475 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 4.106956958770752 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 4.1069016456604 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 4.10697603225708 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 4.106912612915039 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 4.106993675231934 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 4.10678768157959 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 4.106931209564209 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 4.1068644523620605 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 4.106861114501953 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 4.106894493103027 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 4.106866359710693 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 4.107005596160889 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 4.106982707977295 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 4.106936454772949 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 4.106872081756592 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 4.106872081756592 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 4.106957912445068 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 4.107045650482178 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 4.106761932373047 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 4.106837272644043 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 4.106876373291016 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 4.106995582580566 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 4.106917381286621 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 4.106834888458252 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 4.106833457946777 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 4.106788158416748 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 4.106876850128174 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 4.106845378875732 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 4.106949806213379 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 4.106877326965332 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 4.106963634490967 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 4.107036590576172 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 4.106861591339111 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 4.106786251068115 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 4.106969833374023 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 4.106907367706299 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 4.106817722320557 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 4.106894016265869 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 4.106896877288818 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 4.106935024261475 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 4.106786251068115 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 4.106934070587158 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 4.106928825378418 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 4.10697603225708 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 4.106896877288818 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 4.106858253479004 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 4.106831073760986 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 4.106794357299805 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 4.1070075035095215 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 4.106969356536865 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 4.106873035430908 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 4.10684871673584 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 4.106929302215576 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 4.1068830490112305 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 4.106831073760986 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 4.106874465942383 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 4.106873989105225 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 4.106873512268066 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 4.106911659240723 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 4.106899261474609 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 4.106837272644043 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 4.1068830490112305 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 4.106952667236328 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 4.107006549835205 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 4.106760501861572 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 4.1069488525390625 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 4.106912612915039 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 4.106806755065918 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 4.10670804977417 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 4.107028007507324 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 4.1069016456604 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 4.106878280639648 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 4.106810092926025 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 4.1068010330200195 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 4.1067891120910645 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 4.106929779052734 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 4.106780052185059 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 4.106850624084473 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 4.1070122718811035 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 4.106843948364258 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 4.106812000274658 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 4.106741428375244 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 4.106802463531494 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 4.10696268081665 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 4.106877326965332 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 4.106950283050537 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 4.106868743896484 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 4.10683012008667 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 4.107065200805664 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 4.1069440841674805 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 4.107001304626465 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 4.1069111824035645 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 4.106900691986084 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 4.106873512268066 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 4.106827735900879 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 4.107020378112793 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 4.106767654418945 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 4.106900691986084 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 4.106922626495361 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 4.106812000274658 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 4.106963157653809 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 4.106715679168701 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 4.106892108917236 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 4.106752395629883 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 4.106886386871338 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 4.1068196296691895 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 4.106884002685547 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 4.10693359375 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 4.106773376464844 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 4.106973648071289 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 4.1069536209106445 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 4.106927394866943 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 4.106889247894287 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 4.106735706329346 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 4.1069111824035645 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 4.106799602508545 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 4.106923580169678 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 4.1067304611206055 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 4.107003688812256 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 4.106903076171875 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 4.106845378875732 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 4.106841087341309 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 4.106860160827637 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 4.106980323791504 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 4.106853008270264 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 4.107019424438477 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 4.106871604919434 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 4.106896877288818 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 4.1069016456604 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 4.106735706329346 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 4.106814861297607 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 4.106782913208008 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 4.106992721557617 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 4.106903076171875 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 4.106889724731445 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 4.1069536209106445 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 4.106785297393799 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 4.106833457946777 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 4.106962203979492 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 4.106886386871338 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 4.1069416999816895 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 4.106879234313965 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 4.106853008270264 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 4.106792449951172 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 4.106879234313965 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 4.106945037841797 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 4.106863975524902 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 4.106882095336914 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 4.106783390045166 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 4.106927394866943 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 4.106778621673584 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 4.106846809387207 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 4.107359886169434 + }, + { + "layer": "layer.32.self_attn.q_proj", + "ppl": 4.106815814971924 + }, + { + "layer": "layer.32.self_attn.k_proj", + "ppl": 4.10697603225708 + }, + { + "layer": "layer.32.self_attn.v_proj", + "ppl": 4.106896877288818 + }, + { + "layer": "layer.32.self_attn.o_proj", + "ppl": 4.107036590576172 + }, + { + "layer": "layer.32.mlp.gate_proj", + "ppl": 4.106929779052734 + }, + { + "layer": "layer.32.mlp.up_proj", + "ppl": 4.106717109680176 + }, + { + "layer": "layer.32.mlp.down_proj", + "ppl": 4.106827735900879 + }, + { + "layer": "layer.33.self_attn.q_proj", + "ppl": 4.106939315795898 + }, + { + "layer": "layer.33.self_attn.k_proj", + "ppl": 4.106839179992676 + }, + { + "layer": "layer.33.self_attn.v_proj", + "ppl": 4.106992244720459 + }, + { + "layer": "layer.33.self_attn.o_proj", + "ppl": 4.106948375701904 + }, + { + "layer": "layer.33.mlp.gate_proj", + "ppl": 4.1068034172058105 + }, + { + "layer": "layer.33.mlp.up_proj", + "ppl": 4.106810092926025 + }, + { + "layer": "layer.33.mlp.down_proj", + "ppl": 4.1064133644104 + }, + { + "layer": "layer.34.self_attn.q_proj", + "ppl": 4.106959342956543 + }, + { + "layer": "layer.34.self_attn.k_proj", + "ppl": 4.106875419616699 + }, + { + "layer": "layer.34.self_attn.v_proj", + "ppl": 4.106903076171875 + }, + { + "layer": "layer.34.self_attn.o_proj", + "ppl": 4.107118606567383 + }, + { + "layer": "layer.34.mlp.gate_proj", + "ppl": 4.10693359375 + }, + { + "layer": "layer.34.mlp.up_proj", + "ppl": 4.106791019439697 + }, + { + "layer": "layer.34.mlp.down_proj", + "ppl": 4.1069440841674805 + }, + { + "layer": "layer.35.self_attn.q_proj", + "ppl": 4.107043743133545 + }, + { + "layer": "layer.35.self_attn.k_proj", + "ppl": 4.106951713562012 + }, + { + "layer": "layer.35.self_attn.v_proj", + "ppl": 4.106889247894287 + }, + { + "layer": "layer.35.self_attn.o_proj", + "ppl": 4.106784343719482 + }, + { + "layer": "layer.35.mlp.gate_proj", + "ppl": 4.106694221496582 + }, + { + "layer": "layer.35.mlp.up_proj", + "ppl": 4.107034683227539 + }, + { + "layer": "layer.35.mlp.down_proj", + "ppl": 4.106812477111816 + }, + { + "layer": "layer.36.self_attn.q_proj", + "ppl": 4.106947898864746 + }, + { + "layer": "layer.36.self_attn.k_proj", + "ppl": 4.106867790222168 + }, + { + "layer": "layer.36.self_attn.v_proj", + "ppl": 4.106966972351074 + }, + { + "layer": "layer.36.self_attn.o_proj", + "ppl": 4.1067657470703125 + }, + { + "layer": "layer.36.mlp.gate_proj", + "ppl": 4.106870651245117 + }, + { + "layer": "layer.36.mlp.up_proj", + "ppl": 4.106653213500977 + }, + { + "layer": "layer.36.mlp.down_proj", + "ppl": 4.106706142425537 + }, + { + "layer": "layer.37.self_attn.q_proj", + "ppl": 4.106861591339111 + }, + { + "layer": "layer.37.self_attn.k_proj", + "ppl": 4.106943607330322 + }, + { + "layer": "layer.37.self_attn.v_proj", + "ppl": 4.106977939605713 + }, + { + "layer": "layer.37.self_attn.o_proj", + "ppl": 4.106870651245117 + }, + { + "layer": "layer.37.mlp.gate_proj", + "ppl": 4.106778621673584 + }, + { + "layer": "layer.37.mlp.up_proj", + "ppl": 4.106624126434326 + }, + { + "layer": "layer.37.mlp.down_proj", + "ppl": 4.106833457946777 + }, + { + "layer": "layer.38.self_attn.q_proj", + "ppl": 4.1070146560668945 + }, + { + "layer": "layer.38.self_attn.k_proj", + "ppl": 4.106889724731445 + }, + { + "layer": "layer.38.self_attn.v_proj", + "ppl": 4.106963157653809 + }, + { + "layer": "layer.38.self_attn.o_proj", + "ppl": 4.106785774230957 + }, + { + "layer": "layer.38.mlp.gate_proj", + "ppl": 4.106950759887695 + }, + { + "layer": "layer.38.mlp.up_proj", + "ppl": 4.10692834854126 + }, + { + "layer": "layer.38.mlp.down_proj", + "ppl": 4.106707572937012 + }, + { + "layer": "layer.39.self_attn.q_proj", + "ppl": 4.106919288635254 + }, + { + "layer": "layer.39.self_attn.k_proj", + "ppl": 4.1069254875183105 + }, + { + "layer": "layer.39.self_attn.v_proj", + "ppl": 4.106916427612305 + }, + { + "layer": "layer.39.self_attn.o_proj", + "ppl": 4.106945037841797 + }, + { + "layer": "layer.39.mlp.gate_proj", + "ppl": 4.106661796569824 + }, + { + "layer": "layer.39.mlp.up_proj", + "ppl": 4.106870651245117 + }, + { + "layer": "layer.39.mlp.down_proj", + "ppl": 4.106870651245117 + }, + { + "layer": "layer.40.self_attn.q_proj", + "ppl": 4.106863975524902 + }, + { + "layer": "layer.40.self_attn.k_proj", + "ppl": 4.107022762298584 + }, + { + "layer": "layer.40.self_attn.v_proj", + "ppl": 4.10685920715332 + }, + { + "layer": "layer.40.self_attn.o_proj", + "ppl": 4.106858730316162 + }, + { + "layer": "layer.40.mlp.gate_proj", + "ppl": 4.106629848480225 + }, + { + "layer": "layer.40.mlp.up_proj", + "ppl": 4.107084274291992 + }, + { + "layer": "layer.40.mlp.down_proj", + "ppl": 4.106724739074707 + }, + { + "layer": "layer.41.self_attn.q_proj", + "ppl": 4.106852054595947 + }, + { + "layer": "layer.41.self_attn.k_proj", + "ppl": 4.106860637664795 + }, + { + "layer": "layer.41.self_attn.v_proj", + "ppl": 4.106899738311768 + }, + { + "layer": "layer.41.self_attn.o_proj", + "ppl": 4.106930255889893 + }, + { + "layer": "layer.41.mlp.gate_proj", + "ppl": 4.106899738311768 + }, + { + "layer": "layer.41.mlp.up_proj", + "ppl": 4.106950759887695 + }, + { + "layer": "layer.41.mlp.down_proj", + "ppl": 4.1069183349609375 + }, + { + "layer": "layer.42.self_attn.q_proj", + "ppl": 4.106949329376221 + }, + { + "layer": "layer.42.self_attn.k_proj", + "ppl": 4.106888771057129 + }, + { + "layer": "layer.42.self_attn.v_proj", + "ppl": 4.1070332527160645 + }, + { + "layer": "layer.42.self_attn.o_proj", + "ppl": 4.106759071350098 + }, + { + "layer": "layer.42.mlp.gate_proj", + "ppl": 4.1069560050964355 + }, + { + "layer": "layer.42.mlp.up_proj", + "ppl": 4.106851100921631 + }, + { + "layer": "layer.42.mlp.down_proj", + "ppl": 4.106815814971924 + }, + { + "layer": "layer.43.self_attn.q_proj", + "ppl": 4.106947898864746 + }, + { + "layer": "layer.43.self_attn.k_proj", + "ppl": 4.106875419616699 + }, + { + "layer": "layer.43.self_attn.v_proj", + "ppl": 4.1068525314331055 + }, + { + "layer": "layer.43.self_attn.o_proj", + "ppl": 4.10694694519043 + }, + { + "layer": "layer.43.mlp.gate_proj", + "ppl": 4.1069722175598145 + }, + { + "layer": "layer.43.mlp.up_proj", + "ppl": 4.106907367706299 + }, + { + "layer": "layer.43.mlp.down_proj", + "ppl": 4.106929779052734 + }, + { + "layer": "layer.44.self_attn.q_proj", + "ppl": 4.106818199157715 + }, + { + "layer": "layer.44.self_attn.k_proj", + "ppl": 4.106936454772949 + }, + { + "layer": "layer.44.self_attn.v_proj", + "ppl": 4.106822490692139 + }, + { + "layer": "layer.44.self_attn.o_proj", + "ppl": 4.106884002685547 + }, + { + "layer": "layer.44.mlp.gate_proj", + "ppl": 4.106811046600342 + }, + { + "layer": "layer.44.mlp.up_proj", + "ppl": 4.106508255004883 + }, + { + "layer": "layer.44.mlp.down_proj", + "ppl": 4.107132911682129 + }, + { + "layer": "layer.45.self_attn.q_proj", + "ppl": 4.106919288635254 + }, + { + "layer": "layer.45.self_attn.k_proj", + "ppl": 4.107028007507324 + }, + { + "layer": "layer.45.self_attn.v_proj", + "ppl": 4.10683012008667 + }, + { + "layer": "layer.45.self_attn.o_proj", + "ppl": 4.106687545776367 + }, + { + "layer": "layer.45.mlp.gate_proj", + "ppl": 4.1066765785217285 + }, + { + "layer": "layer.45.mlp.up_proj", + "ppl": 4.106649398803711 + }, + { + "layer": "layer.45.mlp.down_proj", + "ppl": 4.107049942016602 + }, + { + "layer": "layer.46.self_attn.q_proj", + "ppl": 4.106754779815674 + }, + { + "layer": "layer.46.self_attn.k_proj", + "ppl": 4.106929779052734 + }, + { + "layer": "layer.46.self_attn.v_proj", + "ppl": 4.106647491455078 + }, + { + "layer": "layer.46.self_attn.o_proj", + "ppl": 4.107000350952148 + }, + { + "layer": "layer.46.mlp.gate_proj", + "ppl": 4.10683012008667 + }, + { + "layer": "layer.46.mlp.up_proj", + "ppl": 4.106624603271484 + }, + { + "layer": "layer.46.mlp.down_proj", + "ppl": 4.106935024261475 + }, + { + "layer": "layer.47.self_attn.q_proj", + "ppl": 4.106904983520508 + }, + { + "layer": "layer.47.self_attn.k_proj", + "ppl": 4.106873512268066 + }, + { + "layer": "layer.47.self_attn.v_proj", + "ppl": 4.107010364532471 + }, + { + "layer": "layer.47.self_attn.o_proj", + "ppl": 4.106889724731445 + }, + { + "layer": "layer.47.mlp.gate_proj", + "ppl": 4.106978416442871 + }, + { + "layer": "layer.47.mlp.up_proj", + "ppl": 4.106584072113037 + }, + { + "layer": "layer.47.mlp.down_proj", + "ppl": 4.106663227081299 + }, + { + "layer": "layer.48.self_attn.q_proj", + "ppl": 4.106964588165283 + }, + { + "layer": "layer.48.self_attn.k_proj", + "ppl": 4.106842041015625 + }, + { + "layer": "layer.48.self_attn.v_proj", + "ppl": 4.106740474700928 + }, + { + "layer": "layer.48.self_attn.o_proj", + "ppl": 4.106762886047363 + }, + { + "layer": "layer.48.mlp.gate_proj", + "ppl": 4.106871604919434 + }, + { + "layer": "layer.48.mlp.up_proj", + "ppl": 4.106558322906494 + }, + { + "layer": "layer.48.mlp.down_proj", + "ppl": 4.1070027351379395 + }, + { + "layer": "layer.49.self_attn.q_proj", + "ppl": 4.106805801391602 + }, + { + "layer": "layer.49.self_attn.k_proj", + "ppl": 4.106891632080078 + }, + { + "layer": "layer.49.self_attn.v_proj", + "ppl": 4.106960296630859 + }, + { + "layer": "layer.49.self_attn.o_proj", + "ppl": 4.106983184814453 + }, + { + "layer": "layer.49.mlp.gate_proj", + "ppl": 4.106653213500977 + }, + { + "layer": "layer.49.mlp.up_proj", + "ppl": 4.106889724731445 + }, + { + "layer": "layer.49.mlp.down_proj", + "ppl": 4.106846332550049 + }, + { + "layer": "layer.50.self_attn.q_proj", + "ppl": 4.106895446777344 + }, + { + "layer": "layer.50.self_attn.k_proj", + "ppl": 4.106762886047363 + }, + { + "layer": "layer.50.self_attn.v_proj", + "ppl": 4.107106685638428 + }, + { + "layer": "layer.50.self_attn.o_proj", + "ppl": 4.107019901275635 + }, + { + "layer": "layer.50.mlp.gate_proj", + "ppl": 4.106886386871338 + }, + { + "layer": "layer.50.mlp.up_proj", + "ppl": 4.106566905975342 + }, + { + "layer": "layer.50.mlp.down_proj", + "ppl": 4.107048988342285 + }, + { + "layer": "layer.51.self_attn.q_proj", + "ppl": 4.107061386108398 + }, + { + "layer": "layer.51.self_attn.k_proj", + "ppl": 4.1068949699401855 + }, + { + "layer": "layer.51.self_attn.v_proj", + "ppl": 4.106844425201416 + }, + { + "layer": "layer.51.self_attn.o_proj", + "ppl": 4.106922626495361 + }, + { + "layer": "layer.51.mlp.gate_proj", + "ppl": 4.106790065765381 + }, + { + "layer": "layer.51.mlp.up_proj", + "ppl": 4.10669469833374 + }, + { + "layer": "layer.51.mlp.down_proj", + "ppl": 4.106770038604736 + }, + { + "layer": "layer.52.self_attn.q_proj", + "ppl": 4.106804847717285 + }, + { + "layer": "layer.52.self_attn.k_proj", + "ppl": 4.106805801391602 + }, + { + "layer": "layer.52.self_attn.v_proj", + "ppl": 4.106969833374023 + }, + { + "layer": "layer.52.self_attn.o_proj", + "ppl": 4.106813430786133 + }, + { + "layer": "layer.52.mlp.gate_proj", + "ppl": 4.1065263748168945 + }, + { + "layer": "layer.52.mlp.up_proj", + "ppl": 4.106884002685547 + }, + { + "layer": "layer.52.mlp.down_proj", + "ppl": 4.106695652008057 + }, + { + "layer": "layer.53.self_attn.q_proj", + "ppl": 4.106993675231934 + }, + { + "layer": "layer.53.self_attn.k_proj", + "ppl": 4.10698127746582 + }, + { + "layer": "layer.53.self_attn.v_proj", + "ppl": 4.106917381286621 + }, + { + "layer": "layer.53.self_attn.o_proj", + "ppl": 4.106783390045166 + }, + { + "layer": "layer.53.mlp.gate_proj", + "ppl": 4.106466293334961 + }, + { + "layer": "layer.53.mlp.up_proj", + "ppl": 4.107027053833008 + }, + { + "layer": "layer.53.mlp.down_proj", + "ppl": 4.1069207191467285 + }, + { + "layer": "layer.54.self_attn.q_proj", + "ppl": 4.106971263885498 + }, + { + "layer": "layer.54.self_attn.k_proj", + "ppl": 4.10692834854126 + }, + { + "layer": "layer.54.self_attn.v_proj", + "ppl": 4.106964588165283 + }, + { + "layer": "layer.54.self_attn.o_proj", + "ppl": 4.106781005859375 + }, + { + "layer": "layer.54.mlp.gate_proj", + "ppl": 4.10678768157959 + }, + { + "layer": "layer.54.mlp.up_proj", + "ppl": 4.10673713684082 + }, + { + "layer": "layer.54.mlp.down_proj", + "ppl": 4.106933116912842 + }, + { + "layer": "layer.55.self_attn.q_proj", + "ppl": 4.106923580169678 + }, + { + "layer": "layer.55.self_attn.k_proj", + "ppl": 4.10691499710083 + }, + { + "layer": "layer.55.self_attn.v_proj", + "ppl": 4.10683536529541 + }, + { + "layer": "layer.55.self_attn.o_proj", + "ppl": 4.106876373291016 + }, + { + "layer": "layer.55.mlp.gate_proj", + "ppl": 4.106630802154541 + }, + { + "layer": "layer.55.mlp.up_proj", + "ppl": 4.106686592102051 + }, + { + "layer": "layer.55.mlp.down_proj", + "ppl": 4.106542587280273 + }, + { + "layer": "layer.56.self_attn.q_proj", + "ppl": 4.106923580169678 + }, + { + "layer": "layer.56.self_attn.k_proj", + "ppl": 4.106812477111816 + }, + { + "layer": "layer.56.self_attn.v_proj", + "ppl": 4.106831073760986 + }, + { + "layer": "layer.56.self_attn.o_proj", + "ppl": 4.106750965118408 + }, + { + "layer": "layer.56.mlp.gate_proj", + "ppl": 4.106784343719482 + }, + { + "layer": "layer.56.mlp.up_proj", + "ppl": 4.106832027435303 + }, + { + "layer": "layer.56.mlp.down_proj", + "ppl": 4.106601238250732 + }, + { + "layer": "layer.57.self_attn.q_proj", + "ppl": 4.106966018676758 + }, + { + "layer": "layer.57.self_attn.k_proj", + "ppl": 4.10683012008667 + }, + { + "layer": "layer.57.self_attn.v_proj", + "ppl": 4.106881141662598 + }, + { + "layer": "layer.57.self_attn.o_proj", + "ppl": 4.106851100921631 + }, + { + "layer": "layer.57.mlp.gate_proj", + "ppl": 4.106785297393799 + }, + { + "layer": "layer.57.mlp.up_proj", + "ppl": 4.107060432434082 + }, + { + "layer": "layer.57.mlp.down_proj", + "ppl": 4.1070556640625 + }, + { + "layer": "layer.58.self_attn.q_proj", + "ppl": 4.10699462890625 + }, + { + "layer": "layer.58.self_attn.k_proj", + "ppl": 4.106889247894287 + }, + { + "layer": "layer.58.self_attn.v_proj", + "ppl": 4.106931209564209 + }, + { + "layer": "layer.58.self_attn.o_proj", + "ppl": 4.106766700744629 + }, + { + "layer": "layer.58.mlp.gate_proj", + "ppl": 4.106771469116211 + }, + { + "layer": "layer.58.mlp.up_proj", + "ppl": 4.106778144836426 + }, + { + "layer": "layer.58.mlp.down_proj", + "ppl": 4.107176303863525 + }, + { + "layer": "layer.59.self_attn.q_proj", + "ppl": 4.106899261474609 + }, + { + "layer": "layer.59.self_attn.k_proj", + "ppl": 4.106889724731445 + }, + { + "layer": "layer.59.self_attn.v_proj", + "ppl": 4.106855392456055 + }, + { + "layer": "layer.59.self_attn.o_proj", + "ppl": 4.106821537017822 + }, + { + "layer": "layer.59.mlp.gate_proj", + "ppl": 4.106753826141357 + }, + { + "layer": "layer.59.mlp.up_proj", + "ppl": 4.106740474700928 + }, + { + "layer": "layer.59.mlp.down_proj", + "ppl": 4.1069416999816895 + }, + { + "layer": "layer.60.self_attn.q_proj", + "ppl": 4.1068925857543945 + }, + { + "layer": "layer.60.self_attn.k_proj", + "ppl": 4.106865406036377 + }, + { + "layer": "layer.60.self_attn.v_proj", + "ppl": 4.107134819030762 + }, + { + "layer": "layer.60.self_attn.o_proj", + "ppl": 4.106810092926025 + }, + { + "layer": "layer.60.mlp.gate_proj", + "ppl": 4.106860637664795 + }, + { + "layer": "layer.60.mlp.up_proj", + "ppl": 4.107129096984863 + }, + { + "layer": "layer.60.mlp.down_proj", + "ppl": 4.106919288635254 + }, + { + "layer": "layer.61.self_attn.q_proj", + "ppl": 4.106850624084473 + }, + { + "layer": "layer.61.self_attn.k_proj", + "ppl": 4.106834411621094 + }, + { + "layer": "layer.61.self_attn.v_proj", + "ppl": 4.106821537017822 + }, + { + "layer": "layer.61.self_attn.o_proj", + "ppl": 4.106993675231934 + }, + { + "layer": "layer.61.mlp.gate_proj", + "ppl": 4.106492519378662 + }, + { + "layer": "layer.61.mlp.up_proj", + "ppl": 4.106689453125 + }, + { + "layer": "layer.61.mlp.down_proj", + "ppl": 4.106812000274658 + }, + { + "layer": "layer.62.self_attn.q_proj", + "ppl": 4.106894016265869 + }, + { + "layer": "layer.62.self_attn.k_proj", + "ppl": 4.106892108917236 + }, + { + "layer": "layer.62.self_attn.v_proj", + "ppl": 4.106905937194824 + }, + { + "layer": "layer.62.self_attn.o_proj", + "ppl": 4.106696128845215 + }, + { + "layer": "layer.62.mlp.gate_proj", + "ppl": 4.1064252853393555 + }, + { + "layer": "layer.62.mlp.up_proj", + "ppl": 4.106868267059326 + }, + { + "layer": "layer.62.mlp.down_proj", + "ppl": 4.107044696807861 + }, + { + "layer": "layer.63.self_attn.q_proj", + "ppl": 4.106893062591553 + }, + { + "layer": "layer.63.self_attn.k_proj", + "ppl": 4.106963634490967 + }, + { + "layer": "layer.63.self_attn.v_proj", + "ppl": 4.106774806976318 + }, + { + "layer": "layer.63.self_attn.o_proj", + "ppl": 4.106854438781738 + }, + { + "layer": "layer.63.mlp.gate_proj", + "ppl": 4.106828689575195 + }, + { + "layer": "layer.63.mlp.up_proj", + "ppl": 4.106839179992676 + }, + { + "layer": "layer.63.mlp.down_proj", + "ppl": 4.10702657699585 + }, + { + "layer": "layer.64.self_attn.q_proj", + "ppl": 4.106788158416748 + }, + { + "layer": "layer.64.self_attn.k_proj", + "ppl": 4.106810092926025 + }, + { + "layer": "layer.64.self_attn.v_proj", + "ppl": 4.106852054595947 + }, + { + "layer": "layer.64.self_attn.o_proj", + "ppl": 4.106967449188232 + }, + { + "layer": "layer.64.mlp.gate_proj", + "ppl": 4.106573104858398 + }, + { + "layer": "layer.64.mlp.up_proj", + "ppl": 4.106992721557617 + }, + { + "layer": "layer.64.mlp.down_proj", + "ppl": 4.10681676864624 + }, + { + "layer": "layer.65.self_attn.q_proj", + "ppl": 4.106734752655029 + }, + { + "layer": "layer.65.self_attn.k_proj", + "ppl": 4.106735706329346 + }, + { + "layer": "layer.65.self_attn.v_proj", + "ppl": 4.1068830490112305 + }, + { + "layer": "layer.65.self_attn.o_proj", + "ppl": 4.106762886047363 + }, + { + "layer": "layer.65.mlp.gate_proj", + "ppl": 4.1065568923950195 + }, + { + "layer": "layer.65.mlp.up_proj", + "ppl": 4.106799125671387 + }, + { + "layer": "layer.65.mlp.down_proj", + "ppl": 4.106886386871338 + }, + { + "layer": "layer.66.self_attn.q_proj", + "ppl": 4.106924057006836 + }, + { + "layer": "layer.66.self_attn.k_proj", + "ppl": 4.106876373291016 + }, + { + "layer": "layer.66.self_attn.v_proj", + "ppl": 4.106762886047363 + }, + { + "layer": "layer.66.self_attn.o_proj", + "ppl": 4.106837749481201 + }, + { + "layer": "layer.66.mlp.gate_proj", + "ppl": 4.106515884399414 + }, + { + "layer": "layer.66.mlp.up_proj", + "ppl": 4.107038974761963 + }, + { + "layer": "layer.66.mlp.down_proj", + "ppl": 4.106561183929443 + }, + { + "layer": "layer.67.self_attn.q_proj", + "ppl": 4.106843948364258 + }, + { + "layer": "layer.67.self_attn.k_proj", + "ppl": 4.1068267822265625 + }, + { + "layer": "layer.67.self_attn.v_proj", + "ppl": 4.106842994689941 + }, + { + "layer": "layer.67.self_attn.o_proj", + "ppl": 4.106961727142334 + }, + { + "layer": "layer.67.mlp.gate_proj", + "ppl": 4.106598377227783 + }, + { + "layer": "layer.67.mlp.up_proj", + "ppl": 4.106889724731445 + }, + { + "layer": "layer.67.mlp.down_proj", + "ppl": 4.107000350952148 + }, + { + "layer": "layer.68.self_attn.q_proj", + "ppl": 4.106785774230957 + }, + { + "layer": "layer.68.self_attn.k_proj", + "ppl": 4.106930255889893 + }, + { + "layer": "layer.68.self_attn.v_proj", + "ppl": 4.106966495513916 + }, + { + "layer": "layer.68.self_attn.o_proj", + "ppl": 4.106743335723877 + }, + { + "layer": "layer.68.mlp.gate_proj", + "ppl": 4.106333255767822 + }, + { + "layer": "layer.68.mlp.up_proj", + "ppl": 4.106931209564209 + }, + { + "layer": "layer.68.mlp.down_proj", + "ppl": 4.106650352478027 + }, + { + "layer": "layer.69.self_attn.q_proj", + "ppl": 4.106945037841797 + }, + { + "layer": "layer.69.self_attn.k_proj", + "ppl": 4.106853008270264 + }, + { + "layer": "layer.69.self_attn.v_proj", + "ppl": 4.106881141662598 + }, + { + "layer": "layer.69.self_attn.o_proj", + "ppl": 4.106881141662598 + }, + { + "layer": "layer.69.mlp.gate_proj", + "ppl": 4.1068925857543945 + }, + { + "layer": "layer.69.mlp.up_proj", + "ppl": 4.106877326965332 + }, + { + "layer": "layer.69.mlp.down_proj", + "ppl": 4.106451988220215 + }, + { + "layer": "layer.70.self_attn.q_proj", + "ppl": 4.106842041015625 + }, + { + "layer": "layer.70.self_attn.k_proj", + "ppl": 4.106881618499756 + }, + { + "layer": "layer.70.self_attn.v_proj", + "ppl": 4.106894493103027 + }, + { + "layer": "layer.70.self_attn.o_proj", + "ppl": 4.106858730316162 + }, + { + "layer": "layer.70.mlp.gate_proj", + "ppl": 4.1064958572387695 + }, + { + "layer": "layer.70.mlp.up_proj", + "ppl": 4.106657981872559 + }, + { + "layer": "layer.70.mlp.down_proj", + "ppl": 4.106827735900879 + }, + { + "layer": "layer.71.self_attn.q_proj", + "ppl": 4.106942653656006 + }, + { + "layer": "layer.71.self_attn.k_proj", + "ppl": 4.106905937194824 + }, + { + "layer": "layer.71.self_attn.v_proj", + "ppl": 4.106865406036377 + }, + { + "layer": "layer.71.self_attn.o_proj", + "ppl": 4.106843948364258 + }, + { + "layer": "layer.71.mlp.gate_proj", + "ppl": 4.106942653656006 + }, + { + "layer": "layer.71.mlp.up_proj", + "ppl": 4.106804847717285 + }, + { + "layer": "layer.71.mlp.down_proj", + "ppl": 4.107132434844971 + }, + { + "layer": "layer.72.self_attn.q_proj", + "ppl": 4.106801986694336 + }, + { + "layer": "layer.72.self_attn.k_proj", + "ppl": 4.106903076171875 + }, + { + "layer": "layer.72.self_attn.v_proj", + "ppl": 4.1069016456604 + }, + { + "layer": "layer.72.self_attn.o_proj", + "ppl": 4.1069416999816895 + }, + { + "layer": "layer.72.mlp.gate_proj", + "ppl": 4.106686592102051 + }, + { + "layer": "layer.72.mlp.up_proj", + "ppl": 4.106869697570801 + }, + { + "layer": "layer.72.mlp.down_proj", + "ppl": 4.1068010330200195 + }, + { + "layer": "layer.73.self_attn.q_proj", + "ppl": 4.106989860534668 + }, + { + "layer": "layer.73.self_attn.k_proj", + "ppl": 4.106840133666992 + }, + { + "layer": "layer.73.self_attn.v_proj", + "ppl": 4.106929302215576 + }, + { + "layer": "layer.73.self_attn.o_proj", + "ppl": 4.10686731338501 + }, + { + "layer": "layer.73.mlp.gate_proj", + "ppl": 4.106245040893555 + }, + { + "layer": "layer.73.mlp.up_proj", + "ppl": 4.106766700744629 + }, + { + "layer": "layer.73.mlp.down_proj", + "ppl": 4.106709003448486 + }, + { + "layer": "layer.74.self_attn.q_proj", + "ppl": 4.106903076171875 + }, + { + "layer": "layer.74.self_attn.k_proj", + "ppl": 4.106989860534668 + }, + { + "layer": "layer.74.self_attn.v_proj", + "ppl": 4.1069183349609375 + }, + { + "layer": "layer.74.self_attn.o_proj", + "ppl": 4.10684871673584 + }, + { + "layer": "layer.74.mlp.gate_proj", + "ppl": 4.106860637664795 + }, + { + "layer": "layer.74.mlp.up_proj", + "ppl": 4.106688499450684 + }, + { + "layer": "layer.74.mlp.down_proj", + "ppl": 4.106952667236328 + }, + { + "layer": "layer.75.self_attn.q_proj", + "ppl": 4.106893062591553 + }, + { + "layer": "layer.75.self_attn.k_proj", + "ppl": 4.106898307800293 + }, + { + "layer": "layer.75.self_attn.v_proj", + "ppl": 4.106749534606934 + }, + { + "layer": "layer.75.self_attn.o_proj", + "ppl": 4.106872081756592 + }, + { + "layer": "layer.75.mlp.gate_proj", + "ppl": 4.106604099273682 + }, + { + "layer": "layer.75.mlp.up_proj", + "ppl": 4.106713771820068 + }, + { + "layer": "layer.75.mlp.down_proj", + "ppl": 4.106762886047363 + }, + { + "layer": "layer.76.self_attn.q_proj", + "ppl": 4.1068549156188965 + }, + { + "layer": "layer.76.self_attn.k_proj", + "ppl": 4.106991291046143 + }, + { + "layer": "layer.76.self_attn.v_proj", + "ppl": 4.106698036193848 + }, + { + "layer": "layer.76.self_attn.o_proj", + "ppl": 4.10683536529541 + }, + { + "layer": "layer.76.mlp.gate_proj", + "ppl": 4.10635232925415 + }, + { + "layer": "layer.76.mlp.up_proj", + "ppl": 4.106674671173096 + }, + { + "layer": "layer.76.mlp.down_proj", + "ppl": 4.106673717498779 + }, + { + "layer": "layer.77.self_attn.q_proj", + "ppl": 4.106898784637451 + }, + { + "layer": "layer.77.self_attn.k_proj", + "ppl": 4.106923580169678 + }, + { + "layer": "layer.77.self_attn.v_proj", + "ppl": 4.106679439544678 + }, + { + "layer": "layer.77.self_attn.o_proj", + "ppl": 4.1069016456604 + }, + { + "layer": "layer.77.mlp.gate_proj", + "ppl": 4.106555461883545 + }, + { + "layer": "layer.77.mlp.up_proj", + "ppl": 4.1067633628845215 + }, + { + "layer": "layer.77.mlp.down_proj", + "ppl": 4.106751441955566 + }, + { + "layer": "layer.78.self_attn.q_proj", + "ppl": 4.106810092926025 + }, + { + "layer": "layer.78.self_attn.k_proj", + "ppl": 4.106899261474609 + }, + { + "layer": "layer.78.self_attn.v_proj", + "ppl": 4.106929302215576 + }, + { + "layer": "layer.78.self_attn.o_proj", + "ppl": 4.107081890106201 + }, + { + "layer": "layer.78.mlp.gate_proj", + "ppl": 4.106260299682617 + }, + { + "layer": "layer.78.mlp.up_proj", + "ppl": 4.106597423553467 + }, + { + "layer": "layer.78.mlp.down_proj", + "ppl": 4.106743335723877 + }, + { + "layer": "layer.79.self_attn.q_proj", + "ppl": 4.106771469116211 + }, + { + "layer": "layer.79.self_attn.k_proj", + "ppl": 4.106843948364258 + }, + { + "layer": "layer.79.self_attn.v_proj", + "ppl": 4.1070942878723145 + }, + { + "layer": "layer.79.self_attn.o_proj", + "ppl": 4.106961727142334 + }, + { + "layer": "layer.79.mlp.gate_proj", + "ppl": 4.106101989746094 + }, + { + "layer": "layer.79.mlp.up_proj", + "ppl": 4.105664253234863 + }, + { + "layer": "layer.79.mlp.down_proj", + "ppl": 4.105630397796631 + } + ] + }, + { + "bit": 10, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 4.080214500427246 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 4.08028507232666 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 4.0801897048950195 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 4.080104827880859 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 4.080255508422852 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 4.08034086227417 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 4.080202102661133 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 4.080134868621826 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 4.080277442932129 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 4.080227375030518 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 4.080236911773682 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 4.080282211303711 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 4.080141067504883 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 4.08017110824585 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 4.080201148986816 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 4.080256938934326 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 4.080170631408691 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 4.08021879196167 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 4.080312252044678 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 4.080135822296143 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 4.080057621002197 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 4.080275058746338 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 4.080256462097168 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 4.080211639404297 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 4.080253601074219 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 4.080117225646973 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 4.080186367034912 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 4.080185413360596 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 4.08015251159668 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 4.080224990844727 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 4.080172061920166 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 4.080223560333252 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 4.080237865447998 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 4.080279350280762 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 4.080204010009766 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 4.0802001953125 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 4.080219268798828 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 4.0801777839660645 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 4.080185413360596 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 4.08016300201416 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 4.080255508422852 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 4.080144882202148 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 4.08017110824585 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 4.080193042755127 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 4.080179214477539 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 4.080151081085205 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 4.080272674560547 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 4.080180644989014 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 4.080215930938721 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 4.0802483558654785 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 4.080282688140869 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 4.080254554748535 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 4.080354690551758 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 4.0802083015441895 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 4.080320835113525 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 4.080292701721191 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 4.080125331878662 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 4.080131530761719 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 4.080251693725586 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 4.0802412033081055 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 4.080219745635986 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 4.080109119415283 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 4.080193042755127 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 4.080190658569336 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 4.080226421356201 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 4.080241680145264 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 4.080256938934326 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 4.080199718475342 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 4.080197811126709 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 4.080281734466553 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 4.080216884613037 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 4.080180644989014 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 4.080318450927734 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 4.080172061920166 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 4.080050468444824 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 4.0802412033081055 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 4.080272197723389 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 4.080245018005371 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 4.080216884613037 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 4.080076694488525 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 4.08024787902832 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 4.080185413360596 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 4.080142498016357 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 4.080317974090576 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 4.0803985595703125 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 4.080201148986816 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 4.080258846282959 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 4.080198287963867 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 4.080134868621826 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 4.080204010009766 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 4.080204010009766 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 4.080220699310303 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 4.080111503601074 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 4.080143928527832 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 4.08026647567749 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 4.0801472663879395 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 4.080338954925537 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 4.080299377441406 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 4.080180644989014 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 4.080256462097168 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 4.080191612243652 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 4.080216884613037 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 4.080204010009766 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 4.08021879196167 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 4.080242156982422 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 4.080198287963867 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 4.080216884613037 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 4.080152988433838 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 4.080177307128906 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 4.0802083015441895 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 4.0802321434021 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 4.080263137817383 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 4.080254554748535 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 4.080231189727783 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 4.080252647399902 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 4.080242156982422 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 4.080202579498291 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 4.080280780792236 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 4.080226421356201 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 4.080125331878662 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 4.0802106857299805 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 4.080153942108154 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 4.080228805541992 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 4.0801777839660645 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 4.080193042755127 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 4.080296039581299 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 4.080264091491699 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 4.080138206481934 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 4.080153465270996 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 4.080198764801025 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 4.080171585083008 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 4.080267429351807 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 4.080043315887451 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 4.080271244049072 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 4.080227851867676 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 4.080056190490723 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 4.08034086227417 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 4.080175876617432 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 4.080178737640381 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 4.080384254455566 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 4.08019495010376 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 4.080214977264404 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 4.080109119415283 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 4.080173015594482 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 4.080197811126709 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 4.080101013183594 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 4.080136775970459 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 4.08027458190918 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 4.080144882202148 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 4.080268383026123 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 4.08022928237915 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 4.0802106857299805 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 4.080271244049072 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 4.080097675323486 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 4.0801262855529785 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 4.080087184906006 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 4.080136775970459 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 4.080082893371582 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 4.080176830291748 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 4.080262184143066 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 4.080158710479736 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 4.080203533172607 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 4.080204010009766 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 4.080275535583496 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 4.080214977264404 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 4.080225467681885 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 4.0802202224731445 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 4.080275058746338 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 4.080219745635986 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 4.080138206481934 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 4.080214977264404 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 4.080113887786865 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 4.08015775680542 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 4.080194473266602 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 4.080408573150635 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 4.080091953277588 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 4.08021354675293 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 4.080101013183594 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 4.080174922943115 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 4.0802178382873535 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 4.08009147644043 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 4.079967498779297 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 4.080152988433838 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 4.080157279968262 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 4.0802741050720215 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 4.080202579498291 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 4.080138206481934 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 4.08018159866333 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 4.080225467681885 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 4.0803093910217285 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 4.080151557922363 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 4.0801873207092285 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 4.0802903175354 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 4.080338954925537 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 4.080063343048096 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 4.080183982849121 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 4.080175876617432 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 4.080333709716797 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 4.0802788734436035 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 4.080100059509277 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 4.0803961753845215 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 4.080043792724609 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 4.080199718475342 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 4.080193996429443 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 4.0801777839660645 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 4.080172061920166 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 4.08001708984375 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 4.08022928237915 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 4.080253601074219 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 4.080190658569336 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 4.080297470092773 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 4.080294609069824 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 4.080246925354004 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 4.080221652984619 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 4.080060005187988 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 4.08013916015625 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 4.07998514175415 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 4.080142498016357 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 4.080167770385742 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 4.080173015594482 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 4.080074787139893 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 4.08033561706543 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 4.080345153808594 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 4.079986572265625 + }, + { + "layer": "layer.32.self_attn.q_proj", + "ppl": 4.080214023590088 + }, + { + "layer": "layer.32.self_attn.k_proj", + "ppl": 4.080272197723389 + }, + { + "layer": "layer.32.self_attn.v_proj", + "ppl": 4.0802083015441895 + }, + { + "layer": "layer.32.self_attn.o_proj", + "ppl": 4.080249786376953 + }, + { + "layer": "layer.32.mlp.gate_proj", + "ppl": 4.0801472663879395 + }, + { + "layer": "layer.32.mlp.up_proj", + "ppl": 4.0801873207092285 + }, + { + "layer": "layer.32.mlp.down_proj", + "ppl": 4.080172538757324 + }, + { + "layer": "layer.33.self_attn.q_proj", + "ppl": 4.080176830291748 + }, + { + "layer": "layer.33.self_attn.k_proj", + "ppl": 4.080329895019531 + }, + { + "layer": "layer.33.self_attn.v_proj", + "ppl": 4.080207347869873 + }, + { + "layer": "layer.33.self_attn.o_proj", + "ppl": 4.080165386199951 + }, + { + "layer": "layer.33.mlp.gate_proj", + "ppl": 4.079769611358643 + }, + { + "layer": "layer.33.mlp.up_proj", + "ppl": 4.080190658569336 + }, + { + "layer": "layer.33.mlp.down_proj", + "ppl": 4.080430030822754 + }, + { + "layer": "layer.34.self_attn.q_proj", + "ppl": 4.080097675323486 + }, + { + "layer": "layer.34.self_attn.k_proj", + "ppl": 4.0802202224731445 + }, + { + "layer": "layer.34.self_attn.v_proj", + "ppl": 4.080145835876465 + }, + { + "layer": "layer.34.self_attn.o_proj", + "ppl": 4.080104827880859 + }, + { + "layer": "layer.34.mlp.gate_proj", + "ppl": 4.080310344696045 + }, + { + "layer": "layer.34.mlp.up_proj", + "ppl": 4.080336093902588 + }, + { + "layer": "layer.34.mlp.down_proj", + "ppl": 4.080108165740967 + }, + { + "layer": "layer.35.self_attn.q_proj", + "ppl": 4.0801215171813965 + }, + { + "layer": "layer.35.self_attn.k_proj", + "ppl": 4.08027982711792 + }, + { + "layer": "layer.35.self_attn.v_proj", + "ppl": 4.08001708984375 + }, + { + "layer": "layer.35.self_attn.o_proj", + "ppl": 4.080245494842529 + }, + { + "layer": "layer.35.mlp.gate_proj", + "ppl": 4.080279350280762 + }, + { + "layer": "layer.35.mlp.up_proj", + "ppl": 4.080006122589111 + }, + { + "layer": "layer.35.mlp.down_proj", + "ppl": 4.0800700187683105 + }, + { + "layer": "layer.36.self_attn.q_proj", + "ppl": 4.080322265625 + }, + { + "layer": "layer.36.self_attn.k_proj", + "ppl": 4.0802001953125 + }, + { + "layer": "layer.36.self_attn.v_proj", + "ppl": 4.080231666564941 + }, + { + "layer": "layer.36.self_attn.o_proj", + "ppl": 4.080174922943115 + }, + { + "layer": "layer.36.mlp.gate_proj", + "ppl": 4.080124378204346 + }, + { + "layer": "layer.36.mlp.up_proj", + "ppl": 4.080410480499268 + }, + { + "layer": "layer.36.mlp.down_proj", + "ppl": 4.079878330230713 + }, + { + "layer": "layer.37.self_attn.q_proj", + "ppl": 4.080132007598877 + }, + { + "layer": "layer.37.self_attn.k_proj", + "ppl": 4.080020427703857 + }, + { + "layer": "layer.37.self_attn.v_proj", + "ppl": 4.080254554748535 + }, + { + "layer": "layer.37.self_attn.o_proj", + "ppl": 4.080242156982422 + }, + { + "layer": "layer.37.mlp.gate_proj", + "ppl": 4.080062389373779 + }, + { + "layer": "layer.37.mlp.up_proj", + "ppl": 4.080245494842529 + }, + { + "layer": "layer.37.mlp.down_proj", + "ppl": 4.080214977264404 + }, + { + "layer": "layer.38.self_attn.q_proj", + "ppl": 4.080219745635986 + }, + { + "layer": "layer.38.self_attn.k_proj", + "ppl": 4.080224514007568 + }, + { + "layer": "layer.38.self_attn.v_proj", + "ppl": 4.080320835113525 + }, + { + "layer": "layer.38.self_attn.o_proj", + "ppl": 4.080164432525635 + }, + { + "layer": "layer.38.mlp.gate_proj", + "ppl": 4.080029010772705 + }, + { + "layer": "layer.38.mlp.up_proj", + "ppl": 4.080221652984619 + }, + { + "layer": "layer.38.mlp.down_proj", + "ppl": 4.080386638641357 + }, + { + "layer": "layer.39.self_attn.q_proj", + "ppl": 4.080241680145264 + }, + { + "layer": "layer.39.self_attn.k_proj", + "ppl": 4.080240249633789 + }, + { + "layer": "layer.39.self_attn.v_proj", + "ppl": 4.080179691314697 + }, + { + "layer": "layer.39.self_attn.o_proj", + "ppl": 4.080137729644775 + }, + { + "layer": "layer.39.mlp.gate_proj", + "ppl": 4.080201148986816 + }, + { + "layer": "layer.39.mlp.up_proj", + "ppl": 4.080419063568115 + }, + { + "layer": "layer.39.mlp.down_proj", + "ppl": 4.080158710479736 + }, + { + "layer": "layer.40.self_attn.q_proj", + "ppl": 4.080176830291748 + }, + { + "layer": "layer.40.self_attn.k_proj", + "ppl": 4.080115795135498 + }, + { + "layer": "layer.40.self_attn.v_proj", + "ppl": 4.080170631408691 + }, + { + "layer": "layer.40.self_attn.o_proj", + "ppl": 4.080164432525635 + }, + { + "layer": "layer.40.mlp.gate_proj", + "ppl": 4.080186367034912 + }, + { + "layer": "layer.40.mlp.up_proj", + "ppl": 4.080111503601074 + }, + { + "layer": "layer.40.mlp.down_proj", + "ppl": 4.080240249633789 + }, + { + "layer": "layer.41.self_attn.q_proj", + "ppl": 4.080226898193359 + }, + { + "layer": "layer.41.self_attn.k_proj", + "ppl": 4.080202102661133 + }, + { + "layer": "layer.41.self_attn.v_proj", + "ppl": 4.080214023590088 + }, + { + "layer": "layer.41.self_attn.o_proj", + "ppl": 4.080299377441406 + }, + { + "layer": "layer.41.mlp.gate_proj", + "ppl": 4.080318450927734 + }, + { + "layer": "layer.41.mlp.up_proj", + "ppl": 4.08021879196167 + }, + { + "layer": "layer.41.mlp.down_proj", + "ppl": 4.080284118652344 + }, + { + "layer": "layer.42.self_attn.q_proj", + "ppl": 4.080299377441406 + }, + { + "layer": "layer.42.self_attn.k_proj", + "ppl": 4.0803751945495605 + }, + { + "layer": "layer.42.self_attn.v_proj", + "ppl": 4.080070972442627 + }, + { + "layer": "layer.42.self_attn.o_proj", + "ppl": 4.0803070068359375 + }, + { + "layer": "layer.42.mlp.gate_proj", + "ppl": 4.080286026000977 + }, + { + "layer": "layer.42.mlp.up_proj", + "ppl": 4.080162048339844 + }, + { + "layer": "layer.42.mlp.down_proj", + "ppl": 4.080042362213135 + }, + { + "layer": "layer.43.self_attn.q_proj", + "ppl": 4.080226421356201 + }, + { + "layer": "layer.43.self_attn.k_proj", + "ppl": 4.080254077911377 + }, + { + "layer": "layer.43.self_attn.v_proj", + "ppl": 4.080256462097168 + }, + { + "layer": "layer.43.self_attn.o_proj", + "ppl": 4.080094814300537 + }, + { + "layer": "layer.43.mlp.gate_proj", + "ppl": 4.080028057098389 + }, + { + "layer": "layer.43.mlp.up_proj", + "ppl": 4.080280780792236 + }, + { + "layer": "layer.43.mlp.down_proj", + "ppl": 4.080255508422852 + }, + { + "layer": "layer.44.self_attn.q_proj", + "ppl": 4.080231189727783 + }, + { + "layer": "layer.44.self_attn.k_proj", + "ppl": 4.080270290374756 + }, + { + "layer": "layer.44.self_attn.v_proj", + "ppl": 4.08024263381958 + }, + { + "layer": "layer.44.self_attn.o_proj", + "ppl": 4.08017635345459 + }, + { + "layer": "layer.44.mlp.gate_proj", + "ppl": 4.080011367797852 + }, + { + "layer": "layer.44.mlp.up_proj", + "ppl": 4.080377101898193 + }, + { + "layer": "layer.44.mlp.down_proj", + "ppl": 4.079769611358643 + }, + { + "layer": "layer.45.self_attn.q_proj", + "ppl": 4.080239295959473 + }, + { + "layer": "layer.45.self_attn.k_proj", + "ppl": 4.08021879196167 + }, + { + "layer": "layer.45.self_attn.v_proj", + "ppl": 4.080313682556152 + }, + { + "layer": "layer.45.self_attn.o_proj", + "ppl": 4.080195903778076 + }, + { + "layer": "layer.45.mlp.gate_proj", + "ppl": 4.079916954040527 + }, + { + "layer": "layer.45.mlp.up_proj", + "ppl": 4.080059051513672 + }, + { + "layer": "layer.45.mlp.down_proj", + "ppl": 4.080000400543213 + }, + { + "layer": "layer.46.self_attn.q_proj", + "ppl": 4.080106735229492 + }, + { + "layer": "layer.46.self_attn.k_proj", + "ppl": 4.07999849319458 + }, + { + "layer": "layer.46.self_attn.v_proj", + "ppl": 4.080245494842529 + }, + { + "layer": "layer.46.self_attn.o_proj", + "ppl": 4.080019474029541 + }, + { + "layer": "layer.46.mlp.gate_proj", + "ppl": 4.080233573913574 + }, + { + "layer": "layer.46.mlp.up_proj", + "ppl": 4.080235958099365 + }, + { + "layer": "layer.46.mlp.down_proj", + "ppl": 4.080252170562744 + }, + { + "layer": "layer.47.self_attn.q_proj", + "ppl": 4.080198764801025 + }, + { + "layer": "layer.47.self_attn.k_proj", + "ppl": 4.080151557922363 + }, + { + "layer": "layer.47.self_attn.v_proj", + "ppl": 4.080153465270996 + }, + { + "layer": "layer.47.self_attn.o_proj", + "ppl": 4.079958915710449 + }, + { + "layer": "layer.47.mlp.gate_proj", + "ppl": 4.080094814300537 + }, + { + "layer": "layer.47.mlp.up_proj", + "ppl": 4.080381870269775 + }, + { + "layer": "layer.47.mlp.down_proj", + "ppl": 4.080105304718018 + }, + { + "layer": "layer.48.self_attn.q_proj", + "ppl": 4.080061912536621 + }, + { + "layer": "layer.48.self_attn.k_proj", + "ppl": 4.080284118652344 + }, + { + "layer": "layer.48.self_attn.v_proj", + "ppl": 4.080287456512451 + }, + { + "layer": "layer.48.self_attn.o_proj", + "ppl": 4.080279350280762 + }, + { + "layer": "layer.48.mlp.gate_proj", + "ppl": 4.080085277557373 + }, + { + "layer": "layer.48.mlp.up_proj", + "ppl": 4.080288410186768 + }, + { + "layer": "layer.48.mlp.down_proj", + "ppl": 4.080024719238281 + }, + { + "layer": "layer.49.self_attn.q_proj", + "ppl": 4.080164432525635 + }, + { + "layer": "layer.49.self_attn.k_proj", + "ppl": 4.080105781555176 + }, + { + "layer": "layer.49.self_attn.v_proj", + "ppl": 4.080225944519043 + }, + { + "layer": "layer.49.self_attn.o_proj", + "ppl": 4.080106258392334 + }, + { + "layer": "layer.49.mlp.gate_proj", + "ppl": 4.080209255218506 + }, + { + "layer": "layer.49.mlp.up_proj", + "ppl": 4.080013751983643 + }, + { + "layer": "layer.49.mlp.down_proj", + "ppl": 4.080198287963867 + }, + { + "layer": "layer.50.self_attn.q_proj", + "ppl": 4.0800981521606445 + }, + { + "layer": "layer.50.self_attn.k_proj", + "ppl": 4.08019495010376 + }, + { + "layer": "layer.50.self_attn.v_proj", + "ppl": 4.080142498016357 + }, + { + "layer": "layer.50.self_attn.o_proj", + "ppl": 4.080143928527832 + }, + { + "layer": "layer.50.mlp.gate_proj", + "ppl": 4.08018159866333 + }, + { + "layer": "layer.50.mlp.up_proj", + "ppl": 4.0806050300598145 + }, + { + "layer": "layer.50.mlp.down_proj", + "ppl": 4.079850196838379 + }, + { + "layer": "layer.51.self_attn.q_proj", + "ppl": 4.08026123046875 + }, + { + "layer": "layer.51.self_attn.k_proj", + "ppl": 4.080140113830566 + }, + { + "layer": "layer.51.self_attn.v_proj", + "ppl": 4.079950332641602 + }, + { + "layer": "layer.51.self_attn.o_proj", + "ppl": 4.080049514770508 + }, + { + "layer": "layer.51.mlp.gate_proj", + "ppl": 4.080185413360596 + }, + { + "layer": "layer.51.mlp.up_proj", + "ppl": 4.0802321434021 + }, + { + "layer": "layer.51.mlp.down_proj", + "ppl": 4.080345153808594 + }, + { + "layer": "layer.52.self_attn.q_proj", + "ppl": 4.080253601074219 + }, + { + "layer": "layer.52.self_attn.k_proj", + "ppl": 4.080193042755127 + }, + { + "layer": "layer.52.self_attn.v_proj", + "ppl": 4.080165386199951 + }, + { + "layer": "layer.52.self_attn.o_proj", + "ppl": 4.08035135269165 + }, + { + "layer": "layer.52.mlp.gate_proj", + "ppl": 4.080235958099365 + }, + { + "layer": "layer.52.mlp.up_proj", + "ppl": 4.080311298370361 + }, + { + "layer": "layer.52.mlp.down_proj", + "ppl": 4.080028057098389 + }, + { + "layer": "layer.53.self_attn.q_proj", + "ppl": 4.08013916015625 + }, + { + "layer": "layer.53.self_attn.k_proj", + "ppl": 4.080144882202148 + }, + { + "layer": "layer.53.self_attn.v_proj", + "ppl": 4.080167293548584 + }, + { + "layer": "layer.53.self_attn.o_proj", + "ppl": 4.080153465270996 + }, + { + "layer": "layer.53.mlp.gate_proj", + "ppl": 4.080071926116943 + }, + { + "layer": "layer.53.mlp.up_proj", + "ppl": 4.0800299644470215 + }, + { + "layer": "layer.53.mlp.down_proj", + "ppl": 4.080092906951904 + }, + { + "layer": "layer.54.self_attn.q_proj", + "ppl": 4.080328941345215 + }, + { + "layer": "layer.54.self_attn.k_proj", + "ppl": 4.080190658569336 + }, + { + "layer": "layer.54.self_attn.v_proj", + "ppl": 4.080136775970459 + }, + { + "layer": "layer.54.self_attn.o_proj", + "ppl": 4.080158710479736 + }, + { + "layer": "layer.54.mlp.gate_proj", + "ppl": 4.080084323883057 + }, + { + "layer": "layer.54.mlp.up_proj", + "ppl": 4.080264568328857 + }, + { + "layer": "layer.54.mlp.down_proj", + "ppl": 4.079834461212158 + }, + { + "layer": "layer.55.self_attn.q_proj", + "ppl": 4.080240249633789 + }, + { + "layer": "layer.55.self_attn.k_proj", + "ppl": 4.080153465270996 + }, + { + "layer": "layer.55.self_attn.v_proj", + "ppl": 4.080190658569336 + }, + { + "layer": "layer.55.self_attn.o_proj", + "ppl": 4.080150127410889 + }, + { + "layer": "layer.55.mlp.gate_proj", + "ppl": 4.080144882202148 + }, + { + "layer": "layer.55.mlp.up_proj", + "ppl": 4.08027458190918 + }, + { + "layer": "layer.55.mlp.down_proj", + "ppl": 4.080190658569336 + }, + { + "layer": "layer.56.self_attn.q_proj", + "ppl": 4.080234527587891 + }, + { + "layer": "layer.56.self_attn.k_proj", + "ppl": 4.080167293548584 + }, + { + "layer": "layer.56.self_attn.v_proj", + "ppl": 4.080226898193359 + }, + { + "layer": "layer.56.self_attn.o_proj", + "ppl": 4.080270290374756 + }, + { + "layer": "layer.56.mlp.gate_proj", + "ppl": 4.079881191253662 + }, + { + "layer": "layer.56.mlp.up_proj", + "ppl": 4.0803422927856445 + }, + { + "layer": "layer.56.mlp.down_proj", + "ppl": 4.08005952835083 + }, + { + "layer": "layer.57.self_attn.q_proj", + "ppl": 4.0800700187683105 + }, + { + "layer": "layer.57.self_attn.k_proj", + "ppl": 4.080221176147461 + }, + { + "layer": "layer.57.self_attn.v_proj", + "ppl": 4.080265522003174 + }, + { + "layer": "layer.57.self_attn.o_proj", + "ppl": 4.080353260040283 + }, + { + "layer": "layer.57.mlp.gate_proj", + "ppl": 4.080112457275391 + }, + { + "layer": "layer.57.mlp.up_proj", + "ppl": 4.0799455642700195 + }, + { + "layer": "layer.57.mlp.down_proj", + "ppl": 4.080083847045898 + }, + { + "layer": "layer.58.self_attn.q_proj", + "ppl": 4.080140113830566 + }, + { + "layer": "layer.58.self_attn.k_proj", + "ppl": 4.080145359039307 + }, + { + "layer": "layer.58.self_attn.v_proj", + "ppl": 4.08024787902832 + }, + { + "layer": "layer.58.self_attn.o_proj", + "ppl": 4.080336570739746 + }, + { + "layer": "layer.58.mlp.gate_proj", + "ppl": 4.080108642578125 + }, + { + "layer": "layer.58.mlp.up_proj", + "ppl": 4.080380916595459 + }, + { + "layer": "layer.58.mlp.down_proj", + "ppl": 4.080039978027344 + }, + { + "layer": "layer.59.self_attn.q_proj", + "ppl": 4.080081462860107 + }, + { + "layer": "layer.59.self_attn.k_proj", + "ppl": 4.080214977264404 + }, + { + "layer": "layer.59.self_attn.v_proj", + "ppl": 4.080175876617432 + }, + { + "layer": "layer.59.self_attn.o_proj", + "ppl": 4.080197811126709 + }, + { + "layer": "layer.59.mlp.gate_proj", + "ppl": 4.079963207244873 + }, + { + "layer": "layer.59.mlp.up_proj", + "ppl": 4.080312728881836 + }, + { + "layer": "layer.59.mlp.down_proj", + "ppl": 4.0801897048950195 + }, + { + "layer": "layer.60.self_attn.q_proj", + "ppl": 4.0801591873168945 + }, + { + "layer": "layer.60.self_attn.k_proj", + "ppl": 4.0802178382873535 + }, + { + "layer": "layer.60.self_attn.v_proj", + "ppl": 4.080047607421875 + }, + { + "layer": "layer.60.self_attn.o_proj", + "ppl": 4.080117225646973 + }, + { + "layer": "layer.60.mlp.gate_proj", + "ppl": 4.079957008361816 + }, + { + "layer": "layer.60.mlp.up_proj", + "ppl": 4.080046653747559 + }, + { + "layer": "layer.60.mlp.down_proj", + "ppl": 4.080183982849121 + }, + { + "layer": "layer.61.self_attn.q_proj", + "ppl": 4.080267429351807 + }, + { + "layer": "layer.61.self_attn.k_proj", + "ppl": 4.0802226066589355 + }, + { + "layer": "layer.61.self_attn.v_proj", + "ppl": 4.080162048339844 + }, + { + "layer": "layer.61.self_attn.o_proj", + "ppl": 4.080148696899414 + }, + { + "layer": "layer.61.mlp.gate_proj", + "ppl": 4.080064296722412 + }, + { + "layer": "layer.61.mlp.up_proj", + "ppl": 4.080369472503662 + }, + { + "layer": "layer.61.mlp.down_proj", + "ppl": 4.080372333526611 + }, + { + "layer": "layer.62.self_attn.q_proj", + "ppl": 4.08027982711792 + }, + { + "layer": "layer.62.self_attn.k_proj", + "ppl": 4.080270290374756 + }, + { + "layer": "layer.62.self_attn.v_proj", + "ppl": 4.080167770385742 + }, + { + "layer": "layer.62.self_attn.o_proj", + "ppl": 4.080043315887451 + }, + { + "layer": "layer.62.mlp.gate_proj", + "ppl": 4.080143928527832 + }, + { + "layer": "layer.62.mlp.up_proj", + "ppl": 4.080144882202148 + }, + { + "layer": "layer.62.mlp.down_proj", + "ppl": 4.079994201660156 + }, + { + "layer": "layer.63.self_attn.q_proj", + "ppl": 4.0801897048950195 + }, + { + "layer": "layer.63.self_attn.k_proj", + "ppl": 4.0801873207092285 + }, + { + "layer": "layer.63.self_attn.v_proj", + "ppl": 4.080052375793457 + }, + { + "layer": "layer.63.self_attn.o_proj", + "ppl": 4.080085277557373 + }, + { + "layer": "layer.63.mlp.gate_proj", + "ppl": 4.080100059509277 + }, + { + "layer": "layer.63.mlp.up_proj", + "ppl": 4.080202102661133 + }, + { + "layer": "layer.63.mlp.down_proj", + "ppl": 4.0798869132995605 + }, + { + "layer": "layer.64.self_attn.q_proj", + "ppl": 4.080153942108154 + }, + { + "layer": "layer.64.self_attn.k_proj", + "ppl": 4.080142021179199 + }, + { + "layer": "layer.64.self_attn.v_proj", + "ppl": 4.079995155334473 + }, + { + "layer": "layer.64.self_attn.o_proj", + "ppl": 4.08013391494751 + }, + { + "layer": "layer.64.mlp.gate_proj", + "ppl": 4.0800018310546875 + }, + { + "layer": "layer.64.mlp.up_proj", + "ppl": 4.0798163414001465 + }, + { + "layer": "layer.64.mlp.down_proj", + "ppl": 4.080000877380371 + }, + { + "layer": "layer.65.self_attn.q_proj", + "ppl": 4.080105781555176 + }, + { + "layer": "layer.65.self_attn.k_proj", + "ppl": 4.080204963684082 + }, + { + "layer": "layer.65.self_attn.v_proj", + "ppl": 4.080117225646973 + }, + { + "layer": "layer.65.self_attn.o_proj", + "ppl": 4.080322265625 + }, + { + "layer": "layer.65.mlp.gate_proj", + "ppl": 4.079955101013184 + }, + { + "layer": "layer.65.mlp.up_proj", + "ppl": 4.080273151397705 + }, + { + "layer": "layer.65.mlp.down_proj", + "ppl": 4.079992771148682 + }, + { + "layer": "layer.66.self_attn.q_proj", + "ppl": 4.080227375030518 + }, + { + "layer": "layer.66.self_attn.k_proj", + "ppl": 4.080203056335449 + }, + { + "layer": "layer.66.self_attn.v_proj", + "ppl": 4.080194473266602 + }, + { + "layer": "layer.66.self_attn.o_proj", + "ppl": 4.080245018005371 + }, + { + "layer": "layer.66.mlp.gate_proj", + "ppl": 4.080290794372559 + }, + { + "layer": "layer.66.mlp.up_proj", + "ppl": 4.080023288726807 + }, + { + "layer": "layer.66.mlp.down_proj", + "ppl": 4.080019474029541 + }, + { + "layer": "layer.67.self_attn.q_proj", + "ppl": 4.080165386199951 + }, + { + "layer": "layer.67.self_attn.k_proj", + "ppl": 4.080112457275391 + }, + { + "layer": "layer.67.self_attn.v_proj", + "ppl": 4.080210208892822 + }, + { + "layer": "layer.67.self_attn.o_proj", + "ppl": 4.080095291137695 + }, + { + "layer": "layer.67.mlp.gate_proj", + "ppl": 4.080211639404297 + }, + { + "layer": "layer.67.mlp.up_proj", + "ppl": 4.0801310539245605 + }, + { + "layer": "layer.67.mlp.down_proj", + "ppl": 4.079946517944336 + }, + { + "layer": "layer.68.self_attn.q_proj", + "ppl": 4.080198287963867 + }, + { + "layer": "layer.68.self_attn.k_proj", + "ppl": 4.080205917358398 + }, + { + "layer": "layer.68.self_attn.v_proj", + "ppl": 4.080112457275391 + }, + { + "layer": "layer.68.self_attn.o_proj", + "ppl": 4.080140113830566 + }, + { + "layer": "layer.68.mlp.gate_proj", + "ppl": 4.079991817474365 + }, + { + "layer": "layer.68.mlp.up_proj", + "ppl": 4.079906940460205 + }, + { + "layer": "layer.68.mlp.down_proj", + "ppl": 4.080118179321289 + }, + { + "layer": "layer.69.self_attn.q_proj", + "ppl": 4.080186367034912 + }, + { + "layer": "layer.69.self_attn.k_proj", + "ppl": 4.080134868621826 + }, + { + "layer": "layer.69.self_attn.v_proj", + "ppl": 4.080226421356201 + }, + { + "layer": "layer.69.self_attn.o_proj", + "ppl": 4.0801167488098145 + }, + { + "layer": "layer.69.mlp.gate_proj", + "ppl": 4.079852104187012 + }, + { + "layer": "layer.69.mlp.up_proj", + "ppl": 4.080157279968262 + }, + { + "layer": "layer.69.mlp.down_proj", + "ppl": 4.08013391494751 + }, + { + "layer": "layer.70.self_attn.q_proj", + "ppl": 4.080243110656738 + }, + { + "layer": "layer.70.self_attn.k_proj", + "ppl": 4.0802106857299805 + }, + { + "layer": "layer.70.self_attn.v_proj", + "ppl": 4.080097198486328 + }, + { + "layer": "layer.70.self_attn.o_proj", + "ppl": 4.080236911773682 + }, + { + "layer": "layer.70.mlp.gate_proj", + "ppl": 4.079891681671143 + }, + { + "layer": "layer.70.mlp.up_proj", + "ppl": 4.080127239227295 + }, + { + "layer": "layer.70.mlp.down_proj", + "ppl": 4.079904556274414 + }, + { + "layer": "layer.71.self_attn.q_proj", + "ppl": 4.080160140991211 + }, + { + "layer": "layer.71.self_attn.k_proj", + "ppl": 4.080166339874268 + }, + { + "layer": "layer.71.self_attn.v_proj", + "ppl": 4.080171585083008 + }, + { + "layer": "layer.71.self_attn.o_proj", + "ppl": 4.080153465270996 + }, + { + "layer": "layer.71.mlp.gate_proj", + "ppl": 4.079885482788086 + }, + { + "layer": "layer.71.mlp.up_proj", + "ppl": 4.080055236816406 + }, + { + "layer": "layer.71.mlp.down_proj", + "ppl": 4.079754829406738 + }, + { + "layer": "layer.72.self_attn.q_proj", + "ppl": 4.080183982849121 + }, + { + "layer": "layer.72.self_attn.k_proj", + "ppl": 4.080136775970459 + }, + { + "layer": "layer.72.self_attn.v_proj", + "ppl": 4.080068588256836 + }, + { + "layer": "layer.72.self_attn.o_proj", + "ppl": 4.079969882965088 + }, + { + "layer": "layer.72.mlp.gate_proj", + "ppl": 4.079777717590332 + }, + { + "layer": "layer.72.mlp.up_proj", + "ppl": 4.079853057861328 + }, + { + "layer": "layer.72.mlp.down_proj", + "ppl": 4.080050945281982 + }, + { + "layer": "layer.73.self_attn.q_proj", + "ppl": 4.080178737640381 + }, + { + "layer": "layer.73.self_attn.k_proj", + "ppl": 4.080170154571533 + }, + { + "layer": "layer.73.self_attn.v_proj", + "ppl": 4.080219745635986 + }, + { + "layer": "layer.73.self_attn.o_proj", + "ppl": 4.080199241638184 + }, + { + "layer": "layer.73.mlp.gate_proj", + "ppl": 4.080232620239258 + }, + { + "layer": "layer.73.mlp.up_proj", + "ppl": 4.080026149749756 + }, + { + "layer": "layer.73.mlp.down_proj", + "ppl": 4.0802459716796875 + }, + { + "layer": "layer.74.self_attn.q_proj", + "ppl": 4.080130100250244 + }, + { + "layer": "layer.74.self_attn.k_proj", + "ppl": 4.080204963684082 + }, + { + "layer": "layer.74.self_attn.v_proj", + "ppl": 4.080183982849121 + }, + { + "layer": "layer.74.self_attn.o_proj", + "ppl": 4.080167293548584 + }, + { + "layer": "layer.74.mlp.gate_proj", + "ppl": 4.079756259918213 + }, + { + "layer": "layer.74.mlp.up_proj", + "ppl": 4.080021858215332 + }, + { + "layer": "layer.74.mlp.down_proj", + "ppl": 4.080059051513672 + }, + { + "layer": "layer.75.self_attn.q_proj", + "ppl": 4.080219268798828 + }, + { + "layer": "layer.75.self_attn.k_proj", + "ppl": 4.080206871032715 + }, + { + "layer": "layer.75.self_attn.v_proj", + "ppl": 4.080183029174805 + }, + { + "layer": "layer.75.self_attn.o_proj", + "ppl": 4.080196380615234 + }, + { + "layer": "layer.75.mlp.gate_proj", + "ppl": 4.079739093780518 + }, + { + "layer": "layer.75.mlp.up_proj", + "ppl": 4.080113887786865 + }, + { + "layer": "layer.75.mlp.down_proj", + "ppl": 4.080115795135498 + }, + { + "layer": "layer.76.self_attn.q_proj", + "ppl": 4.080146789550781 + }, + { + "layer": "layer.76.self_attn.k_proj", + "ppl": 4.080163478851318 + }, + { + "layer": "layer.76.self_attn.v_proj", + "ppl": 4.0803542137146 + }, + { + "layer": "layer.76.self_attn.o_proj", + "ppl": 4.08017110824585 + }, + { + "layer": "layer.76.mlp.gate_proj", + "ppl": 4.0799784660339355 + }, + { + "layer": "layer.76.mlp.up_proj", + "ppl": 4.07989501953125 + }, + { + "layer": "layer.76.mlp.down_proj", + "ppl": 4.079855442047119 + }, + { + "layer": "layer.77.self_attn.q_proj", + "ppl": 4.080132007598877 + }, + { + "layer": "layer.77.self_attn.k_proj", + "ppl": 4.080397129058838 + }, + { + "layer": "layer.77.self_attn.v_proj", + "ppl": 4.080076217651367 + }, + { + "layer": "layer.77.self_attn.o_proj", + "ppl": 4.080029010772705 + }, + { + "layer": "layer.77.mlp.gate_proj", + "ppl": 4.07974910736084 + }, + { + "layer": "layer.77.mlp.up_proj", + "ppl": 4.079941272735596 + }, + { + "layer": "layer.77.mlp.down_proj", + "ppl": 4.079982280731201 + }, + { + "layer": "layer.78.self_attn.q_proj", + "ppl": 4.080216884613037 + }, + { + "layer": "layer.78.self_attn.k_proj", + "ppl": 4.080145835876465 + }, + { + "layer": "layer.78.self_attn.v_proj", + "ppl": 4.080267906188965 + }, + { + "layer": "layer.78.self_attn.o_proj", + "ppl": 4.080286502838135 + }, + { + "layer": "layer.78.mlp.gate_proj", + "ppl": 4.07962703704834 + }, + { + "layer": "layer.78.mlp.up_proj", + "ppl": 4.079881191253662 + }, + { + "layer": "layer.78.mlp.down_proj", + "ppl": 4.079721450805664 + }, + { + "layer": "layer.79.self_attn.q_proj", + "ppl": 4.080110549926758 + }, + { + "layer": "layer.79.self_attn.k_proj", + "ppl": 4.080165386199951 + }, + { + "layer": "layer.79.self_attn.v_proj", + "ppl": 4.080108165740967 + }, + { + "layer": "layer.79.self_attn.o_proj", + "ppl": 4.08017635345459 + }, + { + "layer": "layer.79.mlp.gate_proj", + "ppl": 4.079257011413574 + }, + { + "layer": "layer.79.mlp.up_proj", + "ppl": 4.079193115234375 + }, + { + "layer": "layer.79.mlp.down_proj", + "ppl": 4.07936429977417 + } + ] + }, + { + "bit": 11, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 4.060956954956055 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 4.061032295227051 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 4.061013698577881 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 4.060910701751709 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 4.061017990112305 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 4.060939311981201 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 4.06093168258667 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 4.061027526855469 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 4.060955047607422 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 4.061052322387695 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 4.060978412628174 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 4.060939311981201 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 4.061062812805176 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 4.060833930969238 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 4.060998439788818 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 4.061009407043457 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 4.061004638671875 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 4.061071395874023 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 4.060969829559326 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 4.060970783233643 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 4.061027526855469 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 4.060996055603027 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 4.060887336730957 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 4.060928821563721 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 4.060921669006348 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 4.0609307289123535 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 4.060964107513428 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 4.0609965324401855 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 4.061009407043457 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 4.061068534851074 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 4.060957908630371 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 4.060903072357178 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 4.061002731323242 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 4.06095027923584 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 4.060959815979004 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 4.06101131439209 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 4.061033248901367 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 4.060929775238037 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 4.061038494110107 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 4.0609846115112305 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 4.060951232910156 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 4.061023712158203 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 4.061041831970215 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 4.06097412109375 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 4.060996055603027 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 4.061065196990967 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 4.061035633087158 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 4.060959339141846 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 4.06095552444458 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 4.060886859893799 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 4.061017036437988 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 4.060988903045654 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 4.060939311981201 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 4.061074256896973 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 4.060884952545166 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 4.061037540435791 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 4.060925006866455 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 4.060969829559326 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 4.060927867889404 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 4.060955047607422 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 4.0610833168029785 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 4.060987949371338 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 4.061095714569092 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 4.060929775238037 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 4.060884475708008 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 4.060997486114502 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 4.060906410217285 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 4.060983657836914 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 4.061040878295898 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 4.061081409454346 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 4.061018943786621 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 4.061050891876221 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 4.061100006103516 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 4.061019420623779 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 4.061051845550537 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 4.060922145843506 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 4.060916423797607 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 4.061025619506836 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 4.060970783233643 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 4.0609283447265625 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 4.060840129852295 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 4.061102390289307 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 4.060970783233643 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 4.060870170593262 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 4.060972213745117 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 4.060952186584473 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 4.061004638671875 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 4.060965538024902 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 4.061005115509033 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 4.060976505279541 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 4.0609846115112305 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 4.061066150665283 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 4.060986518859863 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 4.060888290405273 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 4.060940265655518 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 4.060941696166992 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 4.061044216156006 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 4.061010360717773 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 4.06095552444458 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 4.060988903045654 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 4.061079025268555 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 4.060893535614014 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 4.061110019683838 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 4.061008930206299 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 4.061051845550537 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 4.060988903045654 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 4.061031341552734 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 4.061009407043457 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 4.060914039611816 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 4.060977458953857 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 4.061003684997559 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 4.061066150665283 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 4.0610151290893555 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 4.0611443519592285 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 4.06110143661499 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 4.060967922210693 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 4.061010360717773 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 4.060953140258789 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 4.0610527992248535 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 4.060956001281738 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 4.061017036437988 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 4.061027526855469 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 4.061079025268555 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 4.060919284820557 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 4.0608811378479 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 4.061003684997559 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 4.0608367919921875 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 4.060967445373535 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 4.060981273651123 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 4.061111927032471 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 4.060996055603027 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 4.060937404632568 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 4.061028957366943 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 4.060985088348389 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 4.060970783233643 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 4.061055660247803 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 4.060966968536377 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 4.061094760894775 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 4.061013221740723 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 4.060801029205322 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 4.060947418212891 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 4.0610151290893555 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 4.060939311981201 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 4.060988903045654 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 4.061020851135254 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 4.061009407043457 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 4.061033248901367 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 4.060967445373535 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 4.060887813568115 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 4.060957908630371 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 4.060918807983398 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 4.060919284820557 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 4.061012268066406 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 4.061026573181152 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 4.060980796813965 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 4.0610432624816895 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 4.060908794403076 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 4.060907363891602 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 4.060851573944092 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 4.061000347137451 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 4.060962677001953 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 4.061003684997559 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 4.060861587524414 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 4.061059474945068 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 4.060878276824951 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 4.060932636260986 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 4.061020851135254 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 4.0609636306762695 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 4.060945987701416 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 4.061028003692627 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 4.0610737800598145 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 4.0608930587768555 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 4.060955047607422 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 4.060901165008545 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 4.060866832733154 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 4.061004638671875 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 4.060895919799805 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 4.060963153839111 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 4.060973167419434 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 4.060970783233643 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 4.061158180236816 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 4.060930252075195 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 4.060927867889404 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 4.060981273651123 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 4.061039924621582 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 4.061063766479492 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 4.0610432624816895 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 4.060737609863281 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 4.060670852661133 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 4.06093692779541 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 4.061008930206299 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 4.060965538024902 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 4.060986042022705 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 4.0610432624816895 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 4.061097621917725 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 4.060977458953857 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 4.061001777648926 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 4.060965538024902 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 4.0610504150390625 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 4.061026096343994 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 4.060912132263184 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 4.061038970947266 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 4.060934543609619 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 4.061035633087158 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 4.060990333557129 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 4.061058521270752 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 4.061116695404053 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 4.060874938964844 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 4.060976505279541 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 4.060824394226074 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 4.060883522033691 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 4.060948371887207 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 4.060977458953857 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 4.060994625091553 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 4.0610809326171875 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 4.061001777648926 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 4.061058521270752 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 4.060935974121094 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 4.060915470123291 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 4.060935974121094 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 4.0611066818237305 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 4.060952186584473 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 4.060908794403076 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 4.061004638671875 + }, + { + "layer": "layer.32.self_attn.q_proj", + "ppl": 4.06099271774292 + }, + { + "layer": "layer.32.self_attn.k_proj", + "ppl": 4.061071872711182 + }, + { + "layer": "layer.32.self_attn.v_proj", + "ppl": 4.061040878295898 + }, + { + "layer": "layer.32.self_attn.o_proj", + "ppl": 4.060964584350586 + }, + { + "layer": "layer.32.mlp.gate_proj", + "ppl": 4.06083869934082 + }, + { + "layer": "layer.32.mlp.up_proj", + "ppl": 4.060887336730957 + }, + { + "layer": "layer.32.mlp.down_proj", + "ppl": 4.06093168258667 + }, + { + "layer": "layer.33.self_attn.q_proj", + "ppl": 4.060990810394287 + }, + { + "layer": "layer.33.self_attn.k_proj", + "ppl": 4.0609517097473145 + }, + { + "layer": "layer.33.self_attn.v_proj", + "ppl": 4.06087589263916 + }, + { + "layer": "layer.33.self_attn.o_proj", + "ppl": 4.0609636306762695 + }, + { + "layer": "layer.33.mlp.gate_proj", + "ppl": 4.061108589172363 + }, + { + "layer": "layer.33.mlp.up_proj", + "ppl": 4.060866832733154 + }, + { + "layer": "layer.33.mlp.down_proj", + "ppl": 4.0607590675354 + }, + { + "layer": "layer.34.self_attn.q_proj", + "ppl": 4.060863971710205 + }, + { + "layer": "layer.34.self_attn.k_proj", + "ppl": 4.060938358306885 + }, + { + "layer": "layer.34.self_attn.v_proj", + "ppl": 4.060920238494873 + }, + { + "layer": "layer.34.self_attn.o_proj", + "ppl": 4.060998439788818 + }, + { + "layer": "layer.34.mlp.gate_proj", + "ppl": 4.06095027923584 + }, + { + "layer": "layer.34.mlp.up_proj", + "ppl": 4.060939311981201 + }, + { + "layer": "layer.34.mlp.down_proj", + "ppl": 4.061056137084961 + }, + { + "layer": "layer.35.self_attn.q_proj", + "ppl": 4.061044216156006 + }, + { + "layer": "layer.35.self_attn.k_proj", + "ppl": 4.06093692779541 + }, + { + "layer": "layer.35.self_attn.v_proj", + "ppl": 4.060985565185547 + }, + { + "layer": "layer.35.self_attn.o_proj", + "ppl": 4.060949325561523 + }, + { + "layer": "layer.35.mlp.gate_proj", + "ppl": 4.060846328735352 + }, + { + "layer": "layer.35.mlp.up_proj", + "ppl": 4.060948371887207 + }, + { + "layer": "layer.35.mlp.down_proj", + "ppl": 4.061002731323242 + }, + { + "layer": "layer.36.self_attn.q_proj", + "ppl": 4.060944557189941 + }, + { + "layer": "layer.36.self_attn.k_proj", + "ppl": 4.06091833114624 + }, + { + "layer": "layer.36.self_attn.v_proj", + "ppl": 4.06091833114624 + }, + { + "layer": "layer.36.self_attn.o_proj", + "ppl": 4.060967922210693 + }, + { + "layer": "layer.36.mlp.gate_proj", + "ppl": 4.0608696937561035 + }, + { + "layer": "layer.36.mlp.up_proj", + "ppl": 4.060713768005371 + }, + { + "layer": "layer.36.mlp.down_proj", + "ppl": 4.060839653015137 + }, + { + "layer": "layer.37.self_attn.q_proj", + "ppl": 4.060967445373535 + }, + { + "layer": "layer.37.self_attn.k_proj", + "ppl": 4.061127662658691 + }, + { + "layer": "layer.37.self_attn.v_proj", + "ppl": 4.060906410217285 + }, + { + "layer": "layer.37.self_attn.o_proj", + "ppl": 4.061060428619385 + }, + { + "layer": "layer.37.mlp.gate_proj", + "ppl": 4.06089973449707 + }, + { + "layer": "layer.37.mlp.up_proj", + "ppl": 4.060811519622803 + }, + { + "layer": "layer.37.mlp.down_proj", + "ppl": 4.060709476470947 + }, + { + "layer": "layer.38.self_attn.q_proj", + "ppl": 4.060976505279541 + }, + { + "layer": "layer.38.self_attn.k_proj", + "ppl": 4.06099271774292 + }, + { + "layer": "layer.38.self_attn.v_proj", + "ppl": 4.060978412628174 + }, + { + "layer": "layer.38.self_attn.o_proj", + "ppl": 4.060974597930908 + }, + { + "layer": "layer.38.mlp.gate_proj", + "ppl": 4.061115741729736 + }, + { + "layer": "layer.38.mlp.up_proj", + "ppl": 4.060802936553955 + }, + { + "layer": "layer.38.mlp.down_proj", + "ppl": 4.060988903045654 + }, + { + "layer": "layer.39.self_attn.q_proj", + "ppl": 4.061019420623779 + }, + { + "layer": "layer.39.self_attn.k_proj", + "ppl": 4.060925483703613 + }, + { + "layer": "layer.39.self_attn.v_proj", + "ppl": 4.061041831970215 + }, + { + "layer": "layer.39.self_attn.o_proj", + "ppl": 4.060904502868652 + }, + { + "layer": "layer.39.mlp.gate_proj", + "ppl": 4.061127662658691 + }, + { + "layer": "layer.39.mlp.up_proj", + "ppl": 4.060885906219482 + }, + { + "layer": "layer.39.mlp.down_proj", + "ppl": 4.060909748077393 + }, + { + "layer": "layer.40.self_attn.q_proj", + "ppl": 4.0609259605407715 + }, + { + "layer": "layer.40.self_attn.k_proj", + "ppl": 4.0609965324401855 + }, + { + "layer": "layer.40.self_attn.v_proj", + "ppl": 4.060917377471924 + }, + { + "layer": "layer.40.self_attn.o_proj", + "ppl": 4.060928821563721 + }, + { + "layer": "layer.40.mlp.gate_proj", + "ppl": 4.060650825500488 + }, + { + "layer": "layer.40.mlp.up_proj", + "ppl": 4.060903072357178 + }, + { + "layer": "layer.40.mlp.down_proj", + "ppl": 4.060761451721191 + }, + { + "layer": "layer.41.self_attn.q_proj", + "ppl": 4.061052322387695 + }, + { + "layer": "layer.41.self_attn.k_proj", + "ppl": 4.060952186584473 + }, + { + "layer": "layer.41.self_attn.v_proj", + "ppl": 4.061007022857666 + }, + { + "layer": "layer.41.self_attn.o_proj", + "ppl": 4.060986042022705 + }, + { + "layer": "layer.41.mlp.gate_proj", + "ppl": 4.0608978271484375 + }, + { + "layer": "layer.41.mlp.up_proj", + "ppl": 4.060901165008545 + }, + { + "layer": "layer.41.mlp.down_proj", + "ppl": 4.0609846115112305 + }, + { + "layer": "layer.42.self_attn.q_proj", + "ppl": 4.06110143661499 + }, + { + "layer": "layer.42.self_attn.k_proj", + "ppl": 4.060964107513428 + }, + { + "layer": "layer.42.self_attn.v_proj", + "ppl": 4.060935020446777 + }, + { + "layer": "layer.42.self_attn.o_proj", + "ppl": 4.060948371887207 + }, + { + "layer": "layer.42.mlp.gate_proj", + "ppl": 4.060959815979004 + }, + { + "layer": "layer.42.mlp.up_proj", + "ppl": 4.06102180480957 + }, + { + "layer": "layer.42.mlp.down_proj", + "ppl": 4.061221122741699 + }, + { + "layer": "layer.43.self_attn.q_proj", + "ppl": 4.060966491699219 + }, + { + "layer": "layer.43.self_attn.k_proj", + "ppl": 4.060985565185547 + }, + { + "layer": "layer.43.self_attn.v_proj", + "ppl": 4.060880184173584 + }, + { + "layer": "layer.43.self_attn.o_proj", + "ppl": 4.061010837554932 + }, + { + "layer": "layer.43.mlp.gate_proj", + "ppl": 4.061343193054199 + }, + { + "layer": "layer.43.mlp.up_proj", + "ppl": 4.060829162597656 + }, + { + "layer": "layer.43.mlp.down_proj", + "ppl": 4.061039924621582 + }, + { + "layer": "layer.44.self_attn.q_proj", + "ppl": 4.061116695404053 + }, + { + "layer": "layer.44.self_attn.k_proj", + "ppl": 4.061000347137451 + }, + { + "layer": "layer.44.self_attn.v_proj", + "ppl": 4.060983180999756 + }, + { + "layer": "layer.44.self_attn.o_proj", + "ppl": 4.061048984527588 + }, + { + "layer": "layer.44.mlp.gate_proj", + "ppl": 4.061137676239014 + }, + { + "layer": "layer.44.mlp.up_proj", + "ppl": 4.060959339141846 + }, + { + "layer": "layer.44.mlp.down_proj", + "ppl": 4.061372756958008 + }, + { + "layer": "layer.45.self_attn.q_proj", + "ppl": 4.061099052429199 + }, + { + "layer": "layer.45.self_attn.k_proj", + "ppl": 4.060998439788818 + }, + { + "layer": "layer.45.self_attn.v_proj", + "ppl": 4.061035633087158 + }, + { + "layer": "layer.45.self_attn.o_proj", + "ppl": 4.061010360717773 + }, + { + "layer": "layer.45.mlp.gate_proj", + "ppl": 4.061152935028076 + }, + { + "layer": "layer.45.mlp.up_proj", + "ppl": 4.060920715332031 + }, + { + "layer": "layer.45.mlp.down_proj", + "ppl": 4.061302661895752 + }, + { + "layer": "layer.46.self_attn.q_proj", + "ppl": 4.061031341552734 + }, + { + "layer": "layer.46.self_attn.k_proj", + "ppl": 4.061096668243408 + }, + { + "layer": "layer.46.self_attn.v_proj", + "ppl": 4.060906410217285 + }, + { + "layer": "layer.46.self_attn.o_proj", + "ppl": 4.061003684997559 + }, + { + "layer": "layer.46.mlp.gate_proj", + "ppl": 4.060872554779053 + }, + { + "layer": "layer.46.mlp.up_proj", + "ppl": 4.060725688934326 + }, + { + "layer": "layer.46.mlp.down_proj", + "ppl": 4.061163425445557 + }, + { + "layer": "layer.47.self_attn.q_proj", + "ppl": 4.060978412628174 + }, + { + "layer": "layer.47.self_attn.k_proj", + "ppl": 4.0610575675964355 + }, + { + "layer": "layer.47.self_attn.v_proj", + "ppl": 4.061004638671875 + }, + { + "layer": "layer.47.self_attn.o_proj", + "ppl": 4.061123371124268 + }, + { + "layer": "layer.47.mlp.gate_proj", + "ppl": 4.061121940612793 + }, + { + "layer": "layer.47.mlp.up_proj", + "ppl": 4.060876846313477 + }, + { + "layer": "layer.47.mlp.down_proj", + "ppl": 4.061178207397461 + }, + { + "layer": "layer.48.self_attn.q_proj", + "ppl": 4.061120510101318 + }, + { + "layer": "layer.48.self_attn.k_proj", + "ppl": 4.061042785644531 + }, + { + "layer": "layer.48.self_attn.v_proj", + "ppl": 4.061001777648926 + }, + { + "layer": "layer.48.self_attn.o_proj", + "ppl": 4.061014175415039 + }, + { + "layer": "layer.48.mlp.gate_proj", + "ppl": 4.060878276824951 + }, + { + "layer": "layer.48.mlp.up_proj", + "ppl": 4.0608320236206055 + }, + { + "layer": "layer.48.mlp.down_proj", + "ppl": 4.060961723327637 + }, + { + "layer": "layer.49.self_attn.q_proj", + "ppl": 4.061079025268555 + }, + { + "layer": "layer.49.self_attn.k_proj", + "ppl": 4.06088924407959 + }, + { + "layer": "layer.49.self_attn.v_proj", + "ppl": 4.060877799987793 + }, + { + "layer": "layer.49.self_attn.o_proj", + "ppl": 4.061196327209473 + }, + { + "layer": "layer.49.mlp.gate_proj", + "ppl": 4.060986518859863 + }, + { + "layer": "layer.49.mlp.up_proj", + "ppl": 4.06100606918335 + }, + { + "layer": "layer.49.mlp.down_proj", + "ppl": 4.060992240905762 + }, + { + "layer": "layer.50.self_attn.q_proj", + "ppl": 4.061079025268555 + }, + { + "layer": "layer.50.self_attn.k_proj", + "ppl": 4.061009407043457 + }, + { + "layer": "layer.50.self_attn.v_proj", + "ppl": 4.061094760894775 + }, + { + "layer": "layer.50.self_attn.o_proj", + "ppl": 4.060987949371338 + }, + { + "layer": "layer.50.mlp.gate_proj", + "ppl": 4.060991287231445 + }, + { + "layer": "layer.50.mlp.up_proj", + "ppl": 4.060909748077393 + }, + { + "layer": "layer.50.mlp.down_proj", + "ppl": 4.061185359954834 + }, + { + "layer": "layer.51.self_attn.q_proj", + "ppl": 4.060899257659912 + }, + { + "layer": "layer.51.self_attn.k_proj", + "ppl": 4.060856342315674 + }, + { + "layer": "layer.51.self_attn.v_proj", + "ppl": 4.061161041259766 + }, + { + "layer": "layer.51.self_attn.o_proj", + "ppl": 4.061184883117676 + }, + { + "layer": "layer.51.mlp.gate_proj", + "ppl": 4.060948371887207 + }, + { + "layer": "layer.51.mlp.up_proj", + "ppl": 4.06083345413208 + }, + { + "layer": "layer.51.mlp.down_proj", + "ppl": 4.060728549957275 + }, + { + "layer": "layer.52.self_attn.q_proj", + "ppl": 4.061034202575684 + }, + { + "layer": "layer.52.self_attn.k_proj", + "ppl": 4.060952186584473 + }, + { + "layer": "layer.52.self_attn.v_proj", + "ppl": 4.061050891876221 + }, + { + "layer": "layer.52.self_attn.o_proj", + "ppl": 4.060825347900391 + }, + { + "layer": "layer.52.mlp.gate_proj", + "ppl": 4.060890197753906 + }, + { + "layer": "layer.52.mlp.up_proj", + "ppl": 4.060916423797607 + }, + { + "layer": "layer.52.mlp.down_proj", + "ppl": 4.060761451721191 + }, + { + "layer": "layer.53.self_attn.q_proj", + "ppl": 4.0610671043396 + }, + { + "layer": "layer.53.self_attn.k_proj", + "ppl": 4.061014175415039 + }, + { + "layer": "layer.53.self_attn.v_proj", + "ppl": 4.061106204986572 + }, + { + "layer": "layer.53.self_attn.o_proj", + "ppl": 4.061035633087158 + }, + { + "layer": "layer.53.mlp.gate_proj", + "ppl": 4.060886383056641 + }, + { + "layer": "layer.53.mlp.up_proj", + "ppl": 4.060966491699219 + }, + { + "layer": "layer.53.mlp.down_proj", + "ppl": 4.061045169830322 + }, + { + "layer": "layer.54.self_attn.q_proj", + "ppl": 4.060945987701416 + }, + { + "layer": "layer.54.self_attn.k_proj", + "ppl": 4.060904026031494 + }, + { + "layer": "layer.54.self_attn.v_proj", + "ppl": 4.060990333557129 + }, + { + "layer": "layer.54.self_attn.o_proj", + "ppl": 4.061135292053223 + }, + { + "layer": "layer.54.mlp.gate_proj", + "ppl": 4.061098098754883 + }, + { + "layer": "layer.54.mlp.up_proj", + "ppl": 4.0609822273254395 + }, + { + "layer": "layer.54.mlp.down_proj", + "ppl": 4.061060428619385 + }, + { + "layer": "layer.55.self_attn.q_proj", + "ppl": 4.061099052429199 + }, + { + "layer": "layer.55.self_attn.k_proj", + "ppl": 4.0610761642456055 + }, + { + "layer": "layer.55.self_attn.v_proj", + "ppl": 4.060929775238037 + }, + { + "layer": "layer.55.self_attn.o_proj", + "ppl": 4.060991287231445 + }, + { + "layer": "layer.55.mlp.gate_proj", + "ppl": 4.060644626617432 + }, + { + "layer": "layer.55.mlp.up_proj", + "ppl": 4.060882568359375 + }, + { + "layer": "layer.55.mlp.down_proj", + "ppl": 4.060940265655518 + }, + { + "layer": "layer.56.self_attn.q_proj", + "ppl": 4.061009407043457 + }, + { + "layer": "layer.56.self_attn.k_proj", + "ppl": 4.06091833114624 + }, + { + "layer": "layer.56.self_attn.v_proj", + "ppl": 4.0610151290893555 + }, + { + "layer": "layer.56.self_attn.o_proj", + "ppl": 4.060937404632568 + }, + { + "layer": "layer.56.mlp.gate_proj", + "ppl": 4.060934543609619 + }, + { + "layer": "layer.56.mlp.up_proj", + "ppl": 4.0608906745910645 + }, + { + "layer": "layer.56.mlp.down_proj", + "ppl": 4.061005115509033 + }, + { + "layer": "layer.57.self_attn.q_proj", + "ppl": 4.061100006103516 + }, + { + "layer": "layer.57.self_attn.k_proj", + "ppl": 4.061001777648926 + }, + { + "layer": "layer.57.self_attn.v_proj", + "ppl": 4.061094284057617 + }, + { + "layer": "layer.57.self_attn.o_proj", + "ppl": 4.06091833114624 + }, + { + "layer": "layer.57.mlp.gate_proj", + "ppl": 4.060916423797607 + }, + { + "layer": "layer.57.mlp.up_proj", + "ppl": 4.060966491699219 + }, + { + "layer": "layer.57.mlp.down_proj", + "ppl": 4.060909748077393 + }, + { + "layer": "layer.58.self_attn.q_proj", + "ppl": 4.061010360717773 + }, + { + "layer": "layer.58.self_attn.k_proj", + "ppl": 4.061130523681641 + }, + { + "layer": "layer.58.self_attn.v_proj", + "ppl": 4.0610575675964355 + }, + { + "layer": "layer.58.self_attn.o_proj", + "ppl": 4.060896396636963 + }, + { + "layer": "layer.58.mlp.gate_proj", + "ppl": 4.060784816741943 + }, + { + "layer": "layer.58.mlp.up_proj", + "ppl": 4.060784339904785 + }, + { + "layer": "layer.58.mlp.down_proj", + "ppl": 4.061132907867432 + }, + { + "layer": "layer.59.self_attn.q_proj", + "ppl": 4.061037540435791 + }, + { + "layer": "layer.59.self_attn.k_proj", + "ppl": 4.061038970947266 + }, + { + "layer": "layer.59.self_attn.v_proj", + "ppl": 4.061039924621582 + }, + { + "layer": "layer.59.self_attn.o_proj", + "ppl": 4.061124324798584 + }, + { + "layer": "layer.59.mlp.gate_proj", + "ppl": 4.060997486114502 + }, + { + "layer": "layer.59.mlp.up_proj", + "ppl": 4.060990810394287 + }, + { + "layer": "layer.59.mlp.down_proj", + "ppl": 4.060996055603027 + }, + { + "layer": "layer.60.self_attn.q_proj", + "ppl": 4.061044216156006 + }, + { + "layer": "layer.60.self_attn.k_proj", + "ppl": 4.060954570770264 + }, + { + "layer": "layer.60.self_attn.v_proj", + "ppl": 4.061061382293701 + }, + { + "layer": "layer.60.self_attn.o_proj", + "ppl": 4.060990333557129 + }, + { + "layer": "layer.60.mlp.gate_proj", + "ppl": 4.060996055603027 + }, + { + "layer": "layer.60.mlp.up_proj", + "ppl": 4.0611772537231445 + }, + { + "layer": "layer.60.mlp.down_proj", + "ppl": 4.060969829559326 + }, + { + "layer": "layer.61.self_attn.q_proj", + "ppl": 4.061001777648926 + }, + { + "layer": "layer.61.self_attn.k_proj", + "ppl": 4.060975551605225 + }, + { + "layer": "layer.61.self_attn.v_proj", + "ppl": 4.061046123504639 + }, + { + "layer": "layer.61.self_attn.o_proj", + "ppl": 4.06101655960083 + }, + { + "layer": "layer.61.mlp.gate_proj", + "ppl": 4.061025619506836 + }, + { + "layer": "layer.61.mlp.up_proj", + "ppl": 4.060890197753906 + }, + { + "layer": "layer.61.mlp.down_proj", + "ppl": 4.061136245727539 + }, + { + "layer": "layer.62.self_attn.q_proj", + "ppl": 4.060966491699219 + }, + { + "layer": "layer.62.self_attn.k_proj", + "ppl": 4.061000823974609 + }, + { + "layer": "layer.62.self_attn.v_proj", + "ppl": 4.060950756072998 + }, + { + "layer": "layer.62.self_attn.o_proj", + "ppl": 4.060891151428223 + }, + { + "layer": "layer.62.mlp.gate_proj", + "ppl": 4.0608954429626465 + }, + { + "layer": "layer.62.mlp.up_proj", + "ppl": 4.061028957366943 + }, + { + "layer": "layer.62.mlp.down_proj", + "ppl": 4.061062812805176 + }, + { + "layer": "layer.63.self_attn.q_proj", + "ppl": 4.061012268066406 + }, + { + "layer": "layer.63.self_attn.k_proj", + "ppl": 4.061075210571289 + }, + { + "layer": "layer.63.self_attn.v_proj", + "ppl": 4.060933589935303 + }, + { + "layer": "layer.63.self_attn.o_proj", + "ppl": 4.061133861541748 + }, + { + "layer": "layer.63.mlp.gate_proj", + "ppl": 4.060905456542969 + }, + { + "layer": "layer.63.mlp.up_proj", + "ppl": 4.061141014099121 + }, + { + "layer": "layer.63.mlp.down_proj", + "ppl": 4.061102390289307 + }, + { + "layer": "layer.64.self_attn.q_proj", + "ppl": 4.06095027923584 + }, + { + "layer": "layer.64.self_attn.k_proj", + "ppl": 4.060959815979004 + }, + { + "layer": "layer.64.self_attn.v_proj", + "ppl": 4.0611371994018555 + }, + { + "layer": "layer.64.self_attn.o_proj", + "ppl": 4.060874938964844 + }, + { + "layer": "layer.64.mlp.gate_proj", + "ppl": 4.060942649841309 + }, + { + "layer": "layer.64.mlp.up_proj", + "ppl": 4.061256408691406 + }, + { + "layer": "layer.64.mlp.down_proj", + "ppl": 4.060855865478516 + }, + { + "layer": "layer.65.self_attn.q_proj", + "ppl": 4.061074256896973 + }, + { + "layer": "layer.65.self_attn.k_proj", + "ppl": 4.061049938201904 + }, + { + "layer": "layer.65.self_attn.v_proj", + "ppl": 4.061075687408447 + }, + { + "layer": "layer.65.self_attn.o_proj", + "ppl": 4.060927867889404 + }, + { + "layer": "layer.65.mlp.gate_proj", + "ppl": 4.06077766418457 + }, + { + "layer": "layer.65.mlp.up_proj", + "ppl": 4.060845375061035 + }, + { + "layer": "layer.65.mlp.down_proj", + "ppl": 4.060869216918945 + }, + { + "layer": "layer.66.self_attn.q_proj", + "ppl": 4.0611042976379395 + }, + { + "layer": "layer.66.self_attn.k_proj", + "ppl": 4.0610456466674805 + }, + { + "layer": "layer.66.self_attn.v_proj", + "ppl": 4.061010837554932 + }, + { + "layer": "layer.66.self_attn.o_proj", + "ppl": 4.060917854309082 + }, + { + "layer": "layer.66.mlp.gate_proj", + "ppl": 4.060632705688477 + }, + { + "layer": "layer.66.mlp.up_proj", + "ppl": 4.060928821563721 + }, + { + "layer": "layer.66.mlp.down_proj", + "ppl": 4.060823440551758 + }, + { + "layer": "layer.67.self_attn.q_proj", + "ppl": 4.060988903045654 + }, + { + "layer": "layer.67.self_attn.k_proj", + "ppl": 4.060977935791016 + }, + { + "layer": "layer.67.self_attn.v_proj", + "ppl": 4.0610198974609375 + }, + { + "layer": "layer.67.self_attn.o_proj", + "ppl": 4.061098098754883 + }, + { + "layer": "layer.67.mlp.gate_proj", + "ppl": 4.060788154602051 + }, + { + "layer": "layer.67.mlp.up_proj", + "ppl": 4.060987949371338 + }, + { + "layer": "layer.67.mlp.down_proj", + "ppl": 4.061027526855469 + }, + { + "layer": "layer.68.self_attn.q_proj", + "ppl": 4.061091423034668 + }, + { + "layer": "layer.68.self_attn.k_proj", + "ppl": 4.060988903045654 + }, + { + "layer": "layer.68.self_attn.v_proj", + "ppl": 4.060966491699219 + }, + { + "layer": "layer.68.self_attn.o_proj", + "ppl": 4.060905456542969 + }, + { + "layer": "layer.68.mlp.gate_proj", + "ppl": 4.0606279373168945 + }, + { + "layer": "layer.68.mlp.up_proj", + "ppl": 4.061087608337402 + }, + { + "layer": "layer.68.mlp.down_proj", + "ppl": 4.060929775238037 + }, + { + "layer": "layer.69.self_attn.q_proj", + "ppl": 4.060951232910156 + }, + { + "layer": "layer.69.self_attn.k_proj", + "ppl": 4.061030864715576 + }, + { + "layer": "layer.69.self_attn.v_proj", + "ppl": 4.061000347137451 + }, + { + "layer": "layer.69.self_attn.o_proj", + "ppl": 4.061038970947266 + }, + { + "layer": "layer.69.mlp.gate_proj", + "ppl": 4.060903549194336 + }, + { + "layer": "layer.69.mlp.up_proj", + "ppl": 4.060887336730957 + }, + { + "layer": "layer.69.mlp.down_proj", + "ppl": 4.060928821563721 + }, + { + "layer": "layer.70.self_attn.q_proj", + "ppl": 4.06099271774292 + }, + { + "layer": "layer.70.self_attn.k_proj", + "ppl": 4.061004638671875 + }, + { + "layer": "layer.70.self_attn.v_proj", + "ppl": 4.06105375289917 + }, + { + "layer": "layer.70.self_attn.o_proj", + "ppl": 4.060996055603027 + }, + { + "layer": "layer.70.mlp.gate_proj", + "ppl": 4.0608344078063965 + }, + { + "layer": "layer.70.mlp.up_proj", + "ppl": 4.060887813568115 + }, + { + "layer": "layer.70.mlp.down_proj", + "ppl": 4.060986518859863 + }, + { + "layer": "layer.71.self_attn.q_proj", + "ppl": 4.061062812805176 + }, + { + "layer": "layer.71.self_attn.k_proj", + "ppl": 4.061048984527588 + }, + { + "layer": "layer.71.self_attn.v_proj", + "ppl": 4.0610671043396 + }, + { + "layer": "layer.71.self_attn.o_proj", + "ppl": 4.060967922210693 + }, + { + "layer": "layer.71.mlp.gate_proj", + "ppl": 4.06091833114624 + }, + { + "layer": "layer.71.mlp.up_proj", + "ppl": 4.060940265655518 + }, + { + "layer": "layer.71.mlp.down_proj", + "ppl": 4.061230182647705 + }, + { + "layer": "layer.72.self_attn.q_proj", + "ppl": 4.06100606918335 + }, + { + "layer": "layer.72.self_attn.k_proj", + "ppl": 4.060986042022705 + }, + { + "layer": "layer.72.self_attn.v_proj", + "ppl": 4.060971260070801 + }, + { + "layer": "layer.72.self_attn.o_proj", + "ppl": 4.061100006103516 + }, + { + "layer": "layer.72.mlp.gate_proj", + "ppl": 4.060971260070801 + }, + { + "layer": "layer.72.mlp.up_proj", + "ppl": 4.060998439788818 + }, + { + "layer": "layer.72.mlp.down_proj", + "ppl": 4.060882091522217 + }, + { + "layer": "layer.73.self_attn.q_proj", + "ppl": 4.061009407043457 + }, + { + "layer": "layer.73.self_attn.k_proj", + "ppl": 4.061024188995361 + }, + { + "layer": "layer.73.self_attn.v_proj", + "ppl": 4.061058521270752 + }, + { + "layer": "layer.73.self_attn.o_proj", + "ppl": 4.060950756072998 + }, + { + "layer": "layer.73.mlp.gate_proj", + "ppl": 4.060811519622803 + }, + { + "layer": "layer.73.mlp.up_proj", + "ppl": 4.060902118682861 + }, + { + "layer": "layer.73.mlp.down_proj", + "ppl": 4.060855388641357 + }, + { + "layer": "layer.74.self_attn.q_proj", + "ppl": 4.060999393463135 + }, + { + "layer": "layer.74.self_attn.k_proj", + "ppl": 4.060899257659912 + }, + { + "layer": "layer.74.self_attn.v_proj", + "ppl": 4.060942649841309 + }, + { + "layer": "layer.74.self_attn.o_proj", + "ppl": 4.060904502868652 + }, + { + "layer": "layer.74.mlp.gate_proj", + "ppl": 4.060845375061035 + }, + { + "layer": "layer.74.mlp.up_proj", + "ppl": 4.060776233673096 + }, + { + "layer": "layer.74.mlp.down_proj", + "ppl": 4.060940265655518 + }, + { + "layer": "layer.75.self_attn.q_proj", + "ppl": 4.061068058013916 + }, + { + "layer": "layer.75.self_attn.k_proj", + "ppl": 4.0610151290893555 + }, + { + "layer": "layer.75.self_attn.v_proj", + "ppl": 4.0608696937561035 + }, + { + "layer": "layer.75.self_attn.o_proj", + "ppl": 4.0609259605407715 + }, + { + "layer": "layer.75.mlp.gate_proj", + "ppl": 4.060821533203125 + }, + { + "layer": "layer.75.mlp.up_proj", + "ppl": 4.060731410980225 + }, + { + "layer": "layer.75.mlp.down_proj", + "ppl": 4.060877799987793 + }, + { + "layer": "layer.76.self_attn.q_proj", + "ppl": 4.060995101928711 + }, + { + "layer": "layer.76.self_attn.k_proj", + "ppl": 4.060999393463135 + }, + { + "layer": "layer.76.self_attn.v_proj", + "ppl": 4.060923099517822 + }, + { + "layer": "layer.76.self_attn.o_proj", + "ppl": 4.060976505279541 + }, + { + "layer": "layer.76.mlp.gate_proj", + "ppl": 4.060675621032715 + }, + { + "layer": "layer.76.mlp.up_proj", + "ppl": 4.060932636260986 + }, + { + "layer": "layer.76.mlp.down_proj", + "ppl": 4.061005115509033 + }, + { + "layer": "layer.77.self_attn.q_proj", + "ppl": 4.060951232910156 + }, + { + "layer": "layer.77.self_attn.k_proj", + "ppl": 4.061037540435791 + }, + { + "layer": "layer.77.self_attn.v_proj", + "ppl": 4.061042308807373 + }, + { + "layer": "layer.77.self_attn.o_proj", + "ppl": 4.061056137084961 + }, + { + "layer": "layer.77.mlp.gate_proj", + "ppl": 4.060635089874268 + }, + { + "layer": "layer.77.mlp.up_proj", + "ppl": 4.060745716094971 + }, + { + "layer": "layer.77.mlp.down_proj", + "ppl": 4.06089973449707 + }, + { + "layer": "layer.78.self_attn.q_proj", + "ppl": 4.0609130859375 + }, + { + "layer": "layer.78.self_attn.k_proj", + "ppl": 4.06105375289917 + }, + { + "layer": "layer.78.self_attn.v_proj", + "ppl": 4.060924530029297 + }, + { + "layer": "layer.78.self_attn.o_proj", + "ppl": 4.0609211921691895 + }, + { + "layer": "layer.78.mlp.gate_proj", + "ppl": 4.060694694519043 + }, + { + "layer": "layer.78.mlp.up_proj", + "ppl": 4.06079626083374 + }, + { + "layer": "layer.78.mlp.down_proj", + "ppl": 4.0606560707092285 + }, + { + "layer": "layer.79.self_attn.q_proj", + "ppl": 4.060917854309082 + }, + { + "layer": "layer.79.self_attn.k_proj", + "ppl": 4.0609869956970215 + }, + { + "layer": "layer.79.self_attn.v_proj", + "ppl": 4.061119079589844 + }, + { + "layer": "layer.79.self_attn.o_proj", + "ppl": 4.0610151290893555 + }, + { + "layer": "layer.79.mlp.gate_proj", + "ppl": 4.060502052307129 + }, + { + "layer": "layer.79.mlp.up_proj", + "ppl": 4.060367584228516 + }, + { + "layer": "layer.79.mlp.down_proj", + "ppl": 4.059967994689941 + } + ] + }, + { + "bit": 12, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 4.0438995361328125 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 4.043996810913086 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 4.043998718261719 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 4.044002532958984 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 4.0439958572387695 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 4.043962001800537 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 4.043944358825684 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 4.043909549713135 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 4.043972492218018 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 4.044013023376465 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 4.044075965881348 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 4.043966293334961 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 4.043900966644287 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 4.043937683105469 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 4.0439348220825195 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 4.0439300537109375 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 4.043992042541504 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 4.043944358825684 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 4.044012546539307 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 4.044022560119629 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 4.04388952255249 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 4.043976306915283 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 4.043911933898926 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 4.043954372406006 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 4.044007301330566 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 4.0439772605896 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 4.044034957885742 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 4.043983459472656 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 4.043990135192871 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 4.043956279754639 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 4.043911933898926 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 4.044004440307617 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 4.044028282165527 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 4.044003963470459 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 4.043862819671631 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 4.043980121612549 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 4.0439534187316895 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 4.044022560119629 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 4.04403018951416 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 4.044034957885742 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 4.043954849243164 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 4.044008255004883 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 4.044184684753418 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 4.044144153594971 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 4.0440673828125 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 4.043926239013672 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 4.04406213760376 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 4.044047832489014 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 4.043931007385254 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 4.043919563293457 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 4.043973445892334 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 4.044039249420166 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 4.044095039367676 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 4.044010162353516 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 4.044032573699951 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 4.044010162353516 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 4.043870449066162 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 4.044047832489014 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 4.043944358825684 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 4.043924808502197 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 4.0438923835754395 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 4.044018745422363 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 4.043998718261719 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 4.044100761413574 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 4.044013977050781 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 4.04396915435791 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 4.0439910888671875 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 4.043776512145996 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 4.043999671936035 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 4.043944358825684 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 4.0440673828125 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 4.044014930725098 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 4.043957233428955 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 4.043973922729492 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 4.043973922729492 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 4.044034957885742 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 4.043965816497803 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 4.043915748596191 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 4.043932914733887 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 4.043946743011475 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 4.043887138366699 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 4.043880939483643 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 4.043891906738281 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 4.043971061706543 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 4.043938159942627 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 4.043948650360107 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 4.044057369232178 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 4.043954849243164 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 4.044035911560059 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 4.043998718261719 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 4.044013977050781 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 4.044041156768799 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 4.044009685516357 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 4.044099807739258 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 4.043971061706543 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 4.043965816497803 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 4.0439581871032715 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 4.043972969055176 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 4.043936729431152 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 4.043941020965576 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 4.043862342834473 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 4.044010639190674 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 4.044020652770996 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 4.044043064117432 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 4.044017314910889 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 4.04396390914917 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 4.044000625610352 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 4.0439229011535645 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 4.043986797332764 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 4.043894290924072 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 4.043909549713135 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 4.043948650360107 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 4.044020652770996 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 4.043941020965576 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 4.044102191925049 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 4.044084548950195 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 4.043937683105469 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 4.04408073425293 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 4.0439581871032715 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 4.043980121612549 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 4.043867111206055 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 4.044068336486816 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 4.043942451477051 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 4.0439019203186035 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 4.044109344482422 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 4.044095039367676 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 4.044074058532715 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 4.044009208679199 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 4.043951988220215 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 4.043887615203857 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 4.044151782989502 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 4.044114112854004 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 4.04408597946167 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 4.043975830078125 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 4.043999195098877 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 4.044106960296631 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 4.044003963470459 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 4.043883800506592 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 4.043951988220215 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 4.0441508293151855 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 4.0438385009765625 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 4.044023036956787 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 4.043992519378662 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 4.043923377990723 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 4.043875217437744 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 4.043984413146973 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 4.0439252853393555 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 4.043872356414795 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 4.044000625610352 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 4.044039249420166 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 4.044036865234375 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 4.04401969909668 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 4.043981552124023 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 4.0439677238464355 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 4.044000625610352 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 4.043913841247559 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 4.044073104858398 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 4.044012546539307 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 4.044017314910889 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 4.043930530548096 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 4.044004440307617 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 4.043942451477051 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 4.04401969909668 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 4.043981075286865 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 4.044032573699951 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 4.04406213760376 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 4.044075012207031 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 4.0439453125 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 4.043974876403809 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 4.044010162353516 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 4.043972492218018 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 4.044034481048584 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 4.043978214263916 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 4.043917655944824 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 4.044095993041992 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 4.043962478637695 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 4.043948650360107 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 4.043994426727295 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 4.04390287399292 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 4.043949604034424 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 4.043797016143799 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 4.0439252853393555 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 4.043928623199463 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 4.044022560119629 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 4.0439534187316895 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 4.043859481811523 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 4.0440168380737305 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 4.0440545082092285 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 4.044081687927246 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 4.043928146362305 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 4.043891906738281 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 4.043995380401611 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 4.043909549713135 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 4.043869495391846 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 4.043829917907715 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 4.043986797332764 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 4.04393196105957 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 4.0439677238464355 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 4.043907165527344 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 4.04395866394043 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 4.0440144538879395 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 4.043941974639893 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 4.044013977050781 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 4.043912887573242 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 4.043903827667236 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 4.043925762176514 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 4.043967247009277 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 4.043922424316406 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 4.0440263748168945 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 4.043910980224609 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 4.043998718261719 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 4.043941020965576 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 4.043981552124023 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 4.043970108032227 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 4.043866157531738 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 4.043907165527344 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 4.043909549713135 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 4.043966770172119 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 4.043948650360107 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 4.044058322906494 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 4.043884754180908 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 4.044131755828857 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 4.044029235839844 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 4.043849468231201 + }, + { + "layer": "layer.32.self_attn.q_proj", + "ppl": 4.044000148773193 + }, + { + "layer": "layer.32.self_attn.k_proj", + "ppl": 4.0439348220825195 + }, + { + "layer": "layer.32.self_attn.v_proj", + "ppl": 4.043985843658447 + }, + { + "layer": "layer.32.self_attn.o_proj", + "ppl": 4.043918132781982 + }, + { + "layer": "layer.32.mlp.gate_proj", + "ppl": 4.044142246246338 + }, + { + "layer": "layer.32.mlp.up_proj", + "ppl": 4.04396915435791 + }, + { + "layer": "layer.32.mlp.down_proj", + "ppl": 4.043966293334961 + }, + { + "layer": "layer.33.self_attn.q_proj", + "ppl": 4.043950080871582 + }, + { + "layer": "layer.33.self_attn.k_proj", + "ppl": 4.044064044952393 + }, + { + "layer": "layer.33.self_attn.v_proj", + "ppl": 4.044032573699951 + }, + { + "layer": "layer.33.self_attn.o_proj", + "ppl": 4.043998718261719 + }, + { + "layer": "layer.33.mlp.gate_proj", + "ppl": 4.043871879577637 + }, + { + "layer": "layer.33.mlp.up_proj", + "ppl": 4.043970584869385 + }, + { + "layer": "layer.33.mlp.down_proj", + "ppl": 4.043936729431152 + }, + { + "layer": "layer.34.self_attn.q_proj", + "ppl": 4.043912410736084 + }, + { + "layer": "layer.34.self_attn.k_proj", + "ppl": 4.043976306915283 + }, + { + "layer": "layer.34.self_attn.v_proj", + "ppl": 4.043852806091309 + }, + { + "layer": "layer.34.self_attn.o_proj", + "ppl": 4.043832778930664 + }, + { + "layer": "layer.34.mlp.gate_proj", + "ppl": 4.043793201446533 + }, + { + "layer": "layer.34.mlp.up_proj", + "ppl": 4.0438385009765625 + }, + { + "layer": "layer.34.mlp.down_proj", + "ppl": 4.043811321258545 + }, + { + "layer": "layer.35.self_attn.q_proj", + "ppl": 4.043955326080322 + }, + { + "layer": "layer.35.self_attn.k_proj", + "ppl": 4.043931007385254 + }, + { + "layer": "layer.35.self_attn.v_proj", + "ppl": 4.043920516967773 + }, + { + "layer": "layer.35.self_attn.o_proj", + "ppl": 4.043979167938232 + }, + { + "layer": "layer.35.mlp.gate_proj", + "ppl": 4.0438432693481445 + }, + { + "layer": "layer.35.mlp.up_proj", + "ppl": 4.043895244598389 + }, + { + "layer": "layer.35.mlp.down_proj", + "ppl": 4.044033527374268 + }, + { + "layer": "layer.36.self_attn.q_proj", + "ppl": 4.04400634765625 + }, + { + "layer": "layer.36.self_attn.k_proj", + "ppl": 4.043968677520752 + }, + { + "layer": "layer.36.self_attn.v_proj", + "ppl": 4.044033527374268 + }, + { + "layer": "layer.36.self_attn.o_proj", + "ppl": 4.044024467468262 + }, + { + "layer": "layer.36.mlp.gate_proj", + "ppl": 4.043758392333984 + }, + { + "layer": "layer.36.mlp.up_proj", + "ppl": 4.044094085693359 + }, + { + "layer": "layer.36.mlp.down_proj", + "ppl": 4.043839454650879 + }, + { + "layer": "layer.37.self_attn.q_proj", + "ppl": 4.044003486633301 + }, + { + "layer": "layer.37.self_attn.k_proj", + "ppl": 4.043923377990723 + }, + { + "layer": "layer.37.self_attn.v_proj", + "ppl": 4.044018745422363 + }, + { + "layer": "layer.37.self_attn.o_proj", + "ppl": 4.043979167938232 + }, + { + "layer": "layer.37.mlp.gate_proj", + "ppl": 4.044030666351318 + }, + { + "layer": "layer.37.mlp.up_proj", + "ppl": 4.043973445892334 + }, + { + "layer": "layer.37.mlp.down_proj", + "ppl": 4.044153213500977 + }, + { + "layer": "layer.38.self_attn.q_proj", + "ppl": 4.043872833251953 + }, + { + "layer": "layer.38.self_attn.k_proj", + "ppl": 4.044071674346924 + }, + { + "layer": "layer.38.self_attn.v_proj", + "ppl": 4.043915271759033 + }, + { + "layer": "layer.38.self_attn.o_proj", + "ppl": 4.044029235839844 + }, + { + "layer": "layer.38.mlp.gate_proj", + "ppl": 4.043679714202881 + }, + { + "layer": "layer.38.mlp.up_proj", + "ppl": 4.043910503387451 + }, + { + "layer": "layer.38.mlp.down_proj", + "ppl": 4.043898582458496 + }, + { + "layer": "layer.39.self_attn.q_proj", + "ppl": 4.043933868408203 + }, + { + "layer": "layer.39.self_attn.k_proj", + "ppl": 4.043951988220215 + }, + { + "layer": "layer.39.self_attn.v_proj", + "ppl": 4.043939590454102 + }, + { + "layer": "layer.39.self_attn.o_proj", + "ppl": 4.044015884399414 + }, + { + "layer": "layer.39.mlp.gate_proj", + "ppl": 4.0438313484191895 + }, + { + "layer": "layer.39.mlp.up_proj", + "ppl": 4.043797969818115 + }, + { + "layer": "layer.39.mlp.down_proj", + "ppl": 4.0439958572387695 + }, + { + "layer": "layer.40.self_attn.q_proj", + "ppl": 4.04387903213501 + }, + { + "layer": "layer.40.self_attn.k_proj", + "ppl": 4.043964862823486 + }, + { + "layer": "layer.40.self_attn.v_proj", + "ppl": 4.043949604034424 + }, + { + "layer": "layer.40.self_attn.o_proj", + "ppl": 4.043793201446533 + }, + { + "layer": "layer.40.mlp.gate_proj", + "ppl": 4.044032573699951 + }, + { + "layer": "layer.40.mlp.up_proj", + "ppl": 4.043980121612549 + }, + { + "layer": "layer.40.mlp.down_proj", + "ppl": 4.044117450714111 + }, + { + "layer": "layer.41.self_attn.q_proj", + "ppl": 4.043837547302246 + }, + { + "layer": "layer.41.self_attn.k_proj", + "ppl": 4.043874740600586 + }, + { + "layer": "layer.41.self_attn.v_proj", + "ppl": 4.043956279754639 + }, + { + "layer": "layer.41.self_attn.o_proj", + "ppl": 4.043893337249756 + }, + { + "layer": "layer.41.mlp.gate_proj", + "ppl": 4.0440592765808105 + }, + { + "layer": "layer.41.mlp.up_proj", + "ppl": 4.043944358825684 + }, + { + "layer": "layer.41.mlp.down_proj", + "ppl": 4.043822765350342 + }, + { + "layer": "layer.42.self_attn.q_proj", + "ppl": 4.043847560882568 + }, + { + "layer": "layer.42.self_attn.k_proj", + "ppl": 4.044008731842041 + }, + { + "layer": "layer.42.self_attn.v_proj", + "ppl": 4.04403829574585 + }, + { + "layer": "layer.42.self_attn.o_proj", + "ppl": 4.043976306915283 + }, + { + "layer": "layer.42.mlp.gate_proj", + "ppl": 4.043884754180908 + }, + { + "layer": "layer.42.mlp.up_proj", + "ppl": 4.044071674346924 + }, + { + "layer": "layer.42.mlp.down_proj", + "ppl": 4.043790340423584 + }, + { + "layer": "layer.43.self_attn.q_proj", + "ppl": 4.043985843658447 + }, + { + "layer": "layer.43.self_attn.k_proj", + "ppl": 4.04393196105957 + }, + { + "layer": "layer.43.self_attn.v_proj", + "ppl": 4.0440592765808105 + }, + { + "layer": "layer.43.self_attn.o_proj", + "ppl": 4.043970584869385 + }, + { + "layer": "layer.43.mlp.gate_proj", + "ppl": 4.04392671585083 + }, + { + "layer": "layer.43.mlp.up_proj", + "ppl": 4.043827533721924 + }, + { + "layer": "layer.43.mlp.down_proj", + "ppl": 4.044031620025635 + }, + { + "layer": "layer.44.self_attn.q_proj", + "ppl": 4.0439300537109375 + }, + { + "layer": "layer.44.self_attn.k_proj", + "ppl": 4.043989181518555 + }, + { + "layer": "layer.44.self_attn.v_proj", + "ppl": 4.044055461883545 + }, + { + "layer": "layer.44.self_attn.o_proj", + "ppl": 4.043907165527344 + }, + { + "layer": "layer.44.mlp.gate_proj", + "ppl": 4.043827056884766 + }, + { + "layer": "layer.44.mlp.up_proj", + "ppl": 4.043997764587402 + }, + { + "layer": "layer.44.mlp.down_proj", + "ppl": 4.043737888336182 + }, + { + "layer": "layer.45.self_attn.q_proj", + "ppl": 4.0439910888671875 + }, + { + "layer": "layer.45.self_attn.k_proj", + "ppl": 4.043941974639893 + }, + { + "layer": "layer.45.self_attn.v_proj", + "ppl": 4.043936729431152 + }, + { + "layer": "layer.45.self_attn.o_proj", + "ppl": 4.043914794921875 + }, + { + "layer": "layer.45.mlp.gate_proj", + "ppl": 4.043796062469482 + }, + { + "layer": "layer.45.mlp.up_proj", + "ppl": 4.043978691101074 + }, + { + "layer": "layer.45.mlp.down_proj", + "ppl": 4.0437235832214355 + }, + { + "layer": "layer.46.self_attn.q_proj", + "ppl": 4.043998718261719 + }, + { + "layer": "layer.46.self_attn.k_proj", + "ppl": 4.0437445640563965 + }, + { + "layer": "layer.46.self_attn.v_proj", + "ppl": 4.0439453125 + }, + { + "layer": "layer.46.self_attn.o_proj", + "ppl": 4.043819427490234 + }, + { + "layer": "layer.46.mlp.gate_proj", + "ppl": 4.043950080871582 + }, + { + "layer": "layer.46.mlp.up_proj", + "ppl": 4.044095039367676 + }, + { + "layer": "layer.46.mlp.down_proj", + "ppl": 4.044130325317383 + }, + { + "layer": "layer.47.self_attn.q_proj", + "ppl": 4.043972015380859 + }, + { + "layer": "layer.47.self_attn.k_proj", + "ppl": 4.0439043045043945 + }, + { + "layer": "layer.47.self_attn.v_proj", + "ppl": 4.043975830078125 + }, + { + "layer": "layer.47.self_attn.o_proj", + "ppl": 4.043851852416992 + }, + { + "layer": "layer.47.mlp.gate_proj", + "ppl": 4.043847560882568 + }, + { + "layer": "layer.47.mlp.up_proj", + "ppl": 4.04402494430542 + }, + { + "layer": "layer.47.mlp.down_proj", + "ppl": 4.043933868408203 + }, + { + "layer": "layer.48.self_attn.q_proj", + "ppl": 4.04388952255249 + }, + { + "layer": "layer.48.self_attn.k_proj", + "ppl": 4.043926239013672 + }, + { + "layer": "layer.48.self_attn.v_proj", + "ppl": 4.043979167938232 + }, + { + "layer": "layer.48.self_attn.o_proj", + "ppl": 4.043983459472656 + }, + { + "layer": "layer.48.mlp.gate_proj", + "ppl": 4.043944358825684 + }, + { + "layer": "layer.48.mlp.up_proj", + "ppl": 4.04406213760376 + }, + { + "layer": "layer.48.mlp.down_proj", + "ppl": 4.043916702270508 + }, + { + "layer": "layer.49.self_attn.q_proj", + "ppl": 4.043964862823486 + }, + { + "layer": "layer.49.self_attn.k_proj", + "ppl": 4.043968677520752 + }, + { + "layer": "layer.49.self_attn.v_proj", + "ppl": 4.044058322906494 + }, + { + "layer": "layer.49.self_attn.o_proj", + "ppl": 4.043834209442139 + }, + { + "layer": "layer.49.mlp.gate_proj", + "ppl": 4.044046878814697 + }, + { + "layer": "layer.49.mlp.up_proj", + "ppl": 4.043856143951416 + }, + { + "layer": "layer.49.mlp.down_proj", + "ppl": 4.04374885559082 + }, + { + "layer": "layer.50.self_attn.q_proj", + "ppl": 4.043990135192871 + }, + { + "layer": "layer.50.self_attn.k_proj", + "ppl": 4.0439958572387695 + }, + { + "layer": "layer.50.self_attn.v_proj", + "ppl": 4.043920516967773 + }, + { + "layer": "layer.50.self_attn.o_proj", + "ppl": 4.043893814086914 + }, + { + "layer": "layer.50.mlp.gate_proj", + "ppl": 4.044100761413574 + }, + { + "layer": "layer.50.mlp.up_proj", + "ppl": 4.043924331665039 + }, + { + "layer": "layer.50.mlp.down_proj", + "ppl": 4.043729782104492 + }, + { + "layer": "layer.51.self_attn.q_proj", + "ppl": 4.043973922729492 + }, + { + "layer": "layer.51.self_attn.k_proj", + "ppl": 4.043980121612549 + }, + { + "layer": "layer.51.self_attn.v_proj", + "ppl": 4.043888568878174 + }, + { + "layer": "layer.51.self_attn.o_proj", + "ppl": 4.043778896331787 + }, + { + "layer": "layer.51.mlp.gate_proj", + "ppl": 4.04383659362793 + }, + { + "layer": "layer.51.mlp.up_proj", + "ppl": 4.043859481811523 + }, + { + "layer": "layer.51.mlp.down_proj", + "ppl": 4.044010162353516 + }, + { + "layer": "layer.52.self_attn.q_proj", + "ppl": 4.043908596038818 + }, + { + "layer": "layer.52.self_attn.k_proj", + "ppl": 4.043936729431152 + }, + { + "layer": "layer.52.self_attn.v_proj", + "ppl": 4.043880939483643 + }, + { + "layer": "layer.52.self_attn.o_proj", + "ppl": 4.044102668762207 + }, + { + "layer": "layer.52.mlp.gate_proj", + "ppl": 4.043936252593994 + }, + { + "layer": "layer.52.mlp.up_proj", + "ppl": 4.04403829574585 + }, + { + "layer": "layer.52.mlp.down_proj", + "ppl": 4.043817043304443 + }, + { + "layer": "layer.53.self_attn.q_proj", + "ppl": 4.043948650360107 + }, + { + "layer": "layer.53.self_attn.k_proj", + "ppl": 4.04390287399292 + }, + { + "layer": "layer.53.self_attn.v_proj", + "ppl": 4.043962478637695 + }, + { + "layer": "layer.53.self_attn.o_proj", + "ppl": 4.04371976852417 + }, + { + "layer": "layer.53.mlp.gate_proj", + "ppl": 4.043968677520752 + }, + { + "layer": "layer.53.mlp.up_proj", + "ppl": 4.043845176696777 + }, + { + "layer": "layer.53.mlp.down_proj", + "ppl": 4.04392671585083 + }, + { + "layer": "layer.54.self_attn.q_proj", + "ppl": 4.043956279754639 + }, + { + "layer": "layer.54.self_attn.k_proj", + "ppl": 4.043846130371094 + }, + { + "layer": "layer.54.self_attn.v_proj", + "ppl": 4.043888092041016 + }, + { + "layer": "layer.54.self_attn.o_proj", + "ppl": 4.04379415512085 + }, + { + "layer": "layer.54.mlp.gate_proj", + "ppl": 4.043759346008301 + }, + { + "layer": "layer.54.mlp.up_proj", + "ppl": 4.043955326080322 + }, + { + "layer": "layer.54.mlp.down_proj", + "ppl": 4.043949604034424 + }, + { + "layer": "layer.55.self_attn.q_proj", + "ppl": 4.043875694274902 + }, + { + "layer": "layer.55.self_attn.k_proj", + "ppl": 4.043854236602783 + }, + { + "layer": "layer.55.self_attn.v_proj", + "ppl": 4.043936729431152 + }, + { + "layer": "layer.55.self_attn.o_proj", + "ppl": 4.044079303741455 + }, + { + "layer": "layer.55.mlp.gate_proj", + "ppl": 4.043948650360107 + }, + { + "layer": "layer.55.mlp.up_proj", + "ppl": 4.04380464553833 + }, + { + "layer": "layer.55.mlp.down_proj", + "ppl": 4.043880939483643 + }, + { + "layer": "layer.56.self_attn.q_proj", + "ppl": 4.043857097625732 + }, + { + "layer": "layer.56.self_attn.k_proj", + "ppl": 4.0439453125 + }, + { + "layer": "layer.56.self_attn.v_proj", + "ppl": 4.043896675109863 + }, + { + "layer": "layer.56.self_attn.o_proj", + "ppl": 4.04400110244751 + }, + { + "layer": "layer.56.mlp.gate_proj", + "ppl": 4.043951988220215 + }, + { + "layer": "layer.56.mlp.up_proj", + "ppl": 4.043942451477051 + }, + { + "layer": "layer.56.mlp.down_proj", + "ppl": 4.043994426727295 + }, + { + "layer": "layer.57.self_attn.q_proj", + "ppl": 4.0438737869262695 + }, + { + "layer": "layer.57.self_attn.k_proj", + "ppl": 4.04386043548584 + }, + { + "layer": "layer.57.self_attn.v_proj", + "ppl": 4.043858051300049 + }, + { + "layer": "layer.57.self_attn.o_proj", + "ppl": 4.043949604034424 + }, + { + "layer": "layer.57.mlp.gate_proj", + "ppl": 4.043852806091309 + }, + { + "layer": "layer.57.mlp.up_proj", + "ppl": 4.043858051300049 + }, + { + "layer": "layer.57.mlp.down_proj", + "ppl": 4.043900966644287 + }, + { + "layer": "layer.58.self_attn.q_proj", + "ppl": 4.043980121612549 + }, + { + "layer": "layer.58.self_attn.k_proj", + "ppl": 4.043888568878174 + }, + { + "layer": "layer.58.self_attn.v_proj", + "ppl": 4.043999671936035 + }, + { + "layer": "layer.58.self_attn.o_proj", + "ppl": 4.044110298156738 + }, + { + "layer": "layer.58.mlp.gate_proj", + "ppl": 4.0440592765808105 + }, + { + "layer": "layer.58.mlp.up_proj", + "ppl": 4.044112205505371 + }, + { + "layer": "layer.58.mlp.down_proj", + "ppl": 4.043900966644287 + }, + { + "layer": "layer.59.self_attn.q_proj", + "ppl": 4.043820381164551 + }, + { + "layer": "layer.59.self_attn.k_proj", + "ppl": 4.043835163116455 + }, + { + "layer": "layer.59.self_attn.v_proj", + "ppl": 4.043880939483643 + }, + { + "layer": "layer.59.self_attn.o_proj", + "ppl": 4.0439019203186035 + }, + { + "layer": "layer.59.mlp.gate_proj", + "ppl": 4.04395866394043 + }, + { + "layer": "layer.59.mlp.up_proj", + "ppl": 4.044081687927246 + }, + { + "layer": "layer.59.mlp.down_proj", + "ppl": 4.043731689453125 + }, + { + "layer": "layer.60.self_attn.q_proj", + "ppl": 4.0439629554748535 + }, + { + "layer": "layer.60.self_attn.k_proj", + "ppl": 4.0439300537109375 + }, + { + "layer": "layer.60.self_attn.v_proj", + "ppl": 4.043937683105469 + }, + { + "layer": "layer.60.self_attn.o_proj", + "ppl": 4.043882846832275 + }, + { + "layer": "layer.60.mlp.gate_proj", + "ppl": 4.043750286102295 + }, + { + "layer": "layer.60.mlp.up_proj", + "ppl": 4.043828010559082 + }, + { + "layer": "layer.60.mlp.down_proj", + "ppl": 4.0440239906311035 + }, + { + "layer": "layer.61.self_attn.q_proj", + "ppl": 4.043919563293457 + }, + { + "layer": "layer.61.self_attn.k_proj", + "ppl": 4.043914794921875 + }, + { + "layer": "layer.61.self_attn.v_proj", + "ppl": 4.043885231018066 + }, + { + "layer": "layer.61.self_attn.o_proj", + "ppl": 4.043992042541504 + }, + { + "layer": "layer.61.mlp.gate_proj", + "ppl": 4.043686389923096 + }, + { + "layer": "layer.61.mlp.up_proj", + "ppl": 4.043905258178711 + }, + { + "layer": "layer.61.mlp.down_proj", + "ppl": 4.0439300537109375 + }, + { + "layer": "layer.62.self_attn.q_proj", + "ppl": 4.043905258178711 + }, + { + "layer": "layer.62.self_attn.k_proj", + "ppl": 4.044034957885742 + }, + { + "layer": "layer.62.self_attn.v_proj", + "ppl": 4.043938636779785 + }, + { + "layer": "layer.62.self_attn.o_proj", + "ppl": 4.043972969055176 + }, + { + "layer": "layer.62.mlp.gate_proj", + "ppl": 4.04370641708374 + }, + { + "layer": "layer.62.mlp.up_proj", + "ppl": 4.043883800506592 + }, + { + "layer": "layer.62.mlp.down_proj", + "ppl": 4.0437421798706055 + }, + { + "layer": "layer.63.self_attn.q_proj", + "ppl": 4.043998718261719 + }, + { + "layer": "layer.63.self_attn.k_proj", + "ppl": 4.044034481048584 + }, + { + "layer": "layer.63.self_attn.v_proj", + "ppl": 4.04395866394043 + }, + { + "layer": "layer.63.self_attn.o_proj", + "ppl": 4.043893337249756 + }, + { + "layer": "layer.63.mlp.gate_proj", + "ppl": 4.0438995361328125 + }, + { + "layer": "layer.63.mlp.up_proj", + "ppl": 4.043907642364502 + }, + { + "layer": "layer.63.mlp.down_proj", + "ppl": 4.043707847595215 + }, + { + "layer": "layer.64.self_attn.q_proj", + "ppl": 4.044018745422363 + }, + { + "layer": "layer.64.self_attn.k_proj", + "ppl": 4.043853759765625 + }, + { + "layer": "layer.64.self_attn.v_proj", + "ppl": 4.043876647949219 + }, + { + "layer": "layer.64.self_attn.o_proj", + "ppl": 4.043922424316406 + }, + { + "layer": "layer.64.mlp.gate_proj", + "ppl": 4.043790817260742 + }, + { + "layer": "layer.64.mlp.up_proj", + "ppl": 4.043712139129639 + }, + { + "layer": "layer.64.mlp.down_proj", + "ppl": 4.043884754180908 + }, + { + "layer": "layer.65.self_attn.q_proj", + "ppl": 4.043890953063965 + }, + { + "layer": "layer.65.self_attn.k_proj", + "ppl": 4.043797969818115 + }, + { + "layer": "layer.65.self_attn.v_proj", + "ppl": 4.043905258178711 + }, + { + "layer": "layer.65.self_attn.o_proj", + "ppl": 4.043973445892334 + }, + { + "layer": "layer.65.mlp.gate_proj", + "ppl": 4.043887615203857 + }, + { + "layer": "layer.65.mlp.up_proj", + "ppl": 4.0439934730529785 + }, + { + "layer": "layer.65.mlp.down_proj", + "ppl": 4.044065475463867 + }, + { + "layer": "layer.66.self_attn.q_proj", + "ppl": 4.043854236602783 + }, + { + "layer": "layer.66.self_attn.k_proj", + "ppl": 4.044000625610352 + }, + { + "layer": "layer.66.self_attn.v_proj", + "ppl": 4.043939113616943 + }, + { + "layer": "layer.66.self_attn.o_proj", + "ppl": 4.043939590454102 + }, + { + "layer": "layer.66.mlp.gate_proj", + "ppl": 4.044074058532715 + }, + { + "layer": "layer.66.mlp.up_proj", + "ppl": 4.0439019203186035 + }, + { + "layer": "layer.66.mlp.down_proj", + "ppl": 4.0438995361328125 + }, + { + "layer": "layer.67.self_attn.q_proj", + "ppl": 4.043975353240967 + }, + { + "layer": "layer.67.self_attn.k_proj", + "ppl": 4.043898582458496 + }, + { + "layer": "layer.67.self_attn.v_proj", + "ppl": 4.04392147064209 + }, + { + "layer": "layer.67.self_attn.o_proj", + "ppl": 4.043858051300049 + }, + { + "layer": "layer.67.mlp.gate_proj", + "ppl": 4.0438008308410645 + }, + { + "layer": "layer.67.mlp.up_proj", + "ppl": 4.043785095214844 + }, + { + "layer": "layer.67.mlp.down_proj", + "ppl": 4.0439019203186035 + }, + { + "layer": "layer.68.self_attn.q_proj", + "ppl": 4.043926239013672 + }, + { + "layer": "layer.68.self_attn.k_proj", + "ppl": 4.043943405151367 + }, + { + "layer": "layer.68.self_attn.v_proj", + "ppl": 4.043970584869385 + }, + { + "layer": "layer.68.self_attn.o_proj", + "ppl": 4.043906211853027 + }, + { + "layer": "layer.68.mlp.gate_proj", + "ppl": 4.044004917144775 + }, + { + "layer": "layer.68.mlp.up_proj", + "ppl": 4.04398775100708 + }, + { + "layer": "layer.68.mlp.down_proj", + "ppl": 4.043919563293457 + }, + { + "layer": "layer.69.self_attn.q_proj", + "ppl": 4.043932914733887 + }, + { + "layer": "layer.69.self_attn.k_proj", + "ppl": 4.043888568878174 + }, + { + "layer": "layer.69.self_attn.v_proj", + "ppl": 4.044010639190674 + }, + { + "layer": "layer.69.self_attn.o_proj", + "ppl": 4.043942451477051 + }, + { + "layer": "layer.69.mlp.gate_proj", + "ppl": 4.043562889099121 + }, + { + "layer": "layer.69.mlp.up_proj", + "ppl": 4.043938636779785 + }, + { + "layer": "layer.69.mlp.down_proj", + "ppl": 4.043909549713135 + }, + { + "layer": "layer.70.self_attn.q_proj", + "ppl": 4.043975353240967 + }, + { + "layer": "layer.70.self_attn.k_proj", + "ppl": 4.043896198272705 + }, + { + "layer": "layer.70.self_attn.v_proj", + "ppl": 4.043849945068359 + }, + { + "layer": "layer.70.self_attn.o_proj", + "ppl": 4.044025897979736 + }, + { + "layer": "layer.70.mlp.gate_proj", + "ppl": 4.043696880340576 + }, + { + "layer": "layer.70.mlp.up_proj", + "ppl": 4.043882846832275 + }, + { + "layer": "layer.70.mlp.down_proj", + "ppl": 4.043644905090332 + }, + { + "layer": "layer.71.self_attn.q_proj", + "ppl": 4.044027328491211 + }, + { + "layer": "layer.71.self_attn.k_proj", + "ppl": 4.043984413146973 + }, + { + "layer": "layer.71.self_attn.v_proj", + "ppl": 4.043861389160156 + }, + { + "layer": "layer.71.self_attn.o_proj", + "ppl": 4.043956279754639 + }, + { + "layer": "layer.71.mlp.gate_proj", + "ppl": 4.043773651123047 + }, + { + "layer": "layer.71.mlp.up_proj", + "ppl": 4.043957710266113 + }, + { + "layer": "layer.71.mlp.down_proj", + "ppl": 4.043722152709961 + }, + { + "layer": "layer.72.self_attn.q_proj", + "ppl": 4.043973922729492 + }, + { + "layer": "layer.72.self_attn.k_proj", + "ppl": 4.043910503387451 + }, + { + "layer": "layer.72.self_attn.v_proj", + "ppl": 4.043924331665039 + }, + { + "layer": "layer.72.self_attn.o_proj", + "ppl": 4.0439252853393555 + }, + { + "layer": "layer.72.mlp.gate_proj", + "ppl": 4.043594837188721 + }, + { + "layer": "layer.72.mlp.up_proj", + "ppl": 4.043943405151367 + }, + { + "layer": "layer.72.mlp.down_proj", + "ppl": 4.043697357177734 + }, + { + "layer": "layer.73.self_attn.q_proj", + "ppl": 4.043870449066162 + }, + { + "layer": "layer.73.self_attn.k_proj", + "ppl": 4.0438551902771 + }, + { + "layer": "layer.73.self_attn.v_proj", + "ppl": 4.043941974639893 + }, + { + "layer": "layer.73.self_attn.o_proj", + "ppl": 4.044027805328369 + }, + { + "layer": "layer.73.mlp.gate_proj", + "ppl": 4.0438055992126465 + }, + { + "layer": "layer.73.mlp.up_proj", + "ppl": 4.043988227844238 + }, + { + "layer": "layer.73.mlp.down_proj", + "ppl": 4.043882846832275 + }, + { + "layer": "layer.74.self_attn.q_proj", + "ppl": 4.044005393981934 + }, + { + "layer": "layer.74.self_attn.k_proj", + "ppl": 4.043970584869385 + }, + { + "layer": "layer.74.self_attn.v_proj", + "ppl": 4.0439534187316895 + }, + { + "layer": "layer.74.self_attn.o_proj", + "ppl": 4.043971061706543 + }, + { + "layer": "layer.74.mlp.gate_proj", + "ppl": 4.043741226196289 + }, + { + "layer": "layer.74.mlp.up_proj", + "ppl": 4.043886661529541 + }, + { + "layer": "layer.74.mlp.down_proj", + "ppl": 4.043938636779785 + }, + { + "layer": "layer.75.self_attn.q_proj", + "ppl": 4.044022560119629 + }, + { + "layer": "layer.75.self_attn.k_proj", + "ppl": 4.04390287399292 + }, + { + "layer": "layer.75.self_attn.v_proj", + "ppl": 4.043883800506592 + }, + { + "layer": "layer.75.self_attn.o_proj", + "ppl": 4.04392147064209 + }, + { + "layer": "layer.75.mlp.gate_proj", + "ppl": 4.043641567230225 + }, + { + "layer": "layer.75.mlp.up_proj", + "ppl": 4.043879985809326 + }, + { + "layer": "layer.75.mlp.down_proj", + "ppl": 4.0437211990356445 + }, + { + "layer": "layer.76.self_attn.q_proj", + "ppl": 4.044018268585205 + }, + { + "layer": "layer.76.self_attn.k_proj", + "ppl": 4.043949604034424 + }, + { + "layer": "layer.76.self_attn.v_proj", + "ppl": 4.043971061706543 + }, + { + "layer": "layer.76.self_attn.o_proj", + "ppl": 4.0438055992126465 + }, + { + "layer": "layer.76.mlp.gate_proj", + "ppl": 4.043679714202881 + }, + { + "layer": "layer.76.mlp.up_proj", + "ppl": 4.043784141540527 + }, + { + "layer": "layer.76.mlp.down_proj", + "ppl": 4.043767929077148 + }, + { + "layer": "layer.77.self_attn.q_proj", + "ppl": 4.04388427734375 + }, + { + "layer": "layer.77.self_attn.k_proj", + "ppl": 4.044036865234375 + }, + { + "layer": "layer.77.self_attn.v_proj", + "ppl": 4.043900966644287 + }, + { + "layer": "layer.77.self_attn.o_proj", + "ppl": 4.043927192687988 + }, + { + "layer": "layer.77.mlp.gate_proj", + "ppl": 4.043785572052002 + }, + { + "layer": "layer.77.mlp.up_proj", + "ppl": 4.043776988983154 + }, + { + "layer": "layer.77.mlp.down_proj", + "ppl": 4.043773174285889 + }, + { + "layer": "layer.78.self_attn.q_proj", + "ppl": 4.043992519378662 + }, + { + "layer": "layer.78.self_attn.k_proj", + "ppl": 4.043889045715332 + }, + { + "layer": "layer.78.self_attn.v_proj", + "ppl": 4.044020652770996 + }, + { + "layer": "layer.78.self_attn.o_proj", + "ppl": 4.043919086456299 + }, + { + "layer": "layer.78.mlp.gate_proj", + "ppl": 4.043591499328613 + }, + { + "layer": "layer.78.mlp.up_proj", + "ppl": 4.043654918670654 + }, + { + "layer": "layer.78.mlp.down_proj", + "ppl": 4.043878078460693 + }, + { + "layer": "layer.79.self_attn.q_proj", + "ppl": 4.0439043045043945 + }, + { + "layer": "layer.79.self_attn.k_proj", + "ppl": 4.043929100036621 + }, + { + "layer": "layer.79.self_attn.v_proj", + "ppl": 4.043854713439941 + }, + { + "layer": "layer.79.self_attn.o_proj", + "ppl": 4.0439910888671875 + }, + { + "layer": "layer.79.mlp.gate_proj", + "ppl": 4.0433030128479 + }, + { + "layer": "layer.79.mlp.up_proj", + "ppl": 4.043323993682861 + }, + { + "layer": "layer.79.mlp.down_proj", + "ppl": 4.043768882751465 + } + ] + }, + { + "bit": 13, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 4.030300140380859 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 4.030320644378662 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 4.030306339263916 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 4.0303544998168945 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 4.030325412750244 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 4.030334949493408 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 4.0303955078125 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 4.030278205871582 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 4.030249118804932 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 4.030313491821289 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 4.030251979827881 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 4.030374050140381 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 4.030310153961182 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 4.030293941497803 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 4.0303635597229 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 4.030404567718506 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 4.030335903167725 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 4.030294418334961 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 4.030339241027832 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 4.030282974243164 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 4.03031063079834 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 4.030311107635498 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 4.03024959564209 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 4.030337333679199 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 4.030501842498779 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 4.030257701873779 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 4.030274868011475 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 4.030299186706543 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 4.030343532562256 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 4.030369281768799 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 4.030305862426758 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 4.030350208282471 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 4.030240058898926 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 4.03036642074585 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 4.030404090881348 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 4.030263900756836 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 4.030323028564453 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 4.030360698699951 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 4.030361175537109 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 4.030242919921875 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 4.0303730964660645 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 4.030345439910889 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 4.030312538146973 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 4.030207633972168 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 4.030322551727295 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 4.0303215980529785 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 4.030402183532715 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 4.0303239822387695 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 4.030340194702148 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 4.030323505401611 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 4.030276298522949 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 4.030302047729492 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 4.030289173126221 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 4.030385494232178 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 4.030272006988525 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 4.0302653312683105 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 4.030339241027832 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 4.030358791351318 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 4.030379772186279 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 4.030267238616943 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 4.030261993408203 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 4.030203819274902 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 4.030357837677002 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 4.030334949493408 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 4.030344486236572 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 4.030317783355713 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 4.03031063079834 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 4.030341148376465 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 4.0303635597229 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 4.030264377593994 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 4.030228614807129 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 4.030303955078125 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 4.0302839279174805 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 4.030327320098877 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 4.030280590057373 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 4.030345916748047 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 4.0304670333862305 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 4.030299186706543 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 4.030424118041992 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 4.030372619628906 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 4.030280590057373 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 4.03032922744751 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 4.030395984649658 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 4.030294895172119 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 4.0302815437316895 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 4.0301690101623535 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 4.03036642074585 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 4.030246257781982 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 4.030324935913086 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 4.0303192138671875 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 4.030378818511963 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 4.030353546142578 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 4.030253887176514 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 4.030228614807129 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 4.030284881591797 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 4.0304083824157715 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 4.030338287353516 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 4.03028678894043 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 4.030330181121826 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 4.030307769775391 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 4.030357837677002 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 4.030368328094482 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 4.030427932739258 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 4.0303263664245605 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 4.030174255371094 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 4.030248165130615 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 4.030287265777588 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 4.030351161956787 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 4.0302276611328125 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 4.03032112121582 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 4.0301713943481445 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 4.030241012573242 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 4.030404567718506 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 4.030255317687988 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 4.030365467071533 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 4.030368328094482 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 4.030298709869385 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 4.03024435043335 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 4.030287265777588 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 4.03040075302124 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 4.030200958251953 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 4.03030252456665 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 4.030299186706543 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 4.030246257781982 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 4.03032922744751 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 4.030245304107666 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 4.030302047729492 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 4.030287265777588 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 4.030293941497803 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 4.030269622802734 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 4.030420780181885 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 4.030302047729492 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 4.030343532562256 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 4.030219554901123 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 4.030289173126221 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 4.030328273773193 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 4.030330657958984 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 4.03034782409668 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 4.030346870422363 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 4.030206680297852 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 4.030307292938232 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 4.030358791351318 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 4.0303425788879395 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 4.030385494232178 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 4.030406475067139 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 4.030294895172119 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 4.030339241027832 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 4.030287265777588 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 4.030318737030029 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 4.030221939086914 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 4.030294895172119 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 4.030367374420166 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 4.030262470245361 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 4.030279636383057 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 4.030274868011475 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 4.0302653312683105 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 4.030302047729492 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 4.030357360839844 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 4.030345916748047 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 4.030313014984131 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 4.030343532562256 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 4.030291557312012 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 4.030287265777588 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 4.0303239822387695 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 4.030351161956787 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 4.030223369598389 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 4.03037691116333 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 4.03029727935791 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 4.030322074890137 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 4.0302557945251465 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 4.030379295349121 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 4.030351161956787 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 4.030202865600586 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 4.030235290527344 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 4.030130386352539 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 4.030273914337158 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 4.0304460525512695 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 4.030241012573242 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 4.030358791351318 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 4.030311584472656 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 4.030388355255127 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 4.030360698699951 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 4.030272960662842 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 4.03035306930542 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 4.030344486236572 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 4.0303263664245605 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 4.030350208282471 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 4.030280590057373 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 4.030289649963379 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 4.030294418334961 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 4.030388355255127 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 4.030145168304443 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 4.030348777770996 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 4.030266284942627 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 4.030526161193848 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 4.030293941497803 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 4.030209064483643 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 4.030267715454102 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 4.030310153961182 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 4.030259609222412 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 4.0302629470825195 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 4.03045654296875 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 4.030203819274902 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 4.0303239822387695 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 4.030256748199463 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 4.0302534103393555 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 4.030309677124023 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 4.030351161956787 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 4.030399799346924 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 4.030203819274902 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 4.030303955078125 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 4.030335903167725 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 4.030234336853027 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 4.030165672302246 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 4.03032112121582 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 4.030051231384277 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 4.03016996383667 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 4.030291557312012 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 4.030358791351318 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 4.030341148376465 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 4.0302934646606445 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 4.0302839279174805 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 4.030176162719727 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 4.030501842498779 + }, + { + "layer": "layer.32.self_attn.q_proj", + "ppl": 4.030241966247559 + }, + { + "layer": "layer.32.self_attn.k_proj", + "ppl": 4.030435085296631 + }, + { + "layer": "layer.32.self_attn.v_proj", + "ppl": 4.030257701873779 + }, + { + "layer": "layer.32.self_attn.o_proj", + "ppl": 4.030277252197266 + }, + { + "layer": "layer.32.mlp.gate_proj", + "ppl": 4.030152797698975 + }, + { + "layer": "layer.32.mlp.up_proj", + "ppl": 4.03041410446167 + }, + { + "layer": "layer.32.mlp.down_proj", + "ppl": 4.030269622802734 + }, + { + "layer": "layer.33.self_attn.q_proj", + "ppl": 4.0303874015808105 + }, + { + "layer": "layer.33.self_attn.k_proj", + "ppl": 4.030323028564453 + }, + { + "layer": "layer.33.self_attn.v_proj", + "ppl": 4.030262470245361 + }, + { + "layer": "layer.33.self_attn.o_proj", + "ppl": 4.030195236206055 + }, + { + "layer": "layer.33.mlp.gate_proj", + "ppl": 4.03037166595459 + }, + { + "layer": "layer.33.mlp.up_proj", + "ppl": 4.030210018157959 + }, + { + "layer": "layer.33.mlp.down_proj", + "ppl": 4.030142784118652 + }, + { + "layer": "layer.34.self_attn.q_proj", + "ppl": 4.030331134796143 + }, + { + "layer": "layer.34.self_attn.k_proj", + "ppl": 4.030393123626709 + }, + { + "layer": "layer.34.self_attn.v_proj", + "ppl": 4.030179023742676 + }, + { + "layer": "layer.34.self_attn.o_proj", + "ppl": 4.030429840087891 + }, + { + "layer": "layer.34.mlp.gate_proj", + "ppl": 4.030328750610352 + }, + { + "layer": "layer.34.mlp.up_proj", + "ppl": 4.030214309692383 + }, + { + "layer": "layer.34.mlp.down_proj", + "ppl": 4.030300140380859 + }, + { + "layer": "layer.35.self_attn.q_proj", + "ppl": 4.03029727935791 + }, + { + "layer": "layer.35.self_attn.k_proj", + "ppl": 4.030348777770996 + }, + { + "layer": "layer.35.self_attn.v_proj", + "ppl": 4.030304908752441 + }, + { + "layer": "layer.35.self_attn.o_proj", + "ppl": 4.030323028564453 + }, + { + "layer": "layer.35.mlp.gate_proj", + "ppl": 4.030323028564453 + }, + { + "layer": "layer.35.mlp.up_proj", + "ppl": 4.030488014221191 + }, + { + "layer": "layer.35.mlp.down_proj", + "ppl": 4.030218601226807 + }, + { + "layer": "layer.36.self_attn.q_proj", + "ppl": 4.03024435043335 + }, + { + "layer": "layer.36.self_attn.k_proj", + "ppl": 4.030251979827881 + }, + { + "layer": "layer.36.self_attn.v_proj", + "ppl": 4.030226230621338 + }, + { + "layer": "layer.36.self_attn.o_proj", + "ppl": 4.030468463897705 + }, + { + "layer": "layer.36.mlp.gate_proj", + "ppl": 4.0302934646606445 + }, + { + "layer": "layer.36.mlp.up_proj", + "ppl": 4.030391216278076 + }, + { + "layer": "layer.36.mlp.down_proj", + "ppl": 4.0302510261535645 + }, + { + "layer": "layer.37.self_attn.q_proj", + "ppl": 4.0303168296813965 + }, + { + "layer": "layer.37.self_attn.k_proj", + "ppl": 4.03041410446167 + }, + { + "layer": "layer.37.self_attn.v_proj", + "ppl": 4.030323028564453 + }, + { + "layer": "layer.37.self_attn.o_proj", + "ppl": 4.030358791351318 + }, + { + "layer": "layer.37.mlp.gate_proj", + "ppl": 4.030244827270508 + }, + { + "layer": "layer.37.mlp.up_proj", + "ppl": 4.03038215637207 + }, + { + "layer": "layer.37.mlp.down_proj", + "ppl": 4.030214309692383 + }, + { + "layer": "layer.38.self_attn.q_proj", + "ppl": 4.030287265777588 + }, + { + "layer": "layer.38.self_attn.k_proj", + "ppl": 4.030282497406006 + }, + { + "layer": "layer.38.self_attn.v_proj", + "ppl": 4.030273914337158 + }, + { + "layer": "layer.38.self_attn.o_proj", + "ppl": 4.030238151550293 + }, + { + "layer": "layer.38.mlp.gate_proj", + "ppl": 4.030547618865967 + }, + { + "layer": "layer.38.mlp.up_proj", + "ppl": 4.030395984649658 + }, + { + "layer": "layer.38.mlp.down_proj", + "ppl": 4.03023099899292 + }, + { + "layer": "layer.39.self_attn.q_proj", + "ppl": 4.030301094055176 + }, + { + "layer": "layer.39.self_attn.k_proj", + "ppl": 4.030256271362305 + }, + { + "layer": "layer.39.self_attn.v_proj", + "ppl": 4.0303874015808105 + }, + { + "layer": "layer.39.self_attn.o_proj", + "ppl": 4.03030252456665 + }, + { + "layer": "layer.39.mlp.gate_proj", + "ppl": 4.0300798416137695 + }, + { + "layer": "layer.39.mlp.up_proj", + "ppl": 4.030294895172119 + }, + { + "layer": "layer.39.mlp.down_proj", + "ppl": 4.030218601226807 + }, + { + "layer": "layer.40.self_attn.q_proj", + "ppl": 4.030333995819092 + }, + { + "layer": "layer.40.self_attn.k_proj", + "ppl": 4.030334949493408 + }, + { + "layer": "layer.40.self_attn.v_proj", + "ppl": 4.030364513397217 + }, + { + "layer": "layer.40.self_attn.o_proj", + "ppl": 4.0303955078125 + }, + { + "layer": "layer.40.mlp.gate_proj", + "ppl": 4.030187606811523 + }, + { + "layer": "layer.40.mlp.up_proj", + "ppl": 4.030415058135986 + }, + { + "layer": "layer.40.mlp.down_proj", + "ppl": 4.029979705810547 + }, + { + "layer": "layer.41.self_attn.q_proj", + "ppl": 4.030292510986328 + }, + { + "layer": "layer.41.self_attn.k_proj", + "ppl": 4.030210971832275 + }, + { + "layer": "layer.41.self_attn.v_proj", + "ppl": 4.03029727935791 + }, + { + "layer": "layer.41.self_attn.o_proj", + "ppl": 4.030377388000488 + }, + { + "layer": "layer.41.mlp.gate_proj", + "ppl": 4.030291557312012 + }, + { + "layer": "layer.41.mlp.up_proj", + "ppl": 4.030487060546875 + }, + { + "layer": "layer.41.mlp.down_proj", + "ppl": 4.030303001403809 + }, + { + "layer": "layer.42.self_attn.q_proj", + "ppl": 4.030311584472656 + }, + { + "layer": "layer.42.self_attn.k_proj", + "ppl": 4.030353546142578 + }, + { + "layer": "layer.42.self_attn.v_proj", + "ppl": 4.0303425788879395 + }, + { + "layer": "layer.42.self_attn.o_proj", + "ppl": 4.030345439910889 + }, + { + "layer": "layer.42.mlp.gate_proj", + "ppl": 4.030423164367676 + }, + { + "layer": "layer.42.mlp.up_proj", + "ppl": 4.03024435043335 + }, + { + "layer": "layer.42.mlp.down_proj", + "ppl": 4.030306816101074 + }, + { + "layer": "layer.43.self_attn.q_proj", + "ppl": 4.03041410446167 + }, + { + "layer": "layer.43.self_attn.k_proj", + "ppl": 4.030298233032227 + }, + { + "layer": "layer.43.self_attn.v_proj", + "ppl": 4.030315399169922 + }, + { + "layer": "layer.43.self_attn.o_proj", + "ppl": 4.030235767364502 + }, + { + "layer": "layer.43.mlp.gate_proj", + "ppl": 4.030477046966553 + }, + { + "layer": "layer.43.mlp.up_proj", + "ppl": 4.030349254608154 + }, + { + "layer": "layer.43.mlp.down_proj", + "ppl": 4.030450820922852 + }, + { + "layer": "layer.44.self_attn.q_proj", + "ppl": 4.030221939086914 + }, + { + "layer": "layer.44.self_attn.k_proj", + "ppl": 4.030304908752441 + }, + { + "layer": "layer.44.self_attn.v_proj", + "ppl": 4.030227184295654 + }, + { + "layer": "layer.44.self_attn.o_proj", + "ppl": 4.030300140380859 + }, + { + "layer": "layer.44.mlp.gate_proj", + "ppl": 4.030324935913086 + }, + { + "layer": "layer.44.mlp.up_proj", + "ppl": 4.030100345611572 + }, + { + "layer": "layer.44.mlp.down_proj", + "ppl": 4.030471324920654 + }, + { + "layer": "layer.45.self_attn.q_proj", + "ppl": 4.030299186706543 + }, + { + "layer": "layer.45.self_attn.k_proj", + "ppl": 4.030294418334961 + }, + { + "layer": "layer.45.self_attn.v_proj", + "ppl": 4.030175685882568 + }, + { + "layer": "layer.45.self_attn.o_proj", + "ppl": 4.030219554901123 + }, + { + "layer": "layer.45.mlp.gate_proj", + "ppl": 4.030328273773193 + }, + { + "layer": "layer.45.mlp.up_proj", + "ppl": 4.030404090881348 + }, + { + "layer": "layer.45.mlp.down_proj", + "ppl": 4.030442237854004 + }, + { + "layer": "layer.46.self_attn.q_proj", + "ppl": 4.030220985412598 + }, + { + "layer": "layer.46.self_attn.k_proj", + "ppl": 4.03039026260376 + }, + { + "layer": "layer.46.self_attn.v_proj", + "ppl": 4.0301971435546875 + }, + { + "layer": "layer.46.self_attn.o_proj", + "ppl": 4.030364513397217 + }, + { + "layer": "layer.46.mlp.gate_proj", + "ppl": 4.030195236206055 + }, + { + "layer": "layer.46.mlp.up_proj", + "ppl": 4.030271053314209 + }, + { + "layer": "layer.46.mlp.down_proj", + "ppl": 4.030344486236572 + }, + { + "layer": "layer.47.self_attn.q_proj", + "ppl": 4.030328750610352 + }, + { + "layer": "layer.47.self_attn.k_proj", + "ppl": 4.0303778648376465 + }, + { + "layer": "layer.47.self_attn.v_proj", + "ppl": 4.0302934646606445 + }, + { + "layer": "layer.47.self_attn.o_proj", + "ppl": 4.030449867248535 + }, + { + "layer": "layer.47.mlp.gate_proj", + "ppl": 4.030425071716309 + }, + { + "layer": "layer.47.mlp.up_proj", + "ppl": 4.030220031738281 + }, + { + "layer": "layer.47.mlp.down_proj", + "ppl": 4.030379772186279 + }, + { + "layer": "layer.48.self_attn.q_proj", + "ppl": 4.030379295349121 + }, + { + "layer": "layer.48.self_attn.k_proj", + "ppl": 4.03031063079834 + }, + { + "layer": "layer.48.self_attn.v_proj", + "ppl": 4.030262470245361 + }, + { + "layer": "layer.48.self_attn.o_proj", + "ppl": 4.0303568840026855 + }, + { + "layer": "layer.48.mlp.gate_proj", + "ppl": 4.030215263366699 + }, + { + "layer": "layer.48.mlp.up_proj", + "ppl": 4.030137538909912 + }, + { + "layer": "layer.48.mlp.down_proj", + "ppl": 4.030289649963379 + }, + { + "layer": "layer.49.self_attn.q_proj", + "ppl": 4.030249118804932 + }, + { + "layer": "layer.49.self_attn.k_proj", + "ppl": 4.030313491821289 + }, + { + "layer": "layer.49.self_attn.v_proj", + "ppl": 4.030167102813721 + }, + { + "layer": "layer.49.self_attn.o_proj", + "ppl": 4.030340194702148 + }, + { + "layer": "layer.49.mlp.gate_proj", + "ppl": 4.0301361083984375 + }, + { + "layer": "layer.49.mlp.up_proj", + "ppl": 4.030280590057373 + }, + { + "layer": "layer.49.mlp.down_proj", + "ppl": 4.0303497314453125 + }, + { + "layer": "layer.50.self_attn.q_proj", + "ppl": 4.030213356018066 + }, + { + "layer": "layer.50.self_attn.k_proj", + "ppl": 4.030304431915283 + }, + { + "layer": "layer.50.self_attn.v_proj", + "ppl": 4.030389308929443 + }, + { + "layer": "layer.50.self_attn.o_proj", + "ppl": 4.030300140380859 + }, + { + "layer": "layer.50.mlp.gate_proj", + "ppl": 4.030085563659668 + }, + { + "layer": "layer.50.mlp.up_proj", + "ppl": 4.030195236206055 + }, + { + "layer": "layer.50.mlp.down_proj", + "ppl": 4.030358791351318 + }, + { + "layer": "layer.51.self_attn.q_proj", + "ppl": 4.030277252197266 + }, + { + "layer": "layer.51.self_attn.k_proj", + "ppl": 4.030226230621338 + }, + { + "layer": "layer.51.self_attn.v_proj", + "ppl": 4.030412673950195 + }, + { + "layer": "layer.51.self_attn.o_proj", + "ppl": 4.030370235443115 + }, + { + "layer": "layer.51.mlp.gate_proj", + "ppl": 4.030272960662842 + }, + { + "layer": "layer.51.mlp.up_proj", + "ppl": 4.030302047729492 + }, + { + "layer": "layer.51.mlp.down_proj", + "ppl": 4.030211448669434 + }, + { + "layer": "layer.52.self_attn.q_proj", + "ppl": 4.0302557945251465 + }, + { + "layer": "layer.52.self_attn.k_proj", + "ppl": 4.030351161956787 + }, + { + "layer": "layer.52.self_attn.v_proj", + "ppl": 4.030462741851807 + }, + { + "layer": "layer.52.self_attn.o_proj", + "ppl": 4.030141353607178 + }, + { + "layer": "layer.52.mlp.gate_proj", + "ppl": 4.030073165893555 + }, + { + "layer": "layer.52.mlp.up_proj", + "ppl": 4.03047513961792 + }, + { + "layer": "layer.52.mlp.down_proj", + "ppl": 4.030202388763428 + }, + { + "layer": "layer.53.self_attn.q_proj", + "ppl": 4.0303635597229 + }, + { + "layer": "layer.53.self_attn.k_proj", + "ppl": 4.0303826332092285 + }, + { + "layer": "layer.53.self_attn.v_proj", + "ppl": 4.030261993408203 + }, + { + "layer": "layer.53.self_attn.o_proj", + "ppl": 4.030416011810303 + }, + { + "layer": "layer.53.mlp.gate_proj", + "ppl": 4.0301337242126465 + }, + { + "layer": "layer.53.mlp.up_proj", + "ppl": 4.030377388000488 + }, + { + "layer": "layer.53.mlp.down_proj", + "ppl": 4.03020715713501 + }, + { + "layer": "layer.54.self_attn.q_proj", + "ppl": 4.030333518981934 + }, + { + "layer": "layer.54.self_attn.k_proj", + "ppl": 4.030256748199463 + }, + { + "layer": "layer.54.self_attn.v_proj", + "ppl": 4.030430793762207 + }, + { + "layer": "layer.54.self_attn.o_proj", + "ppl": 4.030412673950195 + }, + { + "layer": "layer.54.mlp.gate_proj", + "ppl": 4.0303192138671875 + }, + { + "layer": "layer.54.mlp.up_proj", + "ppl": 4.0303425788879395 + }, + { + "layer": "layer.54.mlp.down_proj", + "ppl": 4.030274391174316 + }, + { + "layer": "layer.55.self_attn.q_proj", + "ppl": 4.030454158782959 + }, + { + "layer": "layer.55.self_attn.k_proj", + "ppl": 4.030357360839844 + }, + { + "layer": "layer.55.self_attn.v_proj", + "ppl": 4.030313491821289 + }, + { + "layer": "layer.55.self_attn.o_proj", + "ppl": 4.030243396759033 + }, + { + "layer": "layer.55.mlp.gate_proj", + "ppl": 4.03028678894043 + }, + { + "layer": "layer.55.mlp.up_proj", + "ppl": 4.030223846435547 + }, + { + "layer": "layer.55.mlp.down_proj", + "ppl": 4.030223369598389 + }, + { + "layer": "layer.56.self_attn.q_proj", + "ppl": 4.030380725860596 + }, + { + "layer": "layer.56.self_attn.k_proj", + "ppl": 4.030240058898926 + }, + { + "layer": "layer.56.self_attn.v_proj", + "ppl": 4.030431270599365 + }, + { + "layer": "layer.56.self_attn.o_proj", + "ppl": 4.03031587600708 + }, + { + "layer": "layer.56.mlp.gate_proj", + "ppl": 4.03035306930542 + }, + { + "layer": "layer.56.mlp.up_proj", + "ppl": 4.03027868270874 + }, + { + "layer": "layer.56.mlp.down_proj", + "ppl": 4.030256748199463 + }, + { + "layer": "layer.57.self_attn.q_proj", + "ppl": 4.030395984649658 + }, + { + "layer": "layer.57.self_attn.k_proj", + "ppl": 4.030384540557861 + }, + { + "layer": "layer.57.self_attn.v_proj", + "ppl": 4.030311584472656 + }, + { + "layer": "layer.57.self_attn.o_proj", + "ppl": 4.0303544998168945 + }, + { + "layer": "layer.57.mlp.gate_proj", + "ppl": 4.0301971435546875 + }, + { + "layer": "layer.57.mlp.up_proj", + "ppl": 4.030294418334961 + }, + { + "layer": "layer.57.mlp.down_proj", + "ppl": 4.030301094055176 + }, + { + "layer": "layer.58.self_attn.q_proj", + "ppl": 4.03039026260376 + }, + { + "layer": "layer.58.self_attn.k_proj", + "ppl": 4.030402183532715 + }, + { + "layer": "layer.58.self_attn.v_proj", + "ppl": 4.030315399169922 + }, + { + "layer": "layer.58.self_attn.o_proj", + "ppl": 4.030324459075928 + }, + { + "layer": "layer.58.mlp.gate_proj", + "ppl": 4.030262470245361 + }, + { + "layer": "layer.58.mlp.up_proj", + "ppl": 4.030433654785156 + }, + { + "layer": "layer.58.mlp.down_proj", + "ppl": 4.0304107666015625 + }, + { + "layer": "layer.59.self_attn.q_proj", + "ppl": 4.030320167541504 + }, + { + "layer": "layer.59.self_attn.k_proj", + "ppl": 4.030431270599365 + }, + { + "layer": "layer.59.self_attn.v_proj", + "ppl": 4.030345916748047 + }, + { + "layer": "layer.59.self_attn.o_proj", + "ppl": 4.030228137969971 + }, + { + "layer": "layer.59.mlp.gate_proj", + "ppl": 4.030168533325195 + }, + { + "layer": "layer.59.mlp.up_proj", + "ppl": 4.030200004577637 + }, + { + "layer": "layer.59.mlp.down_proj", + "ppl": 4.030359745025635 + }, + { + "layer": "layer.60.self_attn.q_proj", + "ppl": 4.030334949493408 + }, + { + "layer": "layer.60.self_attn.k_proj", + "ppl": 4.0302605628967285 + }, + { + "layer": "layer.60.self_attn.v_proj", + "ppl": 4.03032922744751 + }, + { + "layer": "layer.60.self_attn.o_proj", + "ppl": 4.030339241027832 + }, + { + "layer": "layer.60.mlp.gate_proj", + "ppl": 4.03040075302124 + }, + { + "layer": "layer.60.mlp.up_proj", + "ppl": 4.030409812927246 + }, + { + "layer": "layer.60.mlp.down_proj", + "ppl": 4.030307769775391 + }, + { + "layer": "layer.61.self_attn.q_proj", + "ppl": 4.030318260192871 + }, + { + "layer": "layer.61.self_attn.k_proj", + "ppl": 4.030223369598389 + }, + { + "layer": "layer.61.self_attn.v_proj", + "ppl": 4.030406475067139 + }, + { + "layer": "layer.61.self_attn.o_proj", + "ppl": 4.03032112121582 + }, + { + "layer": "layer.61.mlp.gate_proj", + "ppl": 4.030262470245361 + }, + { + "layer": "layer.61.mlp.up_proj", + "ppl": 4.030367374420166 + }, + { + "layer": "layer.61.mlp.down_proj", + "ppl": 4.030375003814697 + }, + { + "layer": "layer.62.self_attn.q_proj", + "ppl": 4.030285835266113 + }, + { + "layer": "layer.62.self_attn.k_proj", + "ppl": 4.030272960662842 + }, + { + "layer": "layer.62.self_attn.v_proj", + "ppl": 4.030372142791748 + }, + { + "layer": "layer.62.self_attn.o_proj", + "ppl": 4.030232906341553 + }, + { + "layer": "layer.62.mlp.gate_proj", + "ppl": 4.030242443084717 + }, + { + "layer": "layer.62.mlp.up_proj", + "ppl": 4.0303215980529785 + }, + { + "layer": "layer.62.mlp.down_proj", + "ppl": 4.030322551727295 + }, + { + "layer": "layer.63.self_attn.q_proj", + "ppl": 4.030407428741455 + }, + { + "layer": "layer.63.self_attn.k_proj", + "ppl": 4.030315399169922 + }, + { + "layer": "layer.63.self_attn.v_proj", + "ppl": 4.030304431915283 + }, + { + "layer": "layer.63.self_attn.o_proj", + "ppl": 4.0302958488464355 + }, + { + "layer": "layer.63.mlp.gate_proj", + "ppl": 4.030387878417969 + }, + { + "layer": "layer.63.mlp.up_proj", + "ppl": 4.030331611633301 + }, + { + "layer": "layer.63.mlp.down_proj", + "ppl": 4.030481338500977 + }, + { + "layer": "layer.64.self_attn.q_proj", + "ppl": 4.030349254608154 + }, + { + "layer": "layer.64.self_attn.k_proj", + "ppl": 4.030374050140381 + }, + { + "layer": "layer.64.self_attn.v_proj", + "ppl": 4.030304431915283 + }, + { + "layer": "layer.64.self_attn.o_proj", + "ppl": 4.0303521156311035 + }, + { + "layer": "layer.64.mlp.gate_proj", + "ppl": 4.030251979827881 + }, + { + "layer": "layer.64.mlp.up_proj", + "ppl": 4.030550956726074 + }, + { + "layer": "layer.64.mlp.down_proj", + "ppl": 4.030398845672607 + }, + { + "layer": "layer.65.self_attn.q_proj", + "ppl": 4.0303239822387695 + }, + { + "layer": "layer.65.self_attn.k_proj", + "ppl": 4.0303802490234375 + }, + { + "layer": "layer.65.self_attn.v_proj", + "ppl": 4.030357837677002 + }, + { + "layer": "layer.65.self_attn.o_proj", + "ppl": 4.03023099899292 + }, + { + "layer": "layer.65.mlp.gate_proj", + "ppl": 4.030256271362305 + }, + { + "layer": "layer.65.mlp.up_proj", + "ppl": 4.030299186706543 + }, + { + "layer": "layer.65.mlp.down_proj", + "ppl": 4.030216693878174 + }, + { + "layer": "layer.66.self_attn.q_proj", + "ppl": 4.0303497314453125 + }, + { + "layer": "layer.66.self_attn.k_proj", + "ppl": 4.030345439910889 + }, + { + "layer": "layer.66.self_attn.v_proj", + "ppl": 4.030251979827881 + }, + { + "layer": "layer.66.self_attn.o_proj", + "ppl": 4.0302815437316895 + }, + { + "layer": "layer.66.mlp.gate_proj", + "ppl": 4.030104160308838 + }, + { + "layer": "layer.66.mlp.up_proj", + "ppl": 4.030310153961182 + }, + { + "layer": "layer.66.mlp.down_proj", + "ppl": 4.030154705047607 + }, + { + "layer": "layer.67.self_attn.q_proj", + "ppl": 4.030268669128418 + }, + { + "layer": "layer.67.self_attn.k_proj", + "ppl": 4.03036642074585 + }, + { + "layer": "layer.67.self_attn.v_proj", + "ppl": 4.030372142791748 + }, + { + "layer": "layer.67.self_attn.o_proj", + "ppl": 4.030425071716309 + }, + { + "layer": "layer.67.mlp.gate_proj", + "ppl": 4.03021764755249 + }, + { + "layer": "layer.67.mlp.up_proj", + "ppl": 4.030328273773193 + }, + { + "layer": "layer.67.mlp.down_proj", + "ppl": 4.030179977416992 + }, + { + "layer": "layer.68.self_attn.q_proj", + "ppl": 4.030320167541504 + }, + { + "layer": "layer.68.self_attn.k_proj", + "ppl": 4.030340194702148 + }, + { + "layer": "layer.68.self_attn.v_proj", + "ppl": 4.030341148376465 + }, + { + "layer": "layer.68.self_attn.o_proj", + "ppl": 4.030383586883545 + }, + { + "layer": "layer.68.mlp.gate_proj", + "ppl": 4.0300445556640625 + }, + { + "layer": "layer.68.mlp.up_proj", + "ppl": 4.030239582061768 + }, + { + "layer": "layer.68.mlp.down_proj", + "ppl": 4.030277252197266 + }, + { + "layer": "layer.69.self_attn.q_proj", + "ppl": 4.030329704284668 + }, + { + "layer": "layer.69.self_attn.k_proj", + "ppl": 4.030404567718506 + }, + { + "layer": "layer.69.self_attn.v_proj", + "ppl": 4.030323028564453 + }, + { + "layer": "layer.69.self_attn.o_proj", + "ppl": 4.0302910804748535 + }, + { + "layer": "layer.69.mlp.gate_proj", + "ppl": 4.0303544998168945 + }, + { + "layer": "layer.69.mlp.up_proj", + "ppl": 4.03025484085083 + }, + { + "layer": "layer.69.mlp.down_proj", + "ppl": 4.030143737792969 + }, + { + "layer": "layer.70.self_attn.q_proj", + "ppl": 4.0303544998168945 + }, + { + "layer": "layer.70.self_attn.k_proj", + "ppl": 4.0304131507873535 + }, + { + "layer": "layer.70.self_attn.v_proj", + "ppl": 4.030371189117432 + }, + { + "layer": "layer.70.self_attn.o_proj", + "ppl": 4.030219554901123 + }, + { + "layer": "layer.70.mlp.gate_proj", + "ppl": 4.030200004577637 + }, + { + "layer": "layer.70.mlp.up_proj", + "ppl": 4.030319690704346 + }, + { + "layer": "layer.70.mlp.down_proj", + "ppl": 4.03039026260376 + }, + { + "layer": "layer.71.self_attn.q_proj", + "ppl": 4.030291557312012 + }, + { + "layer": "layer.71.self_attn.k_proj", + "ppl": 4.030287265777588 + }, + { + "layer": "layer.71.self_attn.v_proj", + "ppl": 4.030355453491211 + }, + { + "layer": "layer.71.self_attn.o_proj", + "ppl": 4.0303263664245605 + }, + { + "layer": "layer.71.mlp.gate_proj", + "ppl": 4.030383586883545 + }, + { + "layer": "layer.71.mlp.up_proj", + "ppl": 4.0303955078125 + }, + { + "layer": "layer.71.mlp.down_proj", + "ppl": 4.030406475067139 + }, + { + "layer": "layer.72.self_attn.q_proj", + "ppl": 4.030257701873779 + }, + { + "layer": "layer.72.self_attn.k_proj", + "ppl": 4.030374050140381 + }, + { + "layer": "layer.72.self_attn.v_proj", + "ppl": 4.030521392822266 + }, + { + "layer": "layer.72.self_attn.o_proj", + "ppl": 4.030257701873779 + }, + { + "layer": "layer.72.mlp.gate_proj", + "ppl": 4.030340194702148 + }, + { + "layer": "layer.72.mlp.up_proj", + "ppl": 4.030359745025635 + }, + { + "layer": "layer.72.mlp.down_proj", + "ppl": 4.03047513961792 + }, + { + "layer": "layer.73.self_attn.q_proj", + "ppl": 4.0303826332092285 + }, + { + "layer": "layer.73.self_attn.k_proj", + "ppl": 4.030323028564453 + }, + { + "layer": "layer.73.self_attn.v_proj", + "ppl": 4.030339241027832 + }, + { + "layer": "layer.73.self_attn.o_proj", + "ppl": 4.030375003814697 + }, + { + "layer": "layer.73.mlp.gate_proj", + "ppl": 4.03014612197876 + }, + { + "layer": "layer.73.mlp.up_proj", + "ppl": 4.030154705047607 + }, + { + "layer": "layer.73.mlp.down_proj", + "ppl": 4.030059814453125 + }, + { + "layer": "layer.74.self_attn.q_proj", + "ppl": 4.030351161956787 + }, + { + "layer": "layer.74.self_attn.k_proj", + "ppl": 4.030394077301025 + }, + { + "layer": "layer.74.self_attn.v_proj", + "ppl": 4.0302815437316895 + }, + { + "layer": "layer.74.self_attn.o_proj", + "ppl": 4.030290603637695 + }, + { + "layer": "layer.74.mlp.gate_proj", + "ppl": 4.030221939086914 + }, + { + "layer": "layer.74.mlp.up_proj", + "ppl": 4.030270099639893 + }, + { + "layer": "layer.74.mlp.down_proj", + "ppl": 4.030185699462891 + }, + { + "layer": "layer.75.self_attn.q_proj", + "ppl": 4.030317306518555 + }, + { + "layer": "layer.75.self_attn.k_proj", + "ppl": 4.0303955078125 + }, + { + "layer": "layer.75.self_attn.v_proj", + "ppl": 4.030362129211426 + }, + { + "layer": "layer.75.self_attn.o_proj", + "ppl": 4.03034782409668 + }, + { + "layer": "layer.75.mlp.gate_proj", + "ppl": 4.030210971832275 + }, + { + "layer": "layer.75.mlp.up_proj", + "ppl": 4.030308246612549 + }, + { + "layer": "layer.75.mlp.down_proj", + "ppl": 4.030346870422363 + }, + { + "layer": "layer.76.self_attn.q_proj", + "ppl": 4.030303478240967 + }, + { + "layer": "layer.76.self_attn.k_proj", + "ppl": 4.030361652374268 + }, + { + "layer": "layer.76.self_attn.v_proj", + "ppl": 4.030385971069336 + }, + { + "layer": "layer.76.self_attn.o_proj", + "ppl": 4.03034782409668 + }, + { + "layer": "layer.76.mlp.gate_proj", + "ppl": 4.030150413513184 + }, + { + "layer": "layer.76.mlp.up_proj", + "ppl": 4.030252933502197 + }, + { + "layer": "layer.76.mlp.down_proj", + "ppl": 4.030365467071533 + }, + { + "layer": "layer.77.self_attn.q_proj", + "ppl": 4.030344486236572 + }, + { + "layer": "layer.77.self_attn.k_proj", + "ppl": 4.030400276184082 + }, + { + "layer": "layer.77.self_attn.v_proj", + "ppl": 4.030399799346924 + }, + { + "layer": "layer.77.self_attn.o_proj", + "ppl": 4.030374526977539 + }, + { + "layer": "layer.77.mlp.gate_proj", + "ppl": 4.0301513671875 + }, + { + "layer": "layer.77.mlp.up_proj", + "ppl": 4.030239105224609 + }, + { + "layer": "layer.77.mlp.down_proj", + "ppl": 4.0303239822387695 + }, + { + "layer": "layer.78.self_attn.q_proj", + "ppl": 4.030406475067139 + }, + { + "layer": "layer.78.self_attn.k_proj", + "ppl": 4.030396461486816 + }, + { + "layer": "layer.78.self_attn.v_proj", + "ppl": 4.030395984649658 + }, + { + "layer": "layer.78.self_attn.o_proj", + "ppl": 4.030366897583008 + }, + { + "layer": "layer.78.mlp.gate_proj", + "ppl": 4.030157089233398 + }, + { + "layer": "layer.78.mlp.up_proj", + "ppl": 4.030274391174316 + }, + { + "layer": "layer.78.mlp.down_proj", + "ppl": 4.030174255371094 + }, + { + "layer": "layer.79.self_attn.q_proj", + "ppl": 4.0302958488464355 + }, + { + "layer": "layer.79.self_attn.k_proj", + "ppl": 4.030288219451904 + }, + { + "layer": "layer.79.self_attn.v_proj", + "ppl": 4.030412197113037 + }, + { + "layer": "layer.79.self_attn.o_proj", + "ppl": 4.030329704284668 + }, + { + "layer": "layer.79.mlp.gate_proj", + "ppl": 4.030055522918701 + }, + { + "layer": "layer.79.mlp.up_proj", + "ppl": 4.030065536499023 + }, + { + "layer": "layer.79.mlp.down_proj", + "ppl": 4.029688358306885 + } + ] + }, + { + "bit": 14, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 4.021470069885254 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 4.021470546722412 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 4.021481037139893 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 4.021359443664551 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 4.0213236808776855 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 4.021483421325684 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 4.0215678215026855 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 4.02144718170166 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 4.021459579467773 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 4.021440505981445 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 4.021522045135498 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 4.02142333984375 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 4.021431922912598 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 4.021459579467773 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 4.021482944488525 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 4.021308422088623 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 4.021378040313721 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 4.021409034729004 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 4.021542549133301 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 4.021379470825195 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 4.021350860595703 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 4.021462440490723 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 4.021394729614258 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 4.021424293518066 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 4.0213823318481445 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 4.021325588226318 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 4.021440505981445 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 4.021463871002197 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 4.021413803100586 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 4.021430492401123 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 4.021367073059082 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 4.021451950073242 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 4.021511554718018 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 4.0213303565979 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 4.021387100219727 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 4.021495342254639 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 4.021398544311523 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 4.021435737609863 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 4.021351337432861 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 4.021452903747559 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 4.021545886993408 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 4.021418571472168 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 4.021402359008789 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 4.021404266357422 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 4.0213422775268555 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 4.0213446617126465 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 4.021413803100586 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 4.021387577056885 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 4.021379470825195 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 4.021459102630615 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 4.021430015563965 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 4.021459579467773 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 4.021445274353027 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 4.0214385986328125 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 4.021356105804443 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 4.021442890167236 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 4.021388530731201 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 4.021406173706055 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 4.021371841430664 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 4.02131986618042 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 4.021420955657959 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 4.021377086639404 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 4.021409034729004 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 4.021406650543213 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 4.02145528793335 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 4.021486282348633 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 4.0213541984558105 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 4.021457195281982 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 4.021430969238281 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 4.02142858505249 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 4.021411895751953 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 4.021430015563965 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 4.021378040313721 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 4.021409034729004 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 4.02145528793335 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 4.021420001983643 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 4.021388053894043 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 4.021475315093994 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 4.021350860595703 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 4.021386623382568 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 4.021286964416504 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 4.021360874176025 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 4.02142858505249 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 4.021319389343262 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 4.021514892578125 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 4.021327018737793 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 4.021428108215332 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 4.021461486816406 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 4.021449565887451 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 4.021382808685303 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 4.021369457244873 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 4.021458148956299 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 4.02146577835083 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 4.021458148956299 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 4.021478176116943 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 4.021345138549805 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 4.021368503570557 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 4.021470069885254 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 4.021365165710449 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 4.021442890167236 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 4.021406173706055 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 4.021240234375 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 4.0214033126831055 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 4.021417617797852 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 4.021430492401123 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 4.021457195281982 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 4.021355152130127 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 4.021397113800049 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 4.0213446617126465 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 4.021478176116943 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 4.021368503570557 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 4.0214409828186035 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 4.0214080810546875 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 4.021480083465576 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 4.021417617797852 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 4.021351337432861 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 4.021396160125732 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 4.021551609039307 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 4.02142333984375 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 4.021415710449219 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 4.02126407623291 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 4.02137565612793 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 4.021430969238281 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 4.021337985992432 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 4.021453857421875 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 4.021482944488525 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 4.021420478820801 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 4.02147912979126 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 4.0213704109191895 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 4.021353244781494 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 4.0214104652404785 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 4.021486759185791 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 4.02136754989624 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 4.021529674530029 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 4.021376132965088 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 4.021477699279785 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 4.021448612213135 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 4.021398544311523 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 4.021296977996826 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 4.021585941314697 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 4.021384239196777 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 4.021467685699463 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 4.021422386169434 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 4.021416664123535 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 4.021500587463379 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 4.021480560302734 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 4.021287441253662 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 4.0214128494262695 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 4.021473407745361 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 4.021437644958496 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 4.021478652954102 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 4.021442890167236 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 4.021364212036133 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 4.021512508392334 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 4.021401405334473 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 4.021422863006592 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 4.021380424499512 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 4.021390914916992 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 4.021330833435059 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 4.021349906921387 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 4.021444320678711 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 4.021364212036133 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 4.02129602432251 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 4.021464824676514 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 4.021445274353027 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 4.021483898162842 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 4.021373748779297 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 4.02136754989624 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 4.021430969238281 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 4.021470546722412 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 4.021364212036133 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 4.021449565887451 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 4.021284103393555 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 4.021452903747559 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 4.021682262420654 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 4.021364212036133 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 4.021413803100586 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 4.02133846282959 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 4.0213541984558105 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 4.021459579467773 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 4.0213541984558105 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 4.02137565612793 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 4.021285057067871 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 4.021425724029541 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 4.0214433670043945 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 4.021290302276611 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 4.02137565612793 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 4.021337985992432 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 4.021530628204346 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 4.021424293518066 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 4.021418571472168 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 4.0213398933410645 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 4.021422863006592 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 4.021342754364014 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 4.02125883102417 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 4.021395683288574 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 4.021492004394531 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 4.021413803100586 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 4.021442890167236 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 4.021398067474365 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 4.021427154541016 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 4.021425247192383 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 4.021346569061279 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 4.021481990814209 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 4.021576881408691 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 4.021451950073242 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 4.021378040313721 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 4.021564483642578 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 4.021548271179199 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 4.021346569061279 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 4.021485805511475 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 4.021392345428467 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 4.021330833435059 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 4.021352291107178 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 4.021247863769531 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 4.021383285522461 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 4.021490573883057 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 4.021293640136719 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 4.0215067863464355 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 4.02146053314209 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 4.02146053314209 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 4.02157735824585 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 4.021600723266602 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 4.021335601806641 + }, + { + "layer": "layer.32.self_attn.q_proj", + "ppl": 4.021362781524658 + }, + { + "layer": "layer.32.self_attn.k_proj", + "ppl": 4.021405220031738 + }, + { + "layer": "layer.32.self_attn.v_proj", + "ppl": 4.021481513977051 + }, + { + "layer": "layer.32.self_attn.o_proj", + "ppl": 4.021459579467773 + }, + { + "layer": "layer.32.mlp.gate_proj", + "ppl": 4.021501541137695 + }, + { + "layer": "layer.32.mlp.up_proj", + "ppl": 4.021383285522461 + }, + { + "layer": "layer.32.mlp.down_proj", + "ppl": 4.021312713623047 + }, + { + "layer": "layer.33.self_attn.q_proj", + "ppl": 4.021437644958496 + }, + { + "layer": "layer.33.self_attn.k_proj", + "ppl": 4.021391868591309 + }, + { + "layer": "layer.33.self_attn.v_proj", + "ppl": 4.021474361419678 + }, + { + "layer": "layer.33.self_attn.o_proj", + "ppl": 4.0214667320251465 + }, + { + "layer": "layer.33.mlp.gate_proj", + "ppl": 4.02130651473999 + }, + { + "layer": "layer.33.mlp.up_proj", + "ppl": 4.021430015563965 + }, + { + "layer": "layer.33.mlp.down_proj", + "ppl": 4.021267890930176 + }, + { + "layer": "layer.34.self_attn.q_proj", + "ppl": 4.021340847015381 + }, + { + "layer": "layer.34.self_attn.k_proj", + "ppl": 4.021327018737793 + }, + { + "layer": "layer.34.self_attn.v_proj", + "ppl": 4.021483898162842 + }, + { + "layer": "layer.34.self_attn.o_proj", + "ppl": 4.021318435668945 + }, + { + "layer": "layer.34.mlp.gate_proj", + "ppl": 4.02146053314209 + }, + { + "layer": "layer.34.mlp.up_proj", + "ppl": 4.021503448486328 + }, + { + "layer": "layer.34.mlp.down_proj", + "ppl": 4.021349906921387 + }, + { + "layer": "layer.35.self_attn.q_proj", + "ppl": 4.0214033126831055 + }, + { + "layer": "layer.35.self_attn.k_proj", + "ppl": 4.021484375 + }, + { + "layer": "layer.35.self_attn.v_proj", + "ppl": 4.021462440490723 + }, + { + "layer": "layer.35.self_attn.o_proj", + "ppl": 4.021394729614258 + }, + { + "layer": "layer.35.mlp.gate_proj", + "ppl": 4.021573066711426 + }, + { + "layer": "layer.35.mlp.up_proj", + "ppl": 4.021308422088623 + }, + { + "layer": "layer.35.mlp.down_proj", + "ppl": 4.021296977996826 + }, + { + "layer": "layer.36.self_attn.q_proj", + "ppl": 4.0214972496032715 + }, + { + "layer": "layer.36.self_attn.k_proj", + "ppl": 4.021407127380371 + }, + { + "layer": "layer.36.self_attn.v_proj", + "ppl": 4.021463871002197 + }, + { + "layer": "layer.36.self_attn.o_proj", + "ppl": 4.0214924812316895 + }, + { + "layer": "layer.36.mlp.gate_proj", + "ppl": 4.021489143371582 + }, + { + "layer": "layer.36.mlp.up_proj", + "ppl": 4.021551609039307 + }, + { + "layer": "layer.36.mlp.down_proj", + "ppl": 4.021243095397949 + }, + { + "layer": "layer.37.self_attn.q_proj", + "ppl": 4.021491527557373 + }, + { + "layer": "layer.37.self_attn.k_proj", + "ppl": 4.021444320678711 + }, + { + "layer": "layer.37.self_attn.v_proj", + "ppl": 4.021416187286377 + }, + { + "layer": "layer.37.self_attn.o_proj", + "ppl": 4.021356105804443 + }, + { + "layer": "layer.37.mlp.gate_proj", + "ppl": 4.02144718170166 + }, + { + "layer": "layer.37.mlp.up_proj", + "ppl": 4.021373748779297 + }, + { + "layer": "layer.37.mlp.down_proj", + "ppl": 4.021404266357422 + }, + { + "layer": "layer.38.self_attn.q_proj", + "ppl": 4.021432876586914 + }, + { + "layer": "layer.38.self_attn.k_proj", + "ppl": 4.0216217041015625 + }, + { + "layer": "layer.38.self_attn.v_proj", + "ppl": 4.021392345428467 + }, + { + "layer": "layer.38.self_attn.o_proj", + "ppl": 4.021544933319092 + }, + { + "layer": "layer.38.mlp.gate_proj", + "ppl": 4.0214762687683105 + }, + { + "layer": "layer.38.mlp.up_proj", + "ppl": 4.021422863006592 + }, + { + "layer": "layer.38.mlp.down_proj", + "ppl": 4.021478176116943 + }, + { + "layer": "layer.39.self_attn.q_proj", + "ppl": 4.021521091461182 + }, + { + "layer": "layer.39.self_attn.k_proj", + "ppl": 4.021411895751953 + }, + { + "layer": "layer.39.self_attn.v_proj", + "ppl": 4.021353244781494 + }, + { + "layer": "layer.39.self_attn.o_proj", + "ppl": 4.021391868591309 + }, + { + "layer": "layer.39.mlp.gate_proj", + "ppl": 4.021478652954102 + }, + { + "layer": "layer.39.mlp.up_proj", + "ppl": 4.021396636962891 + }, + { + "layer": "layer.39.mlp.down_proj", + "ppl": 4.0214128494262695 + }, + { + "layer": "layer.40.self_attn.q_proj", + "ppl": 4.021377086639404 + }, + { + "layer": "layer.40.self_attn.k_proj", + "ppl": 4.021476745605469 + }, + { + "layer": "layer.40.self_attn.v_proj", + "ppl": 4.021496295928955 + }, + { + "layer": "layer.40.self_attn.o_proj", + "ppl": 4.02142333984375 + }, + { + "layer": "layer.40.mlp.gate_proj", + "ppl": 4.021390914916992 + }, + { + "layer": "layer.40.mlp.up_proj", + "ppl": 4.0213704109191895 + }, + { + "layer": "layer.40.mlp.down_proj", + "ppl": 4.0215840339660645 + }, + { + "layer": "layer.41.self_attn.q_proj", + "ppl": 4.021426200866699 + }, + { + "layer": "layer.41.self_attn.k_proj", + "ppl": 4.021396160125732 + }, + { + "layer": "layer.41.self_attn.v_proj", + "ppl": 4.021357536315918 + }, + { + "layer": "layer.41.self_attn.o_proj", + "ppl": 4.021380424499512 + }, + { + "layer": "layer.41.mlp.gate_proj", + "ppl": 4.021591663360596 + }, + { + "layer": "layer.41.mlp.up_proj", + "ppl": 4.021328926086426 + }, + { + "layer": "layer.41.mlp.down_proj", + "ppl": 4.021581172943115 + }, + { + "layer": "layer.42.self_attn.q_proj", + "ppl": 4.021407604217529 + }, + { + "layer": "layer.42.self_attn.k_proj", + "ppl": 4.021397590637207 + }, + { + "layer": "layer.42.self_attn.v_proj", + "ppl": 4.021567344665527 + }, + { + "layer": "layer.42.self_attn.o_proj", + "ppl": 4.021432876586914 + }, + { + "layer": "layer.42.mlp.gate_proj", + "ppl": 4.0214433670043945 + }, + { + "layer": "layer.42.mlp.up_proj", + "ppl": 4.021383285522461 + }, + { + "layer": "layer.42.mlp.down_proj", + "ppl": 4.021379470825195 + }, + { + "layer": "layer.43.self_attn.q_proj", + "ppl": 4.021368503570557 + }, + { + "layer": "layer.43.self_attn.k_proj", + "ppl": 4.021392345428467 + }, + { + "layer": "layer.43.self_attn.v_proj", + "ppl": 4.021544933319092 + }, + { + "layer": "layer.43.self_attn.o_proj", + "ppl": 4.021490573883057 + }, + { + "layer": "layer.43.mlp.gate_proj", + "ppl": 4.021203994750977 + }, + { + "layer": "layer.43.mlp.up_proj", + "ppl": 4.021290302276611 + }, + { + "layer": "layer.43.mlp.down_proj", + "ppl": 4.021487712860107 + }, + { + "layer": "layer.44.self_attn.q_proj", + "ppl": 4.021417140960693 + }, + { + "layer": "layer.44.self_attn.k_proj", + "ppl": 4.0214314460754395 + }, + { + "layer": "layer.44.self_attn.v_proj", + "ppl": 4.0214691162109375 + }, + { + "layer": "layer.44.self_attn.o_proj", + "ppl": 4.021434307098389 + }, + { + "layer": "layer.44.mlp.gate_proj", + "ppl": 4.021271705627441 + }, + { + "layer": "layer.44.mlp.up_proj", + "ppl": 4.021376132965088 + }, + { + "layer": "layer.44.mlp.down_proj", + "ppl": 4.0212016105651855 + }, + { + "layer": "layer.45.self_attn.q_proj", + "ppl": 4.021422386169434 + }, + { + "layer": "layer.45.self_attn.k_proj", + "ppl": 4.021447658538818 + }, + { + "layer": "layer.45.self_attn.v_proj", + "ppl": 4.021427631378174 + }, + { + "layer": "layer.45.self_attn.o_proj", + "ppl": 4.0215349197387695 + }, + { + "layer": "layer.45.mlp.gate_proj", + "ppl": 4.021371364593506 + }, + { + "layer": "layer.45.mlp.up_proj", + "ppl": 4.021144390106201 + }, + { + "layer": "layer.45.mlp.down_proj", + "ppl": 4.021408557891846 + }, + { + "layer": "layer.46.self_attn.q_proj", + "ppl": 4.021453857421875 + }, + { + "layer": "layer.46.self_attn.k_proj", + "ppl": 4.021280765533447 + }, + { + "layer": "layer.46.self_attn.v_proj", + "ppl": 4.02137565612793 + }, + { + "layer": "layer.46.self_attn.o_proj", + "ppl": 4.021392345428467 + }, + { + "layer": "layer.46.mlp.gate_proj", + "ppl": 4.021350860595703 + }, + { + "layer": "layer.46.mlp.up_proj", + "ppl": 4.021554946899414 + }, + { + "layer": "layer.46.mlp.down_proj", + "ppl": 4.02132511138916 + }, + { + "layer": "layer.47.self_attn.q_proj", + "ppl": 4.021387100219727 + }, + { + "layer": "layer.47.self_attn.k_proj", + "ppl": 4.021424293518066 + }, + { + "layer": "layer.47.self_attn.v_proj", + "ppl": 4.02141809463501 + }, + { + "layer": "layer.47.self_attn.o_proj", + "ppl": 4.021289348602295 + }, + { + "layer": "layer.47.mlp.gate_proj", + "ppl": 4.021328449249268 + }, + { + "layer": "layer.47.mlp.up_proj", + "ppl": 4.021412372589111 + }, + { + "layer": "layer.47.mlp.down_proj", + "ppl": 4.021490097045898 + }, + { + "layer": "layer.48.self_attn.q_proj", + "ppl": 4.021377086639404 + }, + { + "layer": "layer.48.self_attn.k_proj", + "ppl": 4.021352291107178 + }, + { + "layer": "layer.48.self_attn.v_proj", + "ppl": 4.021476745605469 + }, + { + "layer": "layer.48.self_attn.o_proj", + "ppl": 4.02139949798584 + }, + { + "layer": "layer.48.mlp.gate_proj", + "ppl": 4.021327972412109 + }, + { + "layer": "layer.48.mlp.up_proj", + "ppl": 4.021557807922363 + }, + { + "layer": "layer.48.mlp.down_proj", + "ppl": 4.0214691162109375 + }, + { + "layer": "layer.49.self_attn.q_proj", + "ppl": 4.021389007568359 + }, + { + "layer": "layer.49.self_attn.k_proj", + "ppl": 4.021337032318115 + }, + { + "layer": "layer.49.self_attn.v_proj", + "ppl": 4.0213518142700195 + }, + { + "layer": "layer.49.self_attn.o_proj", + "ppl": 4.021397113800049 + }, + { + "layer": "layer.49.mlp.gate_proj", + "ppl": 4.021422386169434 + }, + { + "layer": "layer.49.mlp.up_proj", + "ppl": 4.0213189125061035 + }, + { + "layer": "layer.49.mlp.down_proj", + "ppl": 4.021265506744385 + }, + { + "layer": "layer.50.self_attn.q_proj", + "ppl": 4.021395206451416 + }, + { + "layer": "layer.50.self_attn.k_proj", + "ppl": 4.021308898925781 + }, + { + "layer": "layer.50.self_attn.v_proj", + "ppl": 4.021355152130127 + }, + { + "layer": "layer.50.self_attn.o_proj", + "ppl": 4.021312713623047 + }, + { + "layer": "layer.50.mlp.gate_proj", + "ppl": 4.0214924812316895 + }, + { + "layer": "layer.50.mlp.up_proj", + "ppl": 4.021374702453613 + }, + { + "layer": "layer.50.mlp.down_proj", + "ppl": 4.021165370941162 + }, + { + "layer": "layer.51.self_attn.q_proj", + "ppl": 4.021373748779297 + }, + { + "layer": "layer.51.self_attn.k_proj", + "ppl": 4.021292686462402 + }, + { + "layer": "layer.51.self_attn.v_proj", + "ppl": 4.021247863769531 + }, + { + "layer": "layer.51.self_attn.o_proj", + "ppl": 4.0213541984558105 + }, + { + "layer": "layer.51.mlp.gate_proj", + "ppl": 4.021261215209961 + }, + { + "layer": "layer.51.mlp.up_proj", + "ppl": 4.0211992263793945 + }, + { + "layer": "layer.51.mlp.down_proj", + "ppl": 4.021401405334473 + }, + { + "layer": "layer.52.self_attn.q_proj", + "ppl": 4.021439552307129 + }, + { + "layer": "layer.52.self_attn.k_proj", + "ppl": 4.021267890930176 + }, + { + "layer": "layer.52.self_attn.v_proj", + "ppl": 4.021328449249268 + }, + { + "layer": "layer.52.self_attn.o_proj", + "ppl": 4.0215349197387695 + }, + { + "layer": "layer.52.mlp.gate_proj", + "ppl": 4.021297931671143 + }, + { + "layer": "layer.52.mlp.up_proj", + "ppl": 4.021361827850342 + }, + { + "layer": "layer.52.mlp.down_proj", + "ppl": 4.021374702453613 + }, + { + "layer": "layer.53.self_attn.q_proj", + "ppl": 4.021340847015381 + }, + { + "layer": "layer.53.self_attn.k_proj", + "ppl": 4.021405220031738 + }, + { + "layer": "layer.53.self_attn.v_proj", + "ppl": 4.021419525146484 + }, + { + "layer": "layer.53.self_attn.o_proj", + "ppl": 4.021340847015381 + }, + { + "layer": "layer.53.mlp.gate_proj", + "ppl": 4.0213303565979 + }, + { + "layer": "layer.53.mlp.up_proj", + "ppl": 4.021462440490723 + }, + { + "layer": "layer.53.mlp.down_proj", + "ppl": 4.02139949798584 + }, + { + "layer": "layer.54.self_attn.q_proj", + "ppl": 4.021482467651367 + }, + { + "layer": "layer.54.self_attn.k_proj", + "ppl": 4.0213303565979 + }, + { + "layer": "layer.54.self_attn.v_proj", + "ppl": 4.021240234375 + }, + { + "layer": "layer.54.self_attn.o_proj", + "ppl": 4.021300315856934 + }, + { + "layer": "layer.54.mlp.gate_proj", + "ppl": 4.021226406097412 + }, + { + "layer": "layer.54.mlp.up_proj", + "ppl": 4.021373748779297 + }, + { + "layer": "layer.54.mlp.down_proj", + "ppl": 4.021413803100586 + }, + { + "layer": "layer.55.self_attn.q_proj", + "ppl": 4.021523952484131 + }, + { + "layer": "layer.55.self_attn.k_proj", + "ppl": 4.02144193649292 + }, + { + "layer": "layer.55.self_attn.v_proj", + "ppl": 4.02137565612793 + }, + { + "layer": "layer.55.self_attn.o_proj", + "ppl": 4.021392345428467 + }, + { + "layer": "layer.55.mlp.gate_proj", + "ppl": 4.0214033126831055 + }, + { + "layer": "layer.55.mlp.up_proj", + "ppl": 4.021481037139893 + }, + { + "layer": "layer.55.mlp.down_proj", + "ppl": 4.021398544311523 + }, + { + "layer": "layer.56.self_attn.q_proj", + "ppl": 4.021391868591309 + }, + { + "layer": "layer.56.self_attn.k_proj", + "ppl": 4.021437644958496 + }, + { + "layer": "layer.56.self_attn.v_proj", + "ppl": 4.021289348602295 + }, + { + "layer": "layer.56.self_attn.o_proj", + "ppl": 4.021366119384766 + }, + { + "layer": "layer.56.mlp.gate_proj", + "ppl": 4.021214962005615 + }, + { + "layer": "layer.56.mlp.up_proj", + "ppl": 4.021463871002197 + }, + { + "layer": "layer.56.mlp.down_proj", + "ppl": 4.021392345428467 + }, + { + "layer": "layer.57.self_attn.q_proj", + "ppl": 4.021404266357422 + }, + { + "layer": "layer.57.self_attn.k_proj", + "ppl": 4.021353244781494 + }, + { + "layer": "layer.57.self_attn.v_proj", + "ppl": 4.021353244781494 + }, + { + "layer": "layer.57.self_attn.o_proj", + "ppl": 4.021402359008789 + }, + { + "layer": "layer.57.mlp.gate_proj", + "ppl": 4.021297931671143 + }, + { + "layer": "layer.57.mlp.up_proj", + "ppl": 4.02144193649292 + }, + { + "layer": "layer.57.mlp.down_proj", + "ppl": 4.021285057067871 + }, + { + "layer": "layer.58.self_attn.q_proj", + "ppl": 4.021324634552002 + }, + { + "layer": "layer.58.self_attn.k_proj", + "ppl": 4.021389007568359 + }, + { + "layer": "layer.58.self_attn.v_proj", + "ppl": 4.021334648132324 + }, + { + "layer": "layer.58.self_attn.o_proj", + "ppl": 4.021507740020752 + }, + { + "layer": "layer.58.mlp.gate_proj", + "ppl": 4.021459579467773 + }, + { + "layer": "layer.58.mlp.up_proj", + "ppl": 4.021389961242676 + }, + { + "layer": "layer.58.mlp.down_proj", + "ppl": 4.021417140960693 + }, + { + "layer": "layer.59.self_attn.q_proj", + "ppl": 4.021356105804443 + }, + { + "layer": "layer.59.self_attn.k_proj", + "ppl": 4.021363258361816 + }, + { + "layer": "layer.59.self_attn.v_proj", + "ppl": 4.021463871002197 + }, + { + "layer": "layer.59.self_attn.o_proj", + "ppl": 4.021244525909424 + }, + { + "layer": "layer.59.mlp.gate_proj", + "ppl": 4.02143669128418 + }, + { + "layer": "layer.59.mlp.up_proj", + "ppl": 4.021496295928955 + }, + { + "layer": "layer.59.mlp.down_proj", + "ppl": 4.021273136138916 + }, + { + "layer": "layer.60.self_attn.q_proj", + "ppl": 4.021313667297363 + }, + { + "layer": "layer.60.self_attn.k_proj", + "ppl": 4.021432399749756 + }, + { + "layer": "layer.60.self_attn.v_proj", + "ppl": 4.021346569061279 + }, + { + "layer": "layer.60.self_attn.o_proj", + "ppl": 4.021481990814209 + }, + { + "layer": "layer.60.mlp.gate_proj", + "ppl": 4.021157741546631 + }, + { + "layer": "layer.60.mlp.up_proj", + "ppl": 4.021362781524658 + }, + { + "layer": "layer.60.mlp.down_proj", + "ppl": 4.021450519561768 + }, + { + "layer": "layer.61.self_attn.q_proj", + "ppl": 4.021401405334473 + }, + { + "layer": "layer.61.self_attn.k_proj", + "ppl": 4.021391868591309 + }, + { + "layer": "layer.61.self_attn.v_proj", + "ppl": 4.021398544311523 + }, + { + "layer": "layer.61.self_attn.o_proj", + "ppl": 4.0214691162109375 + }, + { + "layer": "layer.61.mlp.gate_proj", + "ppl": 4.02126407623291 + }, + { + "layer": "layer.61.mlp.up_proj", + "ppl": 4.021342754364014 + }, + { + "layer": "layer.61.mlp.down_proj", + "ppl": 4.021483898162842 + }, + { + "layer": "layer.62.self_attn.q_proj", + "ppl": 4.021417617797852 + }, + { + "layer": "layer.62.self_attn.k_proj", + "ppl": 4.021505832672119 + }, + { + "layer": "layer.62.self_attn.v_proj", + "ppl": 4.02142333984375 + }, + { + "layer": "layer.62.self_attn.o_proj", + "ppl": 4.021483898162842 + }, + { + "layer": "layer.62.mlp.gate_proj", + "ppl": 4.021394729614258 + }, + { + "layer": "layer.62.mlp.up_proj", + "ppl": 4.021351337432861 + }, + { + "layer": "layer.62.mlp.down_proj", + "ppl": 4.021263122558594 + }, + { + "layer": "layer.63.self_attn.q_proj", + "ppl": 4.021358489990234 + }, + { + "layer": "layer.63.self_attn.k_proj", + "ppl": 4.021407127380371 + }, + { + "layer": "layer.63.self_attn.v_proj", + "ppl": 4.021442890167236 + }, + { + "layer": "layer.63.self_attn.o_proj", + "ppl": 4.021467208862305 + }, + { + "layer": "layer.63.mlp.gate_proj", + "ppl": 4.021413326263428 + }, + { + "layer": "layer.63.mlp.up_proj", + "ppl": 4.021454811096191 + }, + { + "layer": "layer.63.mlp.down_proj", + "ppl": 4.021328926086426 + }, + { + "layer": "layer.64.self_attn.q_proj", + "ppl": 4.021445274353027 + }, + { + "layer": "layer.64.self_attn.k_proj", + "ppl": 4.021401405334473 + }, + { + "layer": "layer.64.self_attn.v_proj", + "ppl": 4.021390914916992 + }, + { + "layer": "layer.64.self_attn.o_proj", + "ppl": 4.021389961242676 + }, + { + "layer": "layer.64.mlp.gate_proj", + "ppl": 4.021235942840576 + }, + { + "layer": "layer.64.mlp.up_proj", + "ppl": 4.021193981170654 + }, + { + "layer": "layer.64.mlp.down_proj", + "ppl": 4.021223068237305 + }, + { + "layer": "layer.65.self_attn.q_proj", + "ppl": 4.021368503570557 + }, + { + "layer": "layer.65.self_attn.k_proj", + "ppl": 4.021378040313721 + }, + { + "layer": "layer.65.self_attn.v_proj", + "ppl": 4.021405220031738 + }, + { + "layer": "layer.65.self_attn.o_proj", + "ppl": 4.021574974060059 + }, + { + "layer": "layer.65.mlp.gate_proj", + "ppl": 4.021251678466797 + }, + { + "layer": "layer.65.mlp.up_proj", + "ppl": 4.021337032318115 + }, + { + "layer": "layer.65.mlp.down_proj", + "ppl": 4.021367073059082 + }, + { + "layer": "layer.66.self_attn.q_proj", + "ppl": 4.021379470825195 + }, + { + "layer": "layer.66.self_attn.k_proj", + "ppl": 4.021450519561768 + }, + { + "layer": "layer.66.self_attn.v_proj", + "ppl": 4.021488666534424 + }, + { + "layer": "layer.66.self_attn.o_proj", + "ppl": 4.021434783935547 + }, + { + "layer": "layer.66.mlp.gate_proj", + "ppl": 4.021449565887451 + }, + { + "layer": "layer.66.mlp.up_proj", + "ppl": 4.021329402923584 + }, + { + "layer": "layer.66.mlp.down_proj", + "ppl": 4.021524906158447 + }, + { + "layer": "layer.67.self_attn.q_proj", + "ppl": 4.021434783935547 + }, + { + "layer": "layer.67.self_attn.k_proj", + "ppl": 4.021434783935547 + }, + { + "layer": "layer.67.self_attn.v_proj", + "ppl": 4.021485328674316 + }, + { + "layer": "layer.67.self_attn.o_proj", + "ppl": 4.021353244781494 + }, + { + "layer": "layer.67.mlp.gate_proj", + "ppl": 4.021366119384766 + }, + { + "layer": "layer.67.mlp.up_proj", + "ppl": 4.021405220031738 + }, + { + "layer": "layer.67.mlp.down_proj", + "ppl": 4.0215911865234375 + }, + { + "layer": "layer.68.self_attn.q_proj", + "ppl": 4.021448612213135 + }, + { + "layer": "layer.68.self_attn.k_proj", + "ppl": 4.021378040313721 + }, + { + "layer": "layer.68.self_attn.v_proj", + "ppl": 4.021330833435059 + }, + { + "layer": "layer.68.self_attn.o_proj", + "ppl": 4.021350860595703 + }, + { + "layer": "layer.68.mlp.gate_proj", + "ppl": 4.021448612213135 + }, + { + "layer": "layer.68.mlp.up_proj", + "ppl": 4.021453857421875 + }, + { + "layer": "layer.68.mlp.down_proj", + "ppl": 4.0212836265563965 + }, + { + "layer": "layer.69.self_attn.q_proj", + "ppl": 4.021459579467773 + }, + { + "layer": "layer.69.self_attn.k_proj", + "ppl": 4.021397590637207 + }, + { + "layer": "layer.69.self_attn.v_proj", + "ppl": 4.02144718170166 + }, + { + "layer": "layer.69.self_attn.o_proj", + "ppl": 4.021417617797852 + }, + { + "layer": "layer.69.mlp.gate_proj", + "ppl": 4.021241188049316 + }, + { + "layer": "layer.69.mlp.up_proj", + "ppl": 4.0214762687683105 + }, + { + "layer": "layer.69.mlp.down_proj", + "ppl": 4.021498203277588 + }, + { + "layer": "layer.70.self_attn.q_proj", + "ppl": 4.02150297164917 + }, + { + "layer": "layer.70.self_attn.k_proj", + "ppl": 4.021395206451416 + }, + { + "layer": "layer.70.self_attn.v_proj", + "ppl": 4.0213141441345215 + }, + { + "layer": "layer.70.self_attn.o_proj", + "ppl": 4.021422863006592 + }, + { + "layer": "layer.70.mlp.gate_proj", + "ppl": 4.021398544311523 + }, + { + "layer": "layer.70.mlp.up_proj", + "ppl": 4.0213141441345215 + }, + { + "layer": "layer.70.mlp.down_proj", + "ppl": 4.021325588226318 + }, + { + "layer": "layer.71.self_attn.q_proj", + "ppl": 4.0213823318481445 + }, + { + "layer": "layer.71.self_attn.k_proj", + "ppl": 4.021410942077637 + }, + { + "layer": "layer.71.self_attn.v_proj", + "ppl": 4.02139139175415 + }, + { + "layer": "layer.71.self_attn.o_proj", + "ppl": 4.021367073059082 + }, + { + "layer": "layer.71.mlp.gate_proj", + "ppl": 4.021279811859131 + }, + { + "layer": "layer.71.mlp.up_proj", + "ppl": 4.02128267288208 + }, + { + "layer": "layer.71.mlp.down_proj", + "ppl": 4.0212554931640625 + }, + { + "layer": "layer.72.self_attn.q_proj", + "ppl": 4.0214314460754395 + }, + { + "layer": "layer.72.self_attn.k_proj", + "ppl": 4.021392345428467 + }, + { + "layer": "layer.72.self_attn.v_proj", + "ppl": 4.021386623382568 + }, + { + "layer": "layer.72.self_attn.o_proj", + "ppl": 4.021405220031738 + }, + { + "layer": "layer.72.mlp.gate_proj", + "ppl": 4.021261215209961 + }, + { + "layer": "layer.72.mlp.up_proj", + "ppl": 4.021181106567383 + }, + { + "layer": "layer.72.mlp.down_proj", + "ppl": 4.02131462097168 + }, + { + "layer": "layer.73.self_attn.q_proj", + "ppl": 4.021381378173828 + }, + { + "layer": "layer.73.self_attn.k_proj", + "ppl": 4.021348476409912 + }, + { + "layer": "layer.73.self_attn.v_proj", + "ppl": 4.021430969238281 + }, + { + "layer": "layer.73.self_attn.o_proj", + "ppl": 4.021471977233887 + }, + { + "layer": "layer.73.mlp.gate_proj", + "ppl": 4.021457195281982 + }, + { + "layer": "layer.73.mlp.up_proj", + "ppl": 4.021365165710449 + }, + { + "layer": "layer.73.mlp.down_proj", + "ppl": 4.02133321762085 + }, + { + "layer": "layer.74.self_attn.q_proj", + "ppl": 4.021433353424072 + }, + { + "layer": "layer.74.self_attn.k_proj", + "ppl": 4.021402359008789 + }, + { + "layer": "layer.74.self_attn.v_proj", + "ppl": 4.021481990814209 + }, + { + "layer": "layer.74.self_attn.o_proj", + "ppl": 4.021381378173828 + }, + { + "layer": "layer.74.mlp.gate_proj", + "ppl": 4.021304607391357 + }, + { + "layer": "layer.74.mlp.up_proj", + "ppl": 4.021320819854736 + }, + { + "layer": "layer.74.mlp.down_proj", + "ppl": 4.021266460418701 + }, + { + "layer": "layer.75.self_attn.q_proj", + "ppl": 4.021345615386963 + }, + { + "layer": "layer.75.self_attn.k_proj", + "ppl": 4.021371841430664 + }, + { + "layer": "layer.75.self_attn.v_proj", + "ppl": 4.021359443664551 + }, + { + "layer": "layer.75.self_attn.o_proj", + "ppl": 4.0213823318481445 + }, + { + "layer": "layer.75.mlp.gate_proj", + "ppl": 4.021145343780518 + }, + { + "layer": "layer.75.mlp.up_proj", + "ppl": 4.021364212036133 + }, + { + "layer": "layer.75.mlp.down_proj", + "ppl": 4.021227836608887 + }, + { + "layer": "layer.76.self_attn.q_proj", + "ppl": 4.021424293518066 + }, + { + "layer": "layer.76.self_attn.k_proj", + "ppl": 4.021413803100586 + }, + { + "layer": "layer.76.self_attn.v_proj", + "ppl": 4.021383762359619 + }, + { + "layer": "layer.76.self_attn.o_proj", + "ppl": 4.021362781524658 + }, + { + "layer": "layer.76.mlp.gate_proj", + "ppl": 4.021285057067871 + }, + { + "layer": "layer.76.mlp.up_proj", + "ppl": 4.021316051483154 + }, + { + "layer": "layer.76.mlp.down_proj", + "ppl": 4.021153450012207 + }, + { + "layer": "layer.77.self_attn.q_proj", + "ppl": 4.021400451660156 + }, + { + "layer": "layer.77.self_attn.k_proj", + "ppl": 4.021481990814209 + }, + { + "layer": "layer.77.self_attn.v_proj", + "ppl": 4.02144193649292 + }, + { + "layer": "layer.77.self_attn.o_proj", + "ppl": 4.0214009284973145 + }, + { + "layer": "layer.77.mlp.gate_proj", + "ppl": 4.021335124969482 + }, + { + "layer": "layer.77.mlp.up_proj", + "ppl": 4.021164417266846 + }, + { + "layer": "layer.77.mlp.down_proj", + "ppl": 4.021326541900635 + }, + { + "layer": "layer.78.self_attn.q_proj", + "ppl": 4.021358489990234 + }, + { + "layer": "layer.78.self_attn.k_proj", + "ppl": 4.021371841430664 + }, + { + "layer": "layer.78.self_attn.v_proj", + "ppl": 4.02136754989624 + }, + { + "layer": "layer.78.self_attn.o_proj", + "ppl": 4.021419525146484 + }, + { + "layer": "layer.78.mlp.gate_proj", + "ppl": 4.021106243133545 + }, + { + "layer": "layer.78.mlp.up_proj", + "ppl": 4.0212483406066895 + }, + { + "layer": "layer.78.mlp.down_proj", + "ppl": 4.021353244781494 + }, + { + "layer": "layer.79.self_attn.q_proj", + "ppl": 4.021328449249268 + }, + { + "layer": "layer.79.self_attn.k_proj", + "ppl": 4.021468162536621 + }, + { + "layer": "layer.79.self_attn.v_proj", + "ppl": 4.02140998840332 + }, + { + "layer": "layer.79.self_attn.o_proj", + "ppl": 4.021411895751953 + }, + { + "layer": "layer.79.mlp.gate_proj", + "ppl": 4.020795822143555 + }, + { + "layer": "layer.79.mlp.up_proj", + "ppl": 4.0207743644714355 + }, + { + "layer": "layer.79.mlp.down_proj", + "ppl": 4.021050453186035 + } + ] + }, + { + "bit": 15, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 4.009766101837158 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 4.0098490715026855 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 4.009957790374756 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 4.009809970855713 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 4.009856224060059 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 4.00961971282959 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 4.0098185539245605 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 4.009769439697266 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 4.009881019592285 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 4.009838104248047 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 4.009895324707031 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 4.0099334716796875 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 4.009800434112549 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 4.00990629196167 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 4.009844779968262 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 4.009950637817383 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 4.009729862213135 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 4.009812355041504 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 4.009776592254639 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 4.00981330871582 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 4.009867191314697 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 4.009773254394531 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 4.009751796722412 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 4.0098185539245605 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 4.009791851043701 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 4.009925365447998 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 4.009720325469971 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 4.009802341461182 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 4.009902000427246 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 4.009850025177002 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 4.009786128997803 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 4.009839057922363 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 4.0098042488098145 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 4.009831428527832 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 4.009845733642578 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 4.009831428527832 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 4.009769916534424 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 4.009800434112549 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 4.009871482849121 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 4.009830474853516 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 4.00992488861084 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 4.009854316711426 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 4.0098371505737305 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 4.009762287139893 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 4.009904861450195 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 4.009881496429443 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 4.009764194488525 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 4.009823322296143 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 4.009821891784668 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 4.0098981857299805 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 4.009914875030518 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 4.009839057922363 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 4.009829521179199 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 4.009930610656738 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 4.009825229644775 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 4.009881496429443 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 4.0098347663879395 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 4.009908199310303 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 4.009777545928955 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 4.009749889373779 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 4.009913444519043 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 4.009816646575928 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 4.009739398956299 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 4.0099029541015625 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 4.009815692901611 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 4.009850978851318 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 4.00985860824585 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 4.0098466873168945 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 4.009822845458984 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 4.009854316711426 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 4.009880542755127 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 4.009893417358398 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 4.0098466873168945 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 4.009754657745361 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 4.009821891784668 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 4.009826183319092 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 4.009824275970459 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 4.009824275970459 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 4.0098748207092285 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 4.009819507598877 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 4.009868621826172 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 4.009844779968262 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 4.009868621826172 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 4.010006427764893 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 4.009781360626221 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 4.009828567504883 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 4.009869575500488 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 4.00990104675293 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 4.009761810302734 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 4.009857654571533 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 4.009855270385742 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 4.0098748207092285 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 4.009737014770508 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 4.009816646575928 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 4.009860992431641 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 4.009971618652344 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 4.009795188903809 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 4.009941101074219 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 4.009795665740967 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 4.0098371505737305 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 4.009819507598877 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 4.009868621826172 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 4.009735107421875 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 4.009739875793457 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 4.009850978851318 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 4.0098347663879395 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 4.009855270385742 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 4.009898662567139 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 4.00979471206665 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 4.009758472442627 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 4.009758472442627 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 4.009931564331055 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 4.009820461273193 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 4.00974702835083 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 4.009941101074219 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 4.00994348526001 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 4.009953498840332 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 4.009848117828369 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 4.009754180908203 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 4.009856700897217 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 4.00977087020874 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 4.009916305541992 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 4.009825229644775 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 4.00981330871582 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 4.009816646575928 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 4.00965690612793 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 4.009827136993408 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 4.009891510009766 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 4.0098748207092285 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 4.009838104248047 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 4.00991153717041 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 4.009825229644775 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 4.009876251220703 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 4.00988245010376 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 4.009939193725586 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 4.009903907775879 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 4.009864807128906 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 4.009853363037109 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 4.009838104248047 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 4.009798526763916 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 4.009788990020752 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 4.009710311889648 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 4.009803295135498 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 4.009844779968262 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 4.0098795890808105 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 4.009847640991211 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 4.009944915771484 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 4.009908676147461 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 4.009892463684082 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 4.00990104675293 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 4.009795665740967 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 4.009848117828369 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 4.009866237640381 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 4.009840488433838 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 4.009871959686279 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 4.009823322296143 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 4.009852409362793 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 4.009852409362793 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 4.009923934936523 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 4.009805679321289 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 4.009845733642578 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 4.009847640991211 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 4.00985860824585 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 4.0099263191223145 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 4.009788513183594 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 4.009852409362793 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 4.009896755218506 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 4.009871482849121 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 4.009860038757324 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 4.009905815124512 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 4.009888648986816 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 4.009927749633789 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 4.009706020355225 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 4.0098090171813965 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 4.009734630584717 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 4.0097503662109375 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 4.009966850280762 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 4.009891510009766 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 4.009835720062256 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 4.00985860824585 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 4.009860038757324 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 4.00978946685791 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 4.0098748207092285 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 4.0098652839660645 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 4.009823799133301 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 4.009889602661133 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 4.009815692901611 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 4.00983190536499 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 4.0098090171813965 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 4.009883403778076 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 4.009917259216309 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 4.009821891784668 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 4.009772777557373 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 4.009983062744141 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 4.009917736053467 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 4.009892463684082 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 4.009900093078613 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 4.009885787963867 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 4.009860992431641 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 4.009718418121338 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 4.009868621826172 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 4.0097551345825195 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 4.009842395782471 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 4.009783744812012 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 4.009800434112549 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 4.009908676147461 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 4.0099592208862305 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 4.009609222412109 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 4.009822845458984 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 4.009873867034912 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 4.009856700897217 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 4.009850025177002 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 4.009845733642578 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 4.009799003601074 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 4.009964466094971 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 4.00995397567749 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 4.009760856628418 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 4.009861946105957 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 4.009859561920166 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 4.009810924530029 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 4.009872913360596 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 4.0098371505737305 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 4.009890079498291 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 4.010056495666504 + }, + { + "layer": "layer.32.self_attn.q_proj", + "ppl": 4.009819984436035 + }, + { + "layer": "layer.32.self_attn.k_proj", + "ppl": 4.009963035583496 + }, + { + "layer": "layer.32.self_attn.v_proj", + "ppl": 4.009845733642578 + }, + { + "layer": "layer.32.self_attn.o_proj", + "ppl": 4.009894371032715 + }, + { + "layer": "layer.32.mlp.gate_proj", + "ppl": 4.009751796722412 + }, + { + "layer": "layer.32.mlp.up_proj", + "ppl": 4.009944438934326 + }, + { + "layer": "layer.32.mlp.down_proj", + "ppl": 4.009971618652344 + }, + { + "layer": "layer.33.self_attn.q_proj", + "ppl": 4.009905815124512 + }, + { + "layer": "layer.33.self_attn.k_proj", + "ppl": 4.009856700897217 + }, + { + "layer": "layer.33.self_attn.v_proj", + "ppl": 4.009810924530029 + }, + { + "layer": "layer.33.self_attn.o_proj", + "ppl": 4.0098347663879395 + }, + { + "layer": "layer.33.mlp.gate_proj", + "ppl": 4.00985860824585 + }, + { + "layer": "layer.33.mlp.up_proj", + "ppl": 4.009823322296143 + }, + { + "layer": "layer.33.mlp.down_proj", + "ppl": 4.00990629196167 + }, + { + "layer": "layer.34.self_attn.q_proj", + "ppl": 4.009861946105957 + }, + { + "layer": "layer.34.self_attn.k_proj", + "ppl": 4.009851932525635 + }, + { + "layer": "layer.34.self_attn.v_proj", + "ppl": 4.00983190536499 + }, + { + "layer": "layer.34.self_attn.o_proj", + "ppl": 4.01002311706543 + }, + { + "layer": "layer.34.mlp.gate_proj", + "ppl": 4.009951591491699 + }, + { + "layer": "layer.34.mlp.up_proj", + "ppl": 4.009912967681885 + }, + { + "layer": "layer.34.mlp.down_proj", + "ppl": 4.0098090171813965 + }, + { + "layer": "layer.35.self_attn.q_proj", + "ppl": 4.009952545166016 + }, + { + "layer": "layer.35.self_attn.k_proj", + "ppl": 4.009912490844727 + }, + { + "layer": "layer.35.self_attn.v_proj", + "ppl": 4.009764194488525 + }, + { + "layer": "layer.35.self_attn.o_proj", + "ppl": 4.009800434112549 + }, + { + "layer": "layer.35.mlp.gate_proj", + "ppl": 4.009735107421875 + }, + { + "layer": "layer.35.mlp.up_proj", + "ppl": 4.009762763977051 + }, + { + "layer": "layer.35.mlp.down_proj", + "ppl": 4.010009765625 + }, + { + "layer": "layer.36.self_attn.q_proj", + "ppl": 4.009786128997803 + }, + { + "layer": "layer.36.self_attn.k_proj", + "ppl": 4.009876251220703 + }, + { + "layer": "layer.36.self_attn.v_proj", + "ppl": 4.0098347663879395 + }, + { + "layer": "layer.36.self_attn.o_proj", + "ppl": 4.009807586669922 + }, + { + "layer": "layer.36.mlp.gate_proj", + "ppl": 4.009757995605469 + }, + { + "layer": "layer.36.mlp.up_proj", + "ppl": 4.0098185539245605 + }, + { + "layer": "layer.36.mlp.down_proj", + "ppl": 4.009795665740967 + }, + { + "layer": "layer.37.self_attn.q_proj", + "ppl": 4.009786128997803 + }, + { + "layer": "layer.37.self_attn.k_proj", + "ppl": 4.009838104248047 + }, + { + "layer": "layer.37.self_attn.v_proj", + "ppl": 4.0098772048950195 + }, + { + "layer": "layer.37.self_attn.o_proj", + "ppl": 4.009931564331055 + }, + { + "layer": "layer.37.mlp.gate_proj", + "ppl": 4.009704113006592 + }, + { + "layer": "layer.37.mlp.up_proj", + "ppl": 4.009873867034912 + }, + { + "layer": "layer.37.mlp.down_proj", + "ppl": 4.009859561920166 + }, + { + "layer": "layer.38.self_attn.q_proj", + "ppl": 4.009809970855713 + }, + { + "layer": "layer.38.self_attn.k_proj", + "ppl": 4.010002136230469 + }, + { + "layer": "layer.38.self_attn.v_proj", + "ppl": 4.009814262390137 + }, + { + "layer": "layer.38.self_attn.o_proj", + "ppl": 4.009675025939941 + }, + { + "layer": "layer.38.mlp.gate_proj", + "ppl": 4.009969711303711 + }, + { + "layer": "layer.38.mlp.up_proj", + "ppl": 4.009780406951904 + }, + { + "layer": "layer.38.mlp.down_proj", + "ppl": 4.009669780731201 + }, + { + "layer": "layer.39.self_attn.q_proj", + "ppl": 4.009888172149658 + }, + { + "layer": "layer.39.self_attn.k_proj", + "ppl": 4.009970188140869 + }, + { + "layer": "layer.39.self_attn.v_proj", + "ppl": 4.0098724365234375 + }, + { + "layer": "layer.39.self_attn.o_proj", + "ppl": 4.009822845458984 + }, + { + "layer": "layer.39.mlp.gate_proj", + "ppl": 4.009844779968262 + }, + { + "layer": "layer.39.mlp.up_proj", + "ppl": 4.009937286376953 + }, + { + "layer": "layer.39.mlp.down_proj", + "ppl": 4.010169982910156 + }, + { + "layer": "layer.40.self_attn.q_proj", + "ppl": 4.009902000427246 + }, + { + "layer": "layer.40.self_attn.k_proj", + "ppl": 4.009894371032715 + }, + { + "layer": "layer.40.self_attn.v_proj", + "ppl": 4.009805679321289 + }, + { + "layer": "layer.40.self_attn.o_proj", + "ppl": 4.009880065917969 + }, + { + "layer": "layer.40.mlp.gate_proj", + "ppl": 4.009790897369385 + }, + { + "layer": "layer.40.mlp.up_proj", + "ppl": 4.009892463684082 + }, + { + "layer": "layer.40.mlp.down_proj", + "ppl": 4.009622097015381 + }, + { + "layer": "layer.41.self_attn.q_proj", + "ppl": 4.009843826293945 + }, + { + "layer": "layer.41.self_attn.k_proj", + "ppl": 4.009814739227295 + }, + { + "layer": "layer.41.self_attn.v_proj", + "ppl": 4.009951114654541 + }, + { + "layer": "layer.41.self_attn.o_proj", + "ppl": 4.009814739227295 + }, + { + "layer": "layer.41.mlp.gate_proj", + "ppl": 4.009854316711426 + }, + { + "layer": "layer.41.mlp.up_proj", + "ppl": 4.0099568367004395 + }, + { + "layer": "layer.41.mlp.down_proj", + "ppl": 4.0096588134765625 + }, + { + "layer": "layer.42.self_attn.q_proj", + "ppl": 4.009857654571533 + }, + { + "layer": "layer.42.self_attn.k_proj", + "ppl": 4.009819030761719 + }, + { + "layer": "layer.42.self_attn.v_proj", + "ppl": 4.009732246398926 + }, + { + "layer": "layer.42.self_attn.o_proj", + "ppl": 4.009892463684082 + }, + { + "layer": "layer.42.mlp.gate_proj", + "ppl": 4.009862899780273 + }, + { + "layer": "layer.42.mlp.up_proj", + "ppl": 4.009962558746338 + }, + { + "layer": "layer.42.mlp.down_proj", + "ppl": 4.010027885437012 + }, + { + "layer": "layer.43.self_attn.q_proj", + "ppl": 4.009959697723389 + }, + { + "layer": "layer.43.self_attn.k_proj", + "ppl": 4.00988245010376 + }, + { + "layer": "layer.43.self_attn.v_proj", + "ppl": 4.0097432136535645 + }, + { + "layer": "layer.43.self_attn.o_proj", + "ppl": 4.009861946105957 + }, + { + "layer": "layer.43.mlp.gate_proj", + "ppl": 4.009973049163818 + }, + { + "layer": "layer.43.mlp.up_proj", + "ppl": 4.00990629196167 + }, + { + "layer": "layer.43.mlp.down_proj", + "ppl": 4.0098090171813965 + }, + { + "layer": "layer.44.self_attn.q_proj", + "ppl": 4.009930610656738 + }, + { + "layer": "layer.44.self_attn.k_proj", + "ppl": 4.009835720062256 + }, + { + "layer": "layer.44.self_attn.v_proj", + "ppl": 4.009722709655762 + }, + { + "layer": "layer.44.self_attn.o_proj", + "ppl": 4.009802341461182 + }, + { + "layer": "layer.44.mlp.gate_proj", + "ppl": 4.009963035583496 + }, + { + "layer": "layer.44.mlp.up_proj", + "ppl": 4.009860992431641 + }, + { + "layer": "layer.44.mlp.down_proj", + "ppl": 4.0100507736206055 + }, + { + "layer": "layer.45.self_attn.q_proj", + "ppl": 4.009878158569336 + }, + { + "layer": "layer.45.self_attn.k_proj", + "ppl": 4.0099263191223145 + }, + { + "layer": "layer.45.self_attn.v_proj", + "ppl": 4.009820938110352 + }, + { + "layer": "layer.45.self_attn.o_proj", + "ppl": 4.009798526763916 + }, + { + "layer": "layer.45.mlp.gate_proj", + "ppl": 4.009850978851318 + }, + { + "layer": "layer.45.mlp.up_proj", + "ppl": 4.009932041168213 + }, + { + "layer": "layer.45.mlp.down_proj", + "ppl": 4.009932994842529 + }, + { + "layer": "layer.46.self_attn.q_proj", + "ppl": 4.009832859039307 + }, + { + "layer": "layer.46.self_attn.k_proj", + "ppl": 4.009921073913574 + }, + { + "layer": "layer.46.self_attn.v_proj", + "ppl": 4.009857654571533 + }, + { + "layer": "layer.46.self_attn.o_proj", + "ppl": 4.009777069091797 + }, + { + "layer": "layer.46.mlp.gate_proj", + "ppl": 4.009928226470947 + }, + { + "layer": "layer.46.mlp.up_proj", + "ppl": 4.009629726409912 + }, + { + "layer": "layer.46.mlp.down_proj", + "ppl": 4.009957313537598 + }, + { + "layer": "layer.47.self_attn.q_proj", + "ppl": 4.009866237640381 + }, + { + "layer": "layer.47.self_attn.k_proj", + "ppl": 4.009847640991211 + }, + { + "layer": "layer.47.self_attn.v_proj", + "ppl": 4.0098958015441895 + }, + { + "layer": "layer.47.self_attn.o_proj", + "ppl": 4.0098652839660645 + }, + { + "layer": "layer.47.mlp.gate_proj", + "ppl": 4.009896278381348 + }, + { + "layer": "layer.47.mlp.up_proj", + "ppl": 4.009842395782471 + }, + { + "layer": "layer.47.mlp.down_proj", + "ppl": 4.009886741638184 + }, + { + "layer": "layer.48.self_attn.q_proj", + "ppl": 4.009816646575928 + }, + { + "layer": "layer.48.self_attn.k_proj", + "ppl": 4.009822845458984 + }, + { + "layer": "layer.48.self_attn.v_proj", + "ppl": 4.009869575500488 + }, + { + "layer": "layer.48.self_attn.o_proj", + "ppl": 4.009871959686279 + }, + { + "layer": "layer.48.mlp.gate_proj", + "ppl": 4.009832859039307 + }, + { + "layer": "layer.48.mlp.up_proj", + "ppl": 4.009571075439453 + }, + { + "layer": "layer.48.mlp.down_proj", + "ppl": 4.009746074676514 + }, + { + "layer": "layer.49.self_attn.q_proj", + "ppl": 4.009910583496094 + }, + { + "layer": "layer.49.self_attn.k_proj", + "ppl": 4.009860992431641 + }, + { + "layer": "layer.49.self_attn.v_proj", + "ppl": 4.009745121002197 + }, + { + "layer": "layer.49.self_attn.o_proj", + "ppl": 4.009923934936523 + }, + { + "layer": "layer.49.mlp.gate_proj", + "ppl": 4.009902000427246 + }, + { + "layer": "layer.49.mlp.up_proj", + "ppl": 4.009745121002197 + }, + { + "layer": "layer.49.mlp.down_proj", + "ppl": 4.009804725646973 + }, + { + "layer": "layer.50.self_attn.q_proj", + "ppl": 4.009871482849121 + }, + { + "layer": "layer.50.self_attn.k_proj", + "ppl": 4.009884834289551 + }, + { + "layer": "layer.50.self_attn.v_proj", + "ppl": 4.009862899780273 + }, + { + "layer": "layer.50.self_attn.o_proj", + "ppl": 4.0099029541015625 + }, + { + "layer": "layer.50.mlp.gate_proj", + "ppl": 4.009735584259033 + }, + { + "layer": "layer.50.mlp.up_proj", + "ppl": 4.009909629821777 + }, + { + "layer": "layer.50.mlp.down_proj", + "ppl": 4.009852409362793 + }, + { + "layer": "layer.51.self_attn.q_proj", + "ppl": 4.009832859039307 + }, + { + "layer": "layer.51.self_attn.k_proj", + "ppl": 4.009952068328857 + }, + { + "layer": "layer.51.self_attn.v_proj", + "ppl": 4.009924411773682 + }, + { + "layer": "layer.51.self_attn.o_proj", + "ppl": 4.009937763214111 + }, + { + "layer": "layer.51.mlp.gate_proj", + "ppl": 4.0098090171813965 + }, + { + "layer": "layer.51.mlp.up_proj", + "ppl": 4.009842395782471 + }, + { + "layer": "layer.51.mlp.down_proj", + "ppl": 4.009767055511475 + }, + { + "layer": "layer.52.self_attn.q_proj", + "ppl": 4.009766101837158 + }, + { + "layer": "layer.52.self_attn.k_proj", + "ppl": 4.009776592254639 + }, + { + "layer": "layer.52.self_attn.v_proj", + "ppl": 4.010007858276367 + }, + { + "layer": "layer.52.self_attn.o_proj", + "ppl": 4.009723663330078 + }, + { + "layer": "layer.52.mlp.gate_proj", + "ppl": 4.009850978851318 + }, + { + "layer": "layer.52.mlp.up_proj", + "ppl": 4.00988245010376 + }, + { + "layer": "layer.52.mlp.down_proj", + "ppl": 4.0097784996032715 + }, + { + "layer": "layer.53.self_attn.q_proj", + "ppl": 4.009992599487305 + }, + { + "layer": "layer.53.self_attn.k_proj", + "ppl": 4.009786605834961 + }, + { + "layer": "layer.53.self_attn.v_proj", + "ppl": 4.009883880615234 + }, + { + "layer": "layer.53.self_attn.o_proj", + "ppl": 4.010024070739746 + }, + { + "layer": "layer.53.mlp.gate_proj", + "ppl": 4.009883880615234 + }, + { + "layer": "layer.53.mlp.up_proj", + "ppl": 4.009771823883057 + }, + { + "layer": "layer.53.mlp.down_proj", + "ppl": 4.009746551513672 + }, + { + "layer": "layer.54.self_attn.q_proj", + "ppl": 4.009888648986816 + }, + { + "layer": "layer.54.self_attn.k_proj", + "ppl": 4.009888648986816 + }, + { + "layer": "layer.54.self_attn.v_proj", + "ppl": 4.010047435760498 + }, + { + "layer": "layer.54.self_attn.o_proj", + "ppl": 4.0098772048950195 + }, + { + "layer": "layer.54.mlp.gate_proj", + "ppl": 4.010021686553955 + }, + { + "layer": "layer.54.mlp.up_proj", + "ppl": 4.009864807128906 + }, + { + "layer": "layer.54.mlp.down_proj", + "ppl": 4.0098042488098145 + }, + { + "layer": "layer.55.self_attn.q_proj", + "ppl": 4.009891510009766 + }, + { + "layer": "layer.55.self_attn.k_proj", + "ppl": 4.00981330871582 + }, + { + "layer": "layer.55.self_attn.v_proj", + "ppl": 4.009819507598877 + }, + { + "layer": "layer.55.self_attn.o_proj", + "ppl": 4.0096611976623535 + }, + { + "layer": "layer.55.mlp.gate_proj", + "ppl": 4.009798049926758 + }, + { + "layer": "layer.55.mlp.up_proj", + "ppl": 4.009893417358398 + }, + { + "layer": "layer.55.mlp.down_proj", + "ppl": 4.009825229644775 + }, + { + "layer": "layer.56.self_attn.q_proj", + "ppl": 4.009934902191162 + }, + { + "layer": "layer.56.self_attn.k_proj", + "ppl": 4.009782314300537 + }, + { + "layer": "layer.56.self_attn.v_proj", + "ppl": 4.009912014007568 + }, + { + "layer": "layer.56.self_attn.o_proj", + "ppl": 4.009825229644775 + }, + { + "layer": "layer.56.mlp.gate_proj", + "ppl": 4.009889602661133 + }, + { + "layer": "layer.56.mlp.up_proj", + "ppl": 4.009864330291748 + }, + { + "layer": "layer.56.mlp.down_proj", + "ppl": 4.009762763977051 + }, + { + "layer": "layer.57.self_attn.q_proj", + "ppl": 4.009747505187988 + }, + { + "layer": "layer.57.self_attn.k_proj", + "ppl": 4.009964942932129 + }, + { + "layer": "layer.57.self_attn.v_proj", + "ppl": 4.009861946105957 + }, + { + "layer": "layer.57.self_attn.o_proj", + "ppl": 4.009899139404297 + }, + { + "layer": "layer.57.mlp.gate_proj", + "ppl": 4.0097880363464355 + }, + { + "layer": "layer.57.mlp.up_proj", + "ppl": 4.009916305541992 + }, + { + "layer": "layer.57.mlp.down_proj", + "ppl": 4.009861946105957 + }, + { + "layer": "layer.58.self_attn.q_proj", + "ppl": 4.0099077224731445 + }, + { + "layer": "layer.58.self_attn.k_proj", + "ppl": 4.009847640991211 + }, + { + "layer": "layer.58.self_attn.v_proj", + "ppl": 4.00980806350708 + }, + { + "layer": "layer.58.self_attn.o_proj", + "ppl": 4.0098371505737305 + }, + { + "layer": "layer.58.mlp.gate_proj", + "ppl": 4.0097784996032715 + }, + { + "layer": "layer.58.mlp.up_proj", + "ppl": 4.009926795959473 + }, + { + "layer": "layer.58.mlp.down_proj", + "ppl": 4.009744644165039 + }, + { + "layer": "layer.59.self_attn.q_proj", + "ppl": 4.009918212890625 + }, + { + "layer": "layer.59.self_attn.k_proj", + "ppl": 4.009878158569336 + }, + { + "layer": "layer.59.self_attn.v_proj", + "ppl": 4.009848117828369 + }, + { + "layer": "layer.59.self_attn.o_proj", + "ppl": 4.009939193725586 + }, + { + "layer": "layer.59.mlp.gate_proj", + "ppl": 4.009777069091797 + }, + { + "layer": "layer.59.mlp.up_proj", + "ppl": 4.009852409362793 + }, + { + "layer": "layer.59.mlp.down_proj", + "ppl": 4.009888648986816 + }, + { + "layer": "layer.60.self_attn.q_proj", + "ppl": 4.0098652839660645 + }, + { + "layer": "layer.60.self_attn.k_proj", + "ppl": 4.0097479820251465 + }, + { + "layer": "layer.60.self_attn.v_proj", + "ppl": 4.009908676147461 + }, + { + "layer": "layer.60.self_attn.o_proj", + "ppl": 4.009765625 + }, + { + "layer": "layer.60.mlp.gate_proj", + "ppl": 4.009880542755127 + }, + { + "layer": "layer.60.mlp.up_proj", + "ppl": 4.009798049926758 + }, + { + "layer": "layer.60.mlp.down_proj", + "ppl": 4.009822845458984 + }, + { + "layer": "layer.61.self_attn.q_proj", + "ppl": 4.009943008422852 + }, + { + "layer": "layer.61.self_attn.k_proj", + "ppl": 4.0098185539245605 + }, + { + "layer": "layer.61.self_attn.v_proj", + "ppl": 4.009810924530029 + }, + { + "layer": "layer.61.self_attn.o_proj", + "ppl": 4.0097174644470215 + }, + { + "layer": "layer.61.mlp.gate_proj", + "ppl": 4.009803295135498 + }, + { + "layer": "layer.61.mlp.up_proj", + "ppl": 4.009728908538818 + }, + { + "layer": "layer.61.mlp.down_proj", + "ppl": 4.0096917152404785 + }, + { + "layer": "layer.62.self_attn.q_proj", + "ppl": 4.009824275970459 + }, + { + "layer": "layer.62.self_attn.k_proj", + "ppl": 4.009941101074219 + }, + { + "layer": "layer.62.self_attn.v_proj", + "ppl": 4.009766578674316 + }, + { + "layer": "layer.62.self_attn.o_proj", + "ppl": 4.009891986846924 + }, + { + "layer": "layer.62.mlp.gate_proj", + "ppl": 4.009825229644775 + }, + { + "layer": "layer.62.mlp.up_proj", + "ppl": 4.0099287033081055 + }, + { + "layer": "layer.62.mlp.down_proj", + "ppl": 4.009836196899414 + }, + { + "layer": "layer.63.self_attn.q_proj", + "ppl": 4.009929180145264 + }, + { + "layer": "layer.63.self_attn.k_proj", + "ppl": 4.009741306304932 + }, + { + "layer": "layer.63.self_attn.v_proj", + "ppl": 4.009856700897217 + }, + { + "layer": "layer.63.self_attn.o_proj", + "ppl": 4.009814262390137 + }, + { + "layer": "layer.63.mlp.gate_proj", + "ppl": 4.009718418121338 + }, + { + "layer": "layer.63.mlp.up_proj", + "ppl": 4.00978422164917 + }, + { + "layer": "layer.63.mlp.down_proj", + "ppl": 4.009828567504883 + }, + { + "layer": "layer.64.self_attn.q_proj", + "ppl": 4.009796142578125 + }, + { + "layer": "layer.64.self_attn.k_proj", + "ppl": 4.009885787963867 + }, + { + "layer": "layer.64.self_attn.v_proj", + "ppl": 4.0098114013671875 + }, + { + "layer": "layer.64.self_attn.o_proj", + "ppl": 4.0097761154174805 + }, + { + "layer": "layer.64.mlp.gate_proj", + "ppl": 4.009749889373779 + }, + { + "layer": "layer.64.mlp.up_proj", + "ppl": 4.010014057159424 + }, + { + "layer": "layer.64.mlp.down_proj", + "ppl": 4.009871482849121 + }, + { + "layer": "layer.65.self_attn.q_proj", + "ppl": 4.009779453277588 + }, + { + "layer": "layer.65.self_attn.k_proj", + "ppl": 4.009832859039307 + }, + { + "layer": "layer.65.self_attn.v_proj", + "ppl": 4.009852409362793 + }, + { + "layer": "layer.65.self_attn.o_proj", + "ppl": 4.009756565093994 + }, + { + "layer": "layer.65.mlp.gate_proj", + "ppl": 4.009766578674316 + }, + { + "layer": "layer.65.mlp.up_proj", + "ppl": 4.009756088256836 + }, + { + "layer": "layer.65.mlp.down_proj", + "ppl": 4.00966215133667 + }, + { + "layer": "layer.66.self_attn.q_proj", + "ppl": 4.009824275970459 + }, + { + "layer": "layer.66.self_attn.k_proj", + "ppl": 4.009827136993408 + }, + { + "layer": "layer.66.self_attn.v_proj", + "ppl": 4.009879112243652 + }, + { + "layer": "layer.66.self_attn.o_proj", + "ppl": 4.009830474853516 + }, + { + "layer": "layer.66.mlp.gate_proj", + "ppl": 4.009697914123535 + }, + { + "layer": "layer.66.mlp.up_proj", + "ppl": 4.009746074676514 + }, + { + "layer": "layer.66.mlp.down_proj", + "ppl": 4.00966739654541 + }, + { + "layer": "layer.67.self_attn.q_proj", + "ppl": 4.009798526763916 + }, + { + "layer": "layer.67.self_attn.k_proj", + "ppl": 4.0097551345825195 + }, + { + "layer": "layer.67.self_attn.v_proj", + "ppl": 4.009768962860107 + }, + { + "layer": "layer.67.self_attn.o_proj", + "ppl": 4.009847640991211 + }, + { + "layer": "layer.67.mlp.gate_proj", + "ppl": 4.009787082672119 + }, + { + "layer": "layer.67.mlp.up_proj", + "ppl": 4.00990629196167 + }, + { + "layer": "layer.67.mlp.down_proj", + "ppl": 4.009634017944336 + }, + { + "layer": "layer.68.self_attn.q_proj", + "ppl": 4.0098042488098145 + }, + { + "layer": "layer.68.self_attn.k_proj", + "ppl": 4.009831428527832 + }, + { + "layer": "layer.68.self_attn.v_proj", + "ppl": 4.009704113006592 + }, + { + "layer": "layer.68.self_attn.o_proj", + "ppl": 4.009873867034912 + }, + { + "layer": "layer.68.mlp.gate_proj", + "ppl": 4.009741306304932 + }, + { + "layer": "layer.68.mlp.up_proj", + "ppl": 4.009779453277588 + }, + { + "layer": "layer.68.mlp.down_proj", + "ppl": 4.009751796722412 + }, + { + "layer": "layer.69.self_attn.q_proj", + "ppl": 4.009799480438232 + }, + { + "layer": "layer.69.self_attn.k_proj", + "ppl": 4.009754657745361 + }, + { + "layer": "layer.69.self_attn.v_proj", + "ppl": 4.009740829467773 + }, + { + "layer": "layer.69.self_attn.o_proj", + "ppl": 4.009845733642578 + }, + { + "layer": "layer.69.mlp.gate_proj", + "ppl": 4.009814739227295 + }, + { + "layer": "layer.69.mlp.up_proj", + "ppl": 4.009644031524658 + }, + { + "layer": "layer.69.mlp.down_proj", + "ppl": 4.009729862213135 + }, + { + "layer": "layer.70.self_attn.q_proj", + "ppl": 4.009726524353027 + }, + { + "layer": "layer.70.self_attn.k_proj", + "ppl": 4.009738922119141 + }, + { + "layer": "layer.70.self_attn.v_proj", + "ppl": 4.009843349456787 + }, + { + "layer": "layer.70.self_attn.o_proj", + "ppl": 4.0097880363464355 + }, + { + "layer": "layer.70.mlp.gate_proj", + "ppl": 4.009657859802246 + }, + { + "layer": "layer.70.mlp.up_proj", + "ppl": 4.009576797485352 + }, + { + "layer": "layer.70.mlp.down_proj", + "ppl": 4.009800434112549 + }, + { + "layer": "layer.71.self_attn.q_proj", + "ppl": 4.009685039520264 + }, + { + "layer": "layer.71.self_attn.k_proj", + "ppl": 4.009747505187988 + }, + { + "layer": "layer.71.self_attn.v_proj", + "ppl": 4.009673118591309 + }, + { + "layer": "layer.71.self_attn.o_proj", + "ppl": 4.009791851043701 + }, + { + "layer": "layer.71.mlp.gate_proj", + "ppl": 4.009746074676514 + }, + { + "layer": "layer.71.mlp.up_proj", + "ppl": 4.009803295135498 + }, + { + "layer": "layer.71.mlp.down_proj", + "ppl": 4.009895324707031 + }, + { + "layer": "layer.72.self_attn.q_proj", + "ppl": 4.009775638580322 + }, + { + "layer": "layer.72.self_attn.k_proj", + "ppl": 4.009775161743164 + }, + { + "layer": "layer.72.self_attn.v_proj", + "ppl": 4.00977087020874 + }, + { + "layer": "layer.72.self_attn.o_proj", + "ppl": 4.009831428527832 + }, + { + "layer": "layer.72.mlp.gate_proj", + "ppl": 4.009768486022949 + }, + { + "layer": "layer.72.mlp.up_proj", + "ppl": 4.009892463684082 + }, + { + "layer": "layer.72.mlp.down_proj", + "ppl": 4.009814739227295 + }, + { + "layer": "layer.73.self_attn.q_proj", + "ppl": 4.009763717651367 + }, + { + "layer": "layer.73.self_attn.k_proj", + "ppl": 4.0098395347595215 + }, + { + "layer": "layer.73.self_attn.v_proj", + "ppl": 4.009737491607666 + }, + { + "layer": "layer.73.self_attn.o_proj", + "ppl": 4.0097832679748535 + }, + { + "layer": "layer.73.mlp.gate_proj", + "ppl": 4.009640693664551 + }, + { + "layer": "layer.73.mlp.up_proj", + "ppl": 4.009801387786865 + }, + { + "layer": "layer.73.mlp.down_proj", + "ppl": 4.009708881378174 + }, + { + "layer": "layer.74.self_attn.q_proj", + "ppl": 4.0098371505737305 + }, + { + "layer": "layer.74.self_attn.k_proj", + "ppl": 4.00972318649292 + }, + { + "layer": "layer.74.self_attn.v_proj", + "ppl": 4.009726524353027 + }, + { + "layer": "layer.74.self_attn.o_proj", + "ppl": 4.0098772048950195 + }, + { + "layer": "layer.74.mlp.gate_proj", + "ppl": 4.009708881378174 + }, + { + "layer": "layer.74.mlp.up_proj", + "ppl": 4.0098066329956055 + }, + { + "layer": "layer.74.mlp.down_proj", + "ppl": 4.009881019592285 + }, + { + "layer": "layer.75.self_attn.q_proj", + "ppl": 4.009773254394531 + }, + { + "layer": "layer.75.self_attn.k_proj", + "ppl": 4.009814262390137 + }, + { + "layer": "layer.75.self_attn.v_proj", + "ppl": 4.009766101837158 + }, + { + "layer": "layer.75.self_attn.o_proj", + "ppl": 4.009762287139893 + }, + { + "layer": "layer.75.mlp.gate_proj", + "ppl": 4.009759902954102 + }, + { + "layer": "layer.75.mlp.up_proj", + "ppl": 4.009762287139893 + }, + { + "layer": "layer.75.mlp.down_proj", + "ppl": 4.009850025177002 + }, + { + "layer": "layer.76.self_attn.q_proj", + "ppl": 4.009807109832764 + }, + { + "layer": "layer.76.self_attn.k_proj", + "ppl": 4.009758472442627 + }, + { + "layer": "layer.76.self_attn.v_proj", + "ppl": 4.009873867034912 + }, + { + "layer": "layer.76.self_attn.o_proj", + "ppl": 4.00973653793335 + }, + { + "layer": "layer.76.mlp.gate_proj", + "ppl": 4.009623050689697 + }, + { + "layer": "layer.76.mlp.up_proj", + "ppl": 4.009695529937744 + }, + { + "layer": "layer.76.mlp.down_proj", + "ppl": 4.00976037979126 + }, + { + "layer": "layer.77.self_attn.q_proj", + "ppl": 4.009714126586914 + }, + { + "layer": "layer.77.self_attn.k_proj", + "ppl": 4.009718418121338 + }, + { + "layer": "layer.77.self_attn.v_proj", + "ppl": 4.0098395347595215 + }, + { + "layer": "layer.77.self_attn.o_proj", + "ppl": 4.009845733642578 + }, + { + "layer": "layer.77.mlp.gate_proj", + "ppl": 4.0095601081848145 + }, + { + "layer": "layer.77.mlp.up_proj", + "ppl": 4.009745121002197 + }, + { + "layer": "layer.77.mlp.down_proj", + "ppl": 4.009690284729004 + }, + { + "layer": "layer.78.self_attn.q_proj", + "ppl": 4.00980281829834 + }, + { + "layer": "layer.78.self_attn.k_proj", + "ppl": 4.009782314300537 + }, + { + "layer": "layer.78.self_attn.v_proj", + "ppl": 4.009831428527832 + }, + { + "layer": "layer.78.self_attn.o_proj", + "ppl": 4.009876251220703 + }, + { + "layer": "layer.78.mlp.gate_proj", + "ppl": 4.009587287902832 + }, + { + "layer": "layer.78.mlp.up_proj", + "ppl": 4.009625434875488 + }, + { + "layer": "layer.78.mlp.down_proj", + "ppl": 4.00956392288208 + }, + { + "layer": "layer.79.self_attn.q_proj", + "ppl": 4.0097432136535645 + }, + { + "layer": "layer.79.self_attn.k_proj", + "ppl": 4.00981330871582 + }, + { + "layer": "layer.79.self_attn.v_proj", + "ppl": 4.009725093841553 + }, + { + "layer": "layer.79.self_attn.o_proj", + "ppl": 4.009735584259033 + }, + { + "layer": "layer.79.mlp.gate_proj", + "ppl": 4.009614944458008 + }, + { + "layer": "layer.79.mlp.up_proj", + "ppl": 4.009734630584717 + }, + { + "layer": "layer.79.mlp.down_proj", + "ppl": 4.009565353393555 + } + ] + } + ] +} \ No newline at end of file