The error message says that caps changed and matroskamux doesn’t support this.
You may try to scale each decoded source to 720p, set fixed caps and add -v flag, so that you’ll get some details about what caps changed before the error:
gst-launch-1.0 -ev nvcompositor name=mix \
sink_0::xpos=0 sink_0::ypos=0 sink_0::width=1280 sink_0::height=720 \
sink_1::xpos=1280 sink_1::ypos=0 sink_1::width=1280 sink_1::height=720 \
sink_2::xpos=0 sink_2::ypos=720 sink_2::width=1280 sink_2::height=720 \
sink_3::xpos=1280 sink_3::ypos=720 sink_3::width=1280 sink_3::height=720 \
! 'video/x-raw(memory:NVMM),format=RGBA,width=2560,height=1440,framerate=30/1' ! nvvidconv ! 'video/x-raw(memory:NVMM),format=NV12' ! nvv4l2h264enc insert-vui=1 profile=2 \
! h264parse ! video/x-h264,width=2560,height=1440,framerate=30/1 ! matroskamux ! filesink location=test.mkv sync=false \
rtspsrc location=rtsp://127.0.0.1:8554/test ! application/x-rtp,media=video,encoding-name=H264 ! rtph264depay ! h264parse ! nvv4l2decoder ! nvvidconv ! 'video/x-raw(memory:NVMM),format=NV12,width=1280,height=720,pixel-aspect-ratio=1/1' ! queue ! mix.sink_0 \
rtspsrc location=rtsp://127.0.0.1:8554/test ! application/x-rtp,media=video,encoding-name=H264 ! rtph264depay ! h264parse ! nvv4l2decoder ! nvvidconv ! 'video/x-raw(memory:NVMM),format=NV12,width=1280,height=720,pixel-aspect-ratio=1/1' ! queue ! mix.sink_1 \
rtspsrc location=rtsp://127.0.0.1:8554/test ! application/x-rtp,media=video,encoding-name=H264 ! rtph264depay ! h264parse ! nvv4l2decoder ! nvvidconv ! 'video/x-raw(memory:NVMM),format=NV12,width=1280,height=720,pixel-aspect-ratio=1/1' ! queue ! mix.sink_2 \
rtspsrc location=rtsp://127.0.0.1:8554/test ! application/x-rtp,media=video,encoding-name=H264 ! rtph264depay ! h264parse ! nvv4l2decoder ! nvvidconv ! 'video/x-raw(memory:NVMM),format=NV12,width=1280,height=720,pixel-aspect-ratio=1/1' ! queue ! mix.sink_3