SUSv3 defines two limits that implementations must honour:

MQ_OPEN_MAX on Linux: Linux does not define this limit separately. Since message queue descriptors are file descriptors on Linux, the applicable limits are RLIMIT_NOFILE (per-process open file limit) and the system-wide file descriptor limit. In other words, the total count of open files and open message queue descriptors together must not exceed these limits.

#include <mqueue.h>
#include <stdio.h>

int main(void) {
    /* Print standard limits */
    printf("MQ_PRIO_MAX = %ld\n", (long)MQ_PRIO_MAX);

    /* MQ_OPEN_MAX may not be defined on Linux */
#ifdef MQ_OPEN_MAX
    printf("MQ_OPEN_MAX = %ld\n", (long)MQ_OPEN_MAX);
#else
    printf("MQ_OPEN_MAX not defined (Linux uses fd limits)\n");
#endif

    return 0;
}

Linux exposes three files under /proc/sys/fs/mqueue/ that control how message queues are created. These can be read and written (with root) at runtime:

Read current values:

cat /proc/sys/fs/mqueue/msg_max
# 10

cat /proc/sys/fs/mqueue/msgsize_max
# 8192

cat /proc/sys/fs/mqueue/queues_max
# 256

Change values (requires root):

# Allow up to 50 messages per queue
echo 50 > /proc/sys/fs/mqueue/msg_max

# Allow messages up to 65536 bytes
echo 65536 > /proc/sys/fs/mqueue/msgsize_max

# Allow up to 1024 queues system-wide
echo 1024 > /proc/sys/fs/mqueue/queues_max

# Permanent change (survives reboot) via sysctl.conf
echo "fs.mqueue.msg_max = 50"       >> /etc/sysctl.conf
echo "fs.mqueue.msgsize_max = 65536" >> /etc/sysctl.conf
echo "fs.mqueue.queues_max = 1024"  >> /etc/sysctl.conf
sysctl -p

When you call mq_open() with O_CREAT and provide a struct mq_attr, the kernel checks your requested values against the limits:

#include <mqueue.h>
#include <fcntl.h>
#include <stdio.h>
#include <stdlib.h>
#include <errno.h>

int main(void)
{
    struct mq_attr attr;
    mqd_t mqd;

    /* Try to create a queue with message size exceeding msgsize_max (8192) */
    attr.mq_flags   = 0;
    attr.mq_maxmsg  = 10;
    attr.mq_msgsize = 100000;   /* This exceeds the 8192 default */
    attr.mq_curmsgs = 0;

    mqd = mq_open("/test_limit", O_CREAT | O_RDWR, 0600, &attr);
    if (mqd == (mqd_t)-1) {
        if (errno == EINVAL)
            printf("EINVAL: mq_msgsize exceeds msgsize_max limit\n");
        else
            perror("mq_open");
        return 1;
    }

    printf("Queue created successfully.\n");
    mq_close(mqd);
    mq_unlink("/test_limit");
    return 0;
}
/* To fix: either reduce mq_msgsize, or raise the limit as root:
 *   echo 200000 > /proc/sys/fs/mqueue/msgsize_max
 * or run the program as a privileged process. */

Privileged processes (CAP_SYS_RESOURCE):

• msg_max is bypassed — but HARD_MSGMAX (kernel constant) still applies.
• msgsize_max is completely ignored — the process can set any size.
• Once queues_max is reached, only privileged processes can create more queues.

Linux also enforces RLIMIT_MSGQUEUE, which limits the total amount of kernel memory that can be consumed by all message queues owned by the calling process’s real user ID.

This is measured in bytes. The accounting formula per queue is approximately:

/* Memory charged for each queue: */
bytes = mq_maxmsg * (mq_msgsize + sizeof(struct msg_msg));

View and change per-process limits with getrlimit / setrlimit:

#include <sys/resource.h>
#include <stdio.h>

int main(void)
{
    struct rlimit rl;

    if (getrlimit(RLIMIT_MSGQUEUE, &rl) == -1) {
        perror("getrlimit");
        return 1;
    }

    printf("RLIMIT_MSGQUEUE: soft=%ld, hard=%ld\n",
           (long)rl.rlim_cur, (long)rl.rlim_max);

    /* Raise soft limit to hard limit */
    rl.rlim_cur = rl.rlim_max;
    if (setrlimit(RLIMIT_MSGQUEUE, &rl) == -1)
        perror("setrlimit");

    return 0;
}

Or check and set via the shell:

# Show current limit
ulimit -q
# 819200  (800 KB default on many systems)

# Increase to 4 MB for current shell session
ulimit -q 4194304

#include <mqueue.h>
#include <stdio.h>
#include <stdlib.h>
#include <sys/resource.h>

/* Read an integer from a /proc file */
static long read_proc(const char *path)
{
    FILE *fp = fopen(path, "r");
    long val = -1;
    if (fp) { fscanf(fp, "%ld", &val); fclose(fp); }
    return val;
}

int main(void)
{
    struct rlimit rl;

    printf("=== POSIX Message Queue Limits ===\n\n");

    /* Standard limits */
    printf("MQ_PRIO_MAX (max message priority)  : %d\n", MQ_PRIO_MAX);

    /* /proc tunables */
    printf("\n/proc/sys/fs/mqueue tunables:\n");
    printf("  msg_max     (max msgs per queue)   : %ld\n",
           read_proc("/proc/sys/fs/mqueue/msg_max"));
    printf("  msgsize_max (max msg size, bytes)  : %ld\n",
           read_proc("/proc/sys/fs/mqueue/msgsize_max"));
    printf("  queues_max  (max queues system-wide): %ld\n",
           read_proc("/proc/sys/fs/mqueue/queues_max"));

    /* Per-process resource limit */
    if (getrlimit(RLIMIT_MSGQUEUE, &rl) == 0)
        printf("\nRLIMIT_MSGQUEUE (per-user memory cap): soft=%ld hard=%ld\n",
               (long)rl.rlim_cur, (long)rl.rlim_max);

    /* File descriptor limit (applies to mqd on Linux) */
    if (getrlimit(RLIMIT_NOFILE, &rl) == 0)
        printf("RLIMIT_NOFILE   (fd + mqd combined)  : soft=%ld hard=%ld\n",
               (long)rl.rlim_cur, (long)rl.rlim_max);

    return 0;
}