There is a catch-22 when RLIMIT_STACK is exceeded: the kernel delivers SIGSEGV to the process, but to handle that signal the process needs to execute a signal handler — and that handler needs stack space. Since the stack is already full, there is nowhere to run the handler.

The solution is an alternate signal stack. You allocate a separate region of memory, register it with sigaltstack(), and install the SIGSEGV handler with the SA_ONSTACK flag. Then even when the main stack overflows, the handler can run on the alternate stack.

/* Setting up an alternate signal stack for stack overflow handling */
#include <signal.h>
#include <stdlib.h>

#define ALT_STACK_SIZE (SIGSTKSZ * 4)  /* typically 32 KB */

void setup_alt_stack(void)
{
    static char alt_stack_buf[ALT_STACK_SIZE];
    stack_t ss;
    struct sigaction sa;

    /* Register the alternate stack */
    ss.ss_sp    = alt_stack_buf;
    ss.ss_size  = ALT_STACK_SIZE;
    ss.ss_flags = 0;
    sigaltstack(&ss, NULL);

    /* Install SIGSEGV handler that runs on the alternate stack */
    sa.sa_handler = sigsegv_handler;
    sigemptyset(&sa.sa_mask);
    sa.sa_flags = SA_ONSTACK;  /* Run on alternate stack */
    sigaction(SIGSEGV, &sa, NULL);
}

#include 
#include 
#include 
#include <sys/resource.h>

/* Alternate stack storage */
static char alt_stk[65536];

static void sigsegv_handler(int sig)
{
    /* Runs on alternate stack even if main stack overflowed */
    printf("SIGSEGV caught: stack overflow detected!\n");
    _exit(1);   /* Must use _exit() not exit() in signal handler */
}

/* Recursive function that will overflow the stack */
static void recurse(int depth)
{
    char local_buf[4096];  /* Force stack growth each call */
    local_buf[0] = depth;  /* Touch to prevent optimization */
    printf("  depth %d\n", depth);
    recurse(depth + 1);    /* Infinite recursion */
}

int main(void)
{
    struct rlimit rl;
    stack_t ss;
    struct sigaction sa;

    /* Set a small stack limit: 128 KB */
    rl.rlim_cur = 128 * 1024;
    rl.rlim_max = 128 * 1024;
    setrlimit(RLIMIT_STACK, &rl);
    printf("Stack limit set to 128 KB\n");

    /* Set up alternate signal stack */
    ss.ss_sp    = alt_stk;
    ss.ss_size  = sizeof(alt_stk);
    ss.ss_flags = 0;
    sigaltstack(&ss, NULL);

    /* Install SIGSEGV handler on alternate stack */
    sa.sa_handler = sigsegv_handler;
    sigemptyset(&sa.sa_mask);
    sa.sa_flags = SA_ONSTACK;
    sigaction(SIGSEGV, &sa, NULL);

    printf("Starting deep recursion...\n");
    recurse(0);  /* Will overflow and trigger SIGSEGV */

    return 0;
}
/* Compile: gcc -O0 -o stack_demo stack_demo.c
   Expected: recurse() runs until stack is full, then SIGSEGV handler fires */