You can achieve a nonlinear layout using multiple mmap() calls with MAP_FIXED. But each such call creates a separate VMA (Virtual Memory Area) kernel data structure. Problems with many VMAs:

remap_file_pages() solves this by rearranging the page table entries within a single existing VMA — no new VMAs are created.

#define _GNU_SOURCE
#include <sys/mman.h>

int remap_file_pages(void *addr,    /* Address within an existing mmap region   */
                     size_t size,   /* Size of the sub-region to remap (bytes)   */
                     int prot,      /* Must be 0 (ignored, reserved for future)  */
                     size_t pgoff,  /* File offset in PAGE SIZE units             */
                     int flags);    /* Must be 0 (currently unused)              */

/* Returns 0 on success, -1 on error */

This is the exact example from the book. A 3-page file is mapped, then pages 0 and 2 are swapped in the memory view.

#define _GNU_SOURCE
#include <stdio.h>
#include <stdlib.h>
#include <fcntl.h>
#include <sys/mman.h>
#include <unistd.h>
#include <string.h>

int main(void) {
    long ps = sysconf(_SC_PAGESIZE);
    int fd;
    char *addr;

    /* Step 1: Create a 3-page file, write identifiable content to each page */
    fd = open("nonlinear.dat", O_RDWR | O_CREAT | O_TRUNC, 0644);
    if (fd == -1) { perror("open"); exit(1); }
    if (ftruncate(fd, 3 * ps) == -1) { perror("ftruncate"); exit(1); }

    /* Map the file first so we can write to it */
    char *init = mmap(NULL, 3 * ps, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
    if (init == MAP_FAILED) { perror("mmap init"); exit(1); }

    /* Write a marker at the start of each file page */
    memset(init,          'A', ps);   /* file page 0: all 'A' */
    memset(init + ps,     'B', ps);   /* file page 1: all 'B' */
    memset(init + 2 * ps, 'C', ps);   /* file page 2: all 'C' */
    munmap(init, 3 * ps);

    /* Step 2: Create the main MAP_SHARED mapping for remap_file_pages() */
    addr = mmap(NULL, 3 * ps, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
    if (addr == MAP_FAILED) { perror("mmap main"); exit(1); }
    close(fd);

    printf("Before remap:\n");
    printf("  Memory page 0: '%c'\n", addr[0]);        /* A */
    printf("  Memory page 1: '%c'\n", addr[ps]);       /* B */
    printf("  Memory page 2: '%c'\n", addr[2 * ps]);   /* C */

    /*
     * Step 3: Rearrange — swap file pages 0 and 2 in memory view
     *
     * remap_file_pages(addr,          ps, 0, 2, 0)
     *   → memory page 0 now maps file page 2 (pgoff=2)
     *
     * remap_file_pages(addr + 2*ps,   ps, 0, 0, 0)
     *   → memory page 2 now maps file page 0 (pgoff=0)
     */
    if (remap_file_pages(addr, ps, 0, 2, 0) == -1) {
        perror("remap_file_pages page0→file2");
        exit(1);
    }
    if (remap_file_pages(addr + 2 * ps, ps, 0, 0, 0) == -1) {
        perror("remap_file_pages page2→file0");
        exit(1);
    }

    printf("\nAfter remap:\n");
    printf("  Memory page 0: '%c'  (was A, now C)\n", addr[0]);
    printf("  Memory page 1: '%c'  (unchanged B)\n", addr[ps]);
    printf("  Memory page 2: '%c'  (was C, now A)\n", addr[2 * ps]);

    munmap(addr, 3 * ps);
    return 0;
}

/*
 * Expected output:
 * Before remap:
 *   Memory page 0: 'A'
 *   Memory page 1: 'B'
 *   Memory page 2: 'C'
 *
 * After remap:
 *   Memory page 0: 'C'  (was A, now C)
 *   Memory page 1: 'B'  (unchanged B)
 *   Memory page 2: 'A'  (was C, now A)
 */

One powerful feature of remap_file_pages(): the same file page can appear at multiple addresses in the mapping simultaneously.

#define _GNU_SOURCE
#include <stdio.h>
#include <stdlib.h>
#include <fcntl.h>
#include <sys/mman.h>
#include <unistd.h>
#include <string.h>

int main(void) {
    long ps = sysconf(_SC_PAGESIZE);
    int fd;
    char *addr;

    fd = open("repeat.dat", O_RDWR | O_CREAT | O_TRUNC, 0644);
    ftruncate(fd, 4 * ps);

    char *init = mmap(NULL, 4 * ps, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
    memset(init,      'X', ps);   /* file page 0: 'X' */
    memset(init + ps, 'Y', ps);   /* file page 1: 'Y' */
    memset(init + 2 * ps, 'Z', ps); /* file page 2: 'Z' */
    memset(init + 3 * ps, 'W', ps); /* file page 3: 'W' */
    munmap(init, 4 * ps);

    addr = mmap(NULL, 4 * ps, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
    close(fd);

    /* Map file page 0 into both memory page 0 AND memory page 3 */
    remap_file_pages(addr,          ps, 0, 0, 0);  /* page 0 → file page 0 (no change) */
    remap_file_pages(addr + 3 * ps, ps, 0, 0, 0);  /* page 3 → file page 0 (same!) */

    printf("Memory page 0: '%c'\n", addr[0]);           /* X */
    printf("Memory page 3: '%c'\n", addr[3 * ps]);      /* X (same file page) */

    /* Write via memory page 0 — visible through memory page 3 */
    addr[0] = '!';
    printf("After write via page 0:\n");
    printf("  page 0: '%c'\n", addr[0]);         /* ! */
    printf("  page 3: '%c'\n", addr[3 * ps]);    /* ! (same physical page) */

    munmap(addr, 4 * ps);
    return 0;
}

This example demonstrates the VMA savings. Approach A uses multiple mmap()+MAP_FIXED. Approach B uses one mmap() + multiple remap_file_pages().

#define _GNU_SOURCE
#include <stdio.h>
#include <stdlib.h>
#include <fcntl.h>
#include <sys/mman.h>
#include <unistd.h>

int count_vmas(void) {
    FILE *f = fopen("/proc/self/maps", "r");
    int count = 0;
    char buf[512];
    while (fgets(buf, sizeof(buf), f)) count++;
    fclose(f);
    return count;
}

int main(void) {
    long ps = sysconf(_SC_PAGESIZE);
    int fd, n = 100;  /* number of nonlinear remaps */
    char *region;

    fd = open("large.dat", O_RDWR | O_CREAT | O_TRUNC, 0644);
    ftruncate(fd, n * ps);

    /* ---- Approach B: one mmap + remap_file_pages ---- */
    region = mmap(NULL, n * ps, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);

    printf("VMAs before remap_file_pages: %d\n", count_vmas());

    /* Rearrange: reverse the order of all pages */
    for (int i = 0; i < n; i++) {
        remap_file_pages(region + i * ps, ps, 0, n - 1 - i, 0);
    }

    printf("VMAs after %d remap_file_pages calls: %d\n", n, count_vmas());
    /* Still ONE VMA for this region — not 100 */

    munmap(region, n * ps);
    close(fd);
    return 0;
}
/* Typical output:
   VMAs before remap_file_pages: 14
   VMAs after 100 remap_file_pages calls: 14  ← same! */