/* * SPDX-License-Identifier: CDDL 1.0 * * Copyright (c) 2022 Christos Margiolis * Copyright (c) 2023 The FreeBSD Foundation * * Portions of this software were developed by Christos Margiolis * under sponsorship from the FreeBSD Foundation. */ #include #include #include #include #include #include #include #include "kinst.h" MALLOC_DEFINE(M_KINST, "kinst", "Kernel Instruction Tracing"); static d_open_t kinst_open; static d_close_t kinst_close; static d_ioctl_t kinst_ioctl; static void kinst_provide_module(void *, modctl_t *); static void kinst_getargdesc(void *, dtrace_id_t, void *, dtrace_argdesc_t *); static void kinst_destroy(void *, dtrace_id_t, void *); static void kinst_enable(void *, dtrace_id_t, void *); static void kinst_disable(void *, dtrace_id_t, void *); static int kinst_load(void *); static int kinst_unload(void *); static int kinst_modevent(module_t, int, void *); static dtrace_pattr_t kinst_attr = { { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON }, { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN }, { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_ISA }, { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON }, { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_ISA }, }; static const dtrace_pops_t kinst_pops = { .dtps_provide = NULL, .dtps_provide_module = kinst_provide_module, .dtps_enable = kinst_enable, .dtps_disable = kinst_disable, .dtps_suspend = NULL, .dtps_resume = NULL, .dtps_getargdesc = kinst_getargdesc, .dtps_getargval = NULL, .dtps_usermode = NULL, .dtps_destroy = kinst_destroy }; static struct cdevsw kinst_cdevsw = { .d_name = "kinst", .d_version = D_VERSION, .d_flags = D_TRACKCLOSE, .d_open = kinst_open, .d_close = kinst_close, .d_ioctl = kinst_ioctl, }; static dtrace_provider_id_t kinst_id; struct kinst_probe_list *kinst_probetab; static struct cdev *kinst_cdev; /* * Tracing memcpy() will crash the kernel when kinst tries to trace an instance * of the memcpy() calls in kinst_invop(). To fix this, we can use * kinst_memcpy() in those cases, with its arguments marked as 'volatile' to * "outsmart" the compiler and avoid having it replaced by a regular memcpy(). */ volatile void * kinst_memcpy(volatile void *dst, volatile const void *src, size_t len) { volatile const unsigned char *src0; volatile unsigned char *dst0; src0 = src; dst0 = dst; while (len--) *dst0++ = *src0++; return (dst); } bool kinst_excluded(const char *name) { if (kinst_md_excluded(name)) return (true); /* * cpu_switch() can cause a crash if it modifies the value of curthread * while in probe context. */ if (strcmp(name, "cpu_switch") == 0) return (true); /* * Anything beginning with "dtrace_" may be called from probe context * unless it explicitly indicates that it won't be called from probe * context by using the prefix "dtrace_safe_". */ if (strncmp(name, "dtrace_", strlen("dtrace_")) == 0 && strncmp(name, "dtrace_safe_", strlen("dtrace_safe_")) != 0) return (true); /* * Omit instrumentation of functions that are probably in DDB. It * makes it too hard to debug broken kinst. * * NB: kdb_enter() can be excluded, but its call to printf() can't be. * This is generally OK since we're not yet in debugging context. */ if (strncmp(name, "db_", strlen("db_")) == 0 || strncmp(name, "kdb_", strlen("kdb_")) == 0) return (true); /* * Lock owner methods may be called from probe context. */ if (strcmp(name, "owner_mtx") == 0 || strcmp(name, "owner_rm") == 0 || strcmp(name, "owner_rw") == 0 || strcmp(name, "owner_sx") == 0) return (true); /* * The KMSAN runtime can't be instrumented safely. */ if (strncmp(name, "__msan", 6) == 0 || strncmp(name, "kmsan_", 6) == 0) return (1); /* * When DTrace is built into the kernel we need to exclude the kinst * functions from instrumentation. */ #ifndef _KLD_MODULE if (strncmp(name, "kinst_", strlen("kinst_")) == 0) return (true); #endif if (strcmp(name, "trap_check") == 0) return (true); return (false); } void kinst_probe_create(struct kinst_probe *kp, linker_file_t lf) { kp->kp_id = dtrace_probe_create(kinst_id, lf->filename, kp->kp_func, kp->kp_name, 3, kp); LIST_INSERT_HEAD(KINST_GETPROBE(kp->kp_patchpoint), kp, kp_hashnext); } static int kinst_open(struct cdev *dev __unused, int oflags __unused, int devtype __unused, struct thread *td __unused) { return (0); } static int kinst_close(struct cdev *dev __unused, int fflag __unused, int devtype __unused, struct thread *td __unused) { dtrace_condense(kinst_id); return (0); } static int kinst_linker_file_cb(linker_file_t lf, void *arg) { dtrace_kinst_probedesc_t *pd; pd = arg; if (pd->kpd_mod[0] != '\0' && strcmp(pd->kpd_mod, lf->filename) != 0) return (0); /* * Invoke kinst_make_probe_function() once for each function symbol in * the module "lf". */ return (linker_file_function_listall(lf, kinst_make_probe, arg)); } static int kinst_ioctl(struct cdev *dev __unused, u_long cmd, caddr_t addr, int flags __unused, struct thread *td __unused) { dtrace_kinst_probedesc_t *pd; int error = 0; switch (cmd) { case KINSTIOC_MAKEPROBE: pd = (dtrace_kinst_probedesc_t *)addr; pd->kpd_func[sizeof(pd->kpd_func) - 1] = '\0'; pd->kpd_mod[sizeof(pd->kpd_mod) - 1] = '\0'; /* Loop over all functions in the kernel and loaded modules. */ error = linker_file_foreach(kinst_linker_file_cb, pd); break; default: error = ENOTTY; break; } return (error); } static void kinst_provide_module(void *arg, modctl_t *lf) { } static void kinst_getargdesc(void *arg, dtrace_id_t id, void *parg, dtrace_argdesc_t *desc) { desc->dtargd_ndx = DTRACE_ARGNONE; } static void kinst_destroy(void *arg, dtrace_id_t id, void *parg) { struct kinst_probe *kp = parg; LIST_REMOVE(kp, kp_hashnext); #ifndef __amd64__ kinst_trampoline_dealloc(kp->kp_tramp); #endif free(kp, M_KINST); } static void kinst_enable(void *arg, dtrace_id_t id, void *parg) { struct kinst_probe *kp = parg; static bool warned = false; if (!warned) { KINST_LOG( "kinst: This provider is experimental, exercise caution"); warned = true; } kinst_patch_tracepoint(kp, kp->kp_patchval); } static void kinst_disable(void *arg, dtrace_id_t id, void *parg) { struct kinst_probe *kp = parg; kinst_patch_tracepoint(kp, kp->kp_savedval); } static int kinst_load(void *dummy) { int error; error = kinst_trampoline_init(); if (error != 0) return (error); error = kinst_md_init(); if (error != 0) { kinst_trampoline_deinit(); return (error); } error = dtrace_register("kinst", &kinst_attr, DTRACE_PRIV_USER, NULL, &kinst_pops, NULL, &kinst_id); if (error != 0) { kinst_md_deinit(); kinst_trampoline_deinit(); return (error); } kinst_probetab = malloc(KINST_PROBETAB_MAX * sizeof(struct kinst_probe_list), M_KINST, M_WAITOK | M_ZERO); for (int i = 0; i < KINST_PROBETAB_MAX; i++) LIST_INIT(&kinst_probetab[i]); kinst_cdev = make_dev(&kinst_cdevsw, 0, UID_ROOT, GID_WHEEL, 0600, "dtrace/kinst"); dtrace_invop_add(kinst_invop); return (0); } static int kinst_unload(void *dummy) { free(kinst_probetab, M_KINST); kinst_md_deinit(); kinst_trampoline_deinit(); dtrace_invop_remove(kinst_invop); destroy_dev(kinst_cdev); return (dtrace_unregister(kinst_id)); } static int kinst_modevent(module_t mod __unused, int type, void *data __unused) { int error = 0; switch (type) { case MOD_LOAD: break; case MOD_UNLOAD: break; case MOD_SHUTDOWN: break; default: error = EOPNOTSUPP; break; } return (error); } SYSINIT(kinst_load, SI_SUB_DTRACE_PROVIDER, SI_ORDER_ANY, kinst_load, NULL); SYSUNINIT(kinst_unload, SI_SUB_DTRACE_PROVIDER, SI_ORDER_ANY, kinst_unload, NULL); DEV_MODULE(kinst, kinst_modevent, NULL); MODULE_VERSION(kinst, 1); MODULE_DEPEND(kinst, dtrace, 1, 1, 1); MODULE_DEPEND(kinst, opensolaris, 1, 1, 1);