From d495a414643fa6dce2a2df794725d2a0cea7b670 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Kochma=C5=84ski?= Date: Mon, 2 Mar 2026 21:32:46 +0100 Subject: [PATCH] reader: add a new object to the system ecl_token It represents unparsed token constructed by the reader. --- src/c/alloc_2.d | 5 +++++ src/c/clos/instance.d | 3 +++ src/c/printer/write_ugly.d | 7 +++++++ src/c/serialize.d | 1 + src/c/symbols_list.h | 1 + src/c/typespec.d | 2 ++ src/clos/hierarchy.lsp | 1 + src/h/object.h | 9 +++++++++ 8 files changed, 29 insertions(+) diff --git a/src/c/alloc_2.d b/src/c/alloc_2.d index 54a821ee4..57c863983 100644 --- a/src/c/alloc_2.d +++ b/src/c/alloc_2.d @@ -337,6 +337,7 @@ ecl_alloc_object(cl_type t) case t_mailbox: #endif case t_foreign: + case t_token: case t_codeblock: { cl_object obj; ecl_disable_interrupts_env(the_env); @@ -553,6 +554,7 @@ void init_type_info (void) init_tm(t_codeblock, "CODEBLOCK", sizeof(struct ecl_codeblock), -1); init_tm(t_foreign, "FOREIGN", sizeof(struct ecl_foreign), 2); init_tm(t_frame, "STACK-FRAME", sizeof(struct ecl_stack_frame), 0); + init_tm(t_token, "TOKEN", sizeof(struct ecl_token), 2); init_tm(t_weak_pointer, "WEAK-POINTER", sizeof(struct ecl_weak_pointer), 0); #ifdef ECL_SSE2 init_tm(t_sse_pack, "SSE-PACK", sizeof(struct ecl_sse_pack), 0); @@ -712,6 +714,9 @@ void init_type_info (void) to_bitmap(&o, &(o.foreign.tag)); type_info[t_frame].descriptor = to_bitmap(&o, &(o.frame.env)); + type_info[t_token].descriptor = + to_bitmap(&o, &(o.token.string)) | + to_bitmap(&o, &(o.token.escape))); type_info[t_weak_pointer].descriptor = 0; #ifdef ECL_SSE2 type_info[t_sse_pack].descriptor = 0; diff --git a/src/c/clos/instance.d b/src/c/clos/instance.d index 54bb496c0..27d7d31f8 100644 --- a/src/c/clos/instance.d +++ b/src/c/clos/instance.d @@ -394,6 +394,7 @@ enum ecl_built_in_classes { ECL_BUILTIN_CODE_BLOCK, ECL_BUILTIN_FOREIGN_DATA, ECL_BUILTIN_FRAME, + ECL_BUILTIN_TOKEN, ECL_BUILTIN_WEAK_POINTER, ECL_BUILTIN_PROCESS, ECL_BUILTIN_LOCK, @@ -511,6 +512,8 @@ cl_class_of(cl_object x) index = ECL_BUILTIN_FOREIGN_DATA; break; case t_frame: index = ECL_BUILTIN_FRAME; break; + case t_token: + index = ECL_BUILTIN_TOKEN; break; case t_weak_pointer: index = ECL_BUILTIN_WEAK_POINTER; break; #ifdef ECL_SSE2 diff --git a/src/c/printer/write_ugly.d b/src/c/printer/write_ugly.d index d99672ee1..b5f97767b 100644 --- a/src/c/printer/write_ugly.d +++ b/src/c/printer/write_ugly.d @@ -370,6 +370,12 @@ write_frame(cl_object x, cl_object stream) _ecl_write_unreadable(x, "frame", ecl_make_fixnum(x->frame.size), stream); } +static void +write_token(cl_object x, cl_object stream) +{ + _ecl_write_unreadable(x, "token", x->token.string, stream); +} + static void write_weak_pointer(cl_object x, cl_object stream) { @@ -480,6 +486,7 @@ static printer dispatch[FREE+1] = { write_codeblock, /* t_codeblock */ write_foreign, /* t_foreign */ write_frame, /* t_frame */ + write_token, /* t_token */ write_weak_pointer, /* t_weak_pointer */ #ifdef ECL_SSE2 _ecl_write_sse, /* t_sse_pack */ diff --git a/src/c/serialize.d b/src/c/serialize.d index d5d79a8fb..7edd1cb29 100644 --- a/src/c/serialize.d +++ b/src/c/serialize.d @@ -76,6 +76,7 @@ static cl_index object_size[] = { ROUNDED_SIZE(ecl_codeblock), /* t_codeblock */ ROUNDED_SIZE(ecl_foreign), /* t_foreign */ ROUNDED_SIZE(ecl_stack_frame), /* t_frame */ + ROUNDED_SIZE(ecl_token), /* t_token */ ROUNDED_SIZE(ecl_weak_pointer) /* t_weak_pointer */ #ifdef ECL_SSE2 , ROUNDED_SIZE(ecl_sse_pack) /* t_sse_pack */ diff --git a/src/c/symbols_list.h b/src/c/symbols_list.h index dd2917304..c23ca7160 100644 --- a/src/c/symbols_list.h +++ b/src/c/symbols_list.h @@ -1847,6 +1847,7 @@ cl_symbols[] = { {SYS_ "CODE-BLOCK" ECL_FUN(NULL, NULL, -1) ECL_VAR(SI_ORDINARY, OBJNULL)}, +{SYS_ "TOKEN" ECL_FUN(NULL, NULL, -1) ECL_VAR(SI_ORDINARY, OBJNULL)}, {SYS_ "FRAME" ECL_FUN(NULL, NULL, -1) ECL_VAR(SI_ORDINARY, OBJNULL)}, {SYS_ "APPLY-FROM-STACK-FRAME" ECL_FUN("si_apply_from_stack_frame", si_apply_from_stack_frame, 2) ECL_VAR(SI_ORDINARY, OBJNULL)}, diff --git a/src/c/typespec.d b/src/c/typespec.d index 19e89538d..bca1f5c67 100644 --- a/src/c/typespec.d +++ b/src/c/typespec.d @@ -181,6 +181,8 @@ ecl_type_to_symbol(cl_type t) return @'si::foreign-data'; case t_frame: return @'si::frame'; + case t_token: + return @'si::token'; case t_weak_pointer: return @'ext::weak-pointer'; #ifdef ECL_SSE2 diff --git a/src/clos/hierarchy.lsp b/src/clos/hierarchy.lsp index 33b5c4ca4..4a40329a3 100644 --- a/src/clos/hierarchy.lsp +++ b/src/clos/hierarchy.lsp @@ -228,6 +228,7 @@ (si::code-block) (si::foreign-data) (si::frame) + (si::token) (si::weak-pointer) (:threads mp::process) (:threads mp::lock) diff --git a/src/h/object.h b/src/h/object.h index 383490f3d..c3f579fe1 100644 --- a/src/h/object.h +++ b/src/h/object.h @@ -84,6 +84,7 @@ typedef enum { t_codeblock, t_foreign, t_frame, + t_token, t_weak_pointer, #ifdef ECL_SSE2 t_sse_pack, @@ -947,6 +948,13 @@ struct ecl_stack_frame { struct cl_env_struct *env; }; +/* Token is constructed by the reader from constituents and then parsed. */ +struct ecl_token { + _ECL_HDR1(escaped); + cl_object string; /* the token string */ + cl_object escape; /* ranges of escaped characters */ +}; + struct ecl_weak_pointer { /* weak pointer to value */ _ECL_HDR; cl_object value; @@ -1175,6 +1183,7 @@ union cl_lispunion { struct ecl_cclosure cclosure; /* compiled closure */ struct ecl_dummy d; /* dummy */ struct ecl_instance instance; /* clos instance */ + struct ecl_token token; /* token */ #ifdef ECL_THREADS struct ecl_process process; /* process */ struct ecl_lock lock; /* lock */