Fixed some odd bugs in NGC flow control, prepared for file based named O-call subroutines.

Fixed incorrect comment string passed to passed to grbl.on_gcode_comment event.
Added generic redirector for temporarily changing input stream to read from a file. Supports nesting.

CMakeLists.txt

@@ -21,6 +21,7 @@ target_sources(grbl INTERFACE
  ${CMAKE_CURRENT_LIST_DIR}/spindle_control.c
  ${CMAKE_CURRENT_LIST_DIR}/state_machine.c
  ${CMAKE_CURRENT_LIST_DIR}/stream.c
+ ${CMAKE_CURRENT_LIST_DIR}/stream_file.c
  ${CMAKE_CURRENT_LIST_DIR}/stepper.c
  ${CMAKE_CURRENT_LIST_DIR}/stepper2.c
  ${CMAKE_CURRENT_LIST_DIR}/system.c

README.md

@@ -13,7 +13,7 @@ It has been written to complement grblHAL and has features such as proper keyboa
 
 ---
 
-Latest build date is 20241016, see the [changelog](changelog.md) for details.  
+Latest build date is 20241023, see the [changelog](changelog.md) for details.  
 
 __NOTE:__ Build 20240222 has moved the probe input to the ioPorts pool of inputs and will be allocated from it when configured.
 The change is major and _potentially dangerous_, it may damage your probe, so please _verify correct operation_ after installing this, or later, builds.

changelog.md

@@ -1,5 +1,28 @@
 ## grblHAL changelog
 
+<a name="20241023">Build 20241023
+
+Core:
+
+* Fixed some odd bugs in NGC flow control, prepared for file based named O-call subroutines.
+
+* Fixed incorrect comment string passed to passed to `grbl.on_gcode_comment` event.
+
+* Added generic redirector for temporarily changing input stream to read from a file. Supports nesting.
+
+Drivers:
+
+*ESP32: fix for overriding UART0 pins, reverted  and fixed tests for ESP32-S3 conditional code.
+
+Plugins:
+
+* File system macros: updated to use new input stream redirector, allows nesting of `G65` calls
+ \(max 5 levels depending on available memory\). __NOTE:__ Not extensively tested, feedback required.
+
+* SD card: updated to work alongside new file redirector.
+
+---
+
 <a name="20241019">Build 20241019
 
 Core:

core_handlers.h

@@ -122,6 +122,7 @@ typedef void (*on_gcode_message_ptr)(char *msg);
 typedef void (*on_rt_reports_added_ptr)(report_tracking_flags_t report);
 typedef const char *(*on_set_axis_setting_unit_ptr)(setting_id_t setting_id, uint_fast8_t axis_idx);
 typedef status_code_t (*on_file_open_ptr)(const char *fname, vfs_file_t *handle, bool stream);
+typedef status_code_t (*on_file_end_ptr)(vfs_file_t *handle, status_code_t status);
 typedef status_code_t (*on_unknown_sys_command_ptr)(sys_state_t state, char *line); // return Status_Unhandled.
 typedef status_code_t (*on_user_command_ptr)(char *line);
 typedef sys_commands_t *(*on_get_commands_ptr)(void);
@@ -252,6 +253,7 @@ typedef struct {
     on_spindle_selected_ptr on_spindle_selected;        //!< Called when spindle is selected, do not change HAL pointers here!
     on_reset_ptr on_reset;                              //!< Called from interrupt context.
     on_file_open_ptr on_file_open;                      //!< Called when a file is opened for streaming.
+    on_file_end_ptr on_file_end;                        //!< Called when a file opened for streaming reaches the end.
     user_mcode_ptrs_t user_mcode;                       //!< Optional handlers for user defined M-codes.
     // core entry points - set up by core before driver_init() is called.
     home_machine_ptr home_machine;

gcode.c

@@ -37,6 +37,7 @@
 #if NGC_EXPRESSIONS_ENABLE
 #include "ngc_expr.h"
 #include "ngc_flowctrl.h"
+//#include "string_registers.h"
 #ifndef NGC_N_ASSIGN_PARAMETERS_PER_BLOCK
 #define NGC_N_ASSIGN_PARAMETERS_PER_BLOCK 10
 #endif
@@ -587,7 +588,7 @@ char *gc_normalize_block (char *block, char **message)
 
             case '(':
                 // TODO: generate error if a left parenthesis is found inside a comment...
-                comment = s1;
+                comment = s1 + 1;
                 break;
 
             case ')':
@@ -597,27 +598,27 @@ char *gc_normalize_block (char *block, char **message)
 
                         if(message && *message == NULL) {
 #if NGC_EXPRESSIONS_ENABLE
-                            if(!strncmp(comment, "(DEBUG,", 7)) { // Debug message string substitution
+                            if(!strncasecmp(comment, "DEBUG,", 6)) { // Debug message string substitution
                                 if(settings.flags.ngc_debug_out) {
-                                    comment += 7;
+                                    comment += 6;
                                     ngc_substitute_parameters(comment, message);
                                 }
                                 *comment = '\0'; // Do not generate grbl.on_gcode_comment event!
-                            } else if(!strncmp(comment, "(PRINT,", 7)) { // Print message string substitution
-                                comment += 7;
+                            } else if(!strncasecmp(comment, "PRINT,", 6)) { // Print message string substitution
+                                comment += 6;
                                 ngc_substitute_parameters(comment, message);
                                 *comment = '\0'; // Do not generate grbl.on_gcode_comment event!
-                            } else if(!strncmp(comment, "(MSG,", 5)) {
-                                    comment += 5;
+                            } else if(!strncasecmp(comment, "MSG,", 4)) {
+                                    comment += 4;
                                     ngc_substitute_parameters(comment, message);
                                 }
                             }
 #else
-                            size_t len = s1 - comment - 4;
+                            size_t len = s1 - comment - 3;
 
-                            if(!strncmp(comment, "(MSG,", 5) && (*message = malloc(len))) {
+                            if(!strncasecmp(comment, "MSG,", 4) && (*message = malloc(len))) {
 
-                                comment += 5;
+                                comment += 4;
                                 while(*comment == ' ') {
                                     comment++;
                                     len--;
@@ -639,14 +640,6 @@ char *gc_normalize_block (char *block, char **message)
                     *s2++ = CAPS(c);
                 break;
         }
-
-#if NGC_EXPRESSIONS_ENABLE
-        if(comment && s1 - comment < (strncmp(comment, "(DEBU", 5) && strncmp(comment, "(PRIN", 5) ? 5 : 7))
-            *s1 = CAPS(c);
-#else
-        if(comment && s1 - comment < 5)
-            *s1 = CAPS(c);
-#endif
         s1++;
     }
 
@@ -923,7 +916,7 @@ status_code_t gc_execute_block (char *block)
                 if((status = ngc_read_name(block, &char_counter, o_slabel)) != Status_OK)
                     FAIL(status);
                 gc_block.words.o = On;
-                if(gc_block.values.o = string_register_set_name(o_slabel) == 0)
+                if((gc_block.values.o = string_register_set_name(o_slabel)) == 0)
                     FAIL(Status_FlowControlOutOfMemory);
                 continue;
 #else

grbl.h

@@ -42,7 +42,7 @@
 #else
 #define GRBL_VERSION "1.1f"
 #endif
-#define GRBL_BUILD 20241019
+#define GRBL_BUILD 20241023
 
 #define GRBL_URL "https://github.com/grblHAL"
 

ngc_flowctrl.c

@@ -25,11 +25,15 @@
 
 #if NGC_EXPRESSIONS_ENABLE
 
+#include <math.h>
+#include <stdio.h>
 #include <string.h>
 
 #include "errors.h"
 #include "ngc_expr.h"
 #include "ngc_params.h"
+#include "stream_file.h"
+//#include "string_registers.h"
 
 #ifndef NGC_STACK_DEPTH
 #define NGC_STACK_DEPTH 20
@@ -193,6 +197,28 @@ static status_code_t read_command (char *line, uint_fast8_t *pos, ngc_cmd_t *ope
     return status;
 }
 
+static ngc_sub_t *add_sub (uint32_t o_label, vfs_file_t *file)
+{
+    ngc_sub_t *sub;
+
+    if((sub = malloc(sizeof(ngc_sub_t))) != NULL) {
+        sub->o_label = o_label;
+        sub->file = file;
+        sub->file_pos = vfs_tell(file);
+        sub->next = NULL;
+        if(subs == NULL)
+            subs = sub;
+        else {
+            ngc_sub_t *last = subs;
+            while(last->next)
+                last = last->next;
+            last->next = sub;
+        }
+    }
+
+    return sub;
+}
+
 static void clear_subs (vfs_file_t *file)
 {
     ngc_sub_t *current = subs, *prev = NULL, *next;
@@ -249,7 +275,10 @@ static void stack_unwind_sub (uint32_t o_label)
         stack_pull();
 
     if(stack_idx >= 0) {
-        vfs_seek(stack[stack_idx].file, stack[stack_idx].file_pos);
+        if(o_label > NGC_MAX_PARAM_ID)
+            stream_redirect_close(stack[stack_idx].file);
+        else
+            vfs_seek(stack[stack_idx].file, stack[stack_idx].file_pos);
         stack_pull();
     }
 
@@ -272,6 +301,56 @@ void ngc_flowctrl_init (void)
         stack_pull();
 }
 
+/*
+
+static status_code_t onError (status_code_t status)
+{
+    static bool closing = false;
+
+    if(stack_idx >= 0) {
+
+        uint32_t o_label = 0;
+        ngc_sub_t *sub;
+
+        if((sub = subs)) do {
+            if(sub->o_label > NGC_MAX_PARAM_ID)
+                o_label = sub->o_label;
+        } while((sub = sub->next));
+
+        if((sub = subs)) do {
+            if(sub->o_label == o_label)
+                break;
+        } while((sub = sub->next));
+
+        if(sub) {
+            if(!closing) {
+                char msg[100];
+                closing = true;
+                char *name = string_register_get_by_id((string_register_id_t)sub->o_label);
+                sprintf(msg, "error %d in named sub %s.macro", (uint8_t)status, name);
+                report_message(msg, Message_Warning);
+            }
+
+            sub->o_label = 1;
+
+            stream_redirect_close(sub->file);
+            status = grbl.report.status_message(status);
+        }
+        closing = false;
+        ngc_flowctrl_init();
+    }
+
+    return status;
+}
+
+static status_code_t onFileEnd (vfs_file_t *file, status_code_t status)
+{
+    if(stack_idx >= 0 && stack[stack_idx].file == file)
+        ngc_flowctrl_unwind_stack(file);
+
+    return status;
+}
+*/
 status_code_t ngc_flowctrl (uint32_t o_label, char *line, uint_fast8_t *pos, bool *skip)
 {
     float value;
@@ -346,13 +425,11 @@ status_code_t ngc_flowctrl (uint32_t o_label, char *line, uint_fast8_t *pos, boo
                     if(last_op == NGCFlowCtrl_Do && o_label == stack[stack_idx].o_label)
                         stack_pull();
                 } else if(!skipping && (status = ngc_eval_expression(line, pos, &value)) == Status_OK) {
-                    if(last_op == NGCFlowCtrl_Do) {
-                        if(o_label == stack[stack_idx].o_label) {
-                            if(value != 0.0f)
-                                vfs_seek(stack[stack_idx].file, stack[stack_idx].file_pos);
-                            else
-                                stack_pull();
-                        }
+                    if(last_op == NGCFlowCtrl_Do && o_label == stack[stack_idx].o_label) {
+                        if(value != 0.0f)
+                            vfs_seek(stack[stack_idx].file, stack[stack_idx].file_pos);
+                        else
+                            stack_pull();
                     } else if((status = stack_push(o_label, operation)) == Status_OK) {
                         if(!(stack[stack_idx].skip = value == 0.0f)) {
                             if((stack[stack_idx].expr = malloc(strlen(expr) + 1))) {
@@ -371,11 +448,13 @@ status_code_t ngc_flowctrl (uint32_t o_label, char *line, uint_fast8_t *pos, boo
         case NGCFlowCtrl_EndWhile:
             if(hal.stream.file) {
                 if(last_op == NGCFlowCtrl_While) {
-                    if(!skipping && o_label == stack[stack_idx].o_label) {
-                        uint_fast8_t pos = 0;
-                        if(!stack[stack_idx].skip && (status = ngc_eval_expression(stack[stack_idx].expr, &pos, &value)) == Status_OK) {
-                            if(!(stack[stack_idx].skip = value == 0))
-                                vfs_seek(stack[stack_idx].file, stack[stack_idx].file_pos);
+                    if(o_label == stack[stack_idx].o_label) {
+                        if(!skipping) {
+                            uint_fast8_t pos = 0;
+                            if(!stack[stack_idx].skip && (status = ngc_eval_expression(stack[stack_idx].expr, &pos, &value)) == Status_OK) {
+                                if(!(stack[stack_idx].skip = value == 0.0f))
+                                    vfs_seek(stack[stack_idx].file, stack[stack_idx].file_pos);
+                            }
                         }
                         if(stack[stack_idx].skip)
                             stack_pull();
@@ -390,7 +469,8 @@ status_code_t ngc_flowctrl (uint32_t o_label, char *line, uint_fast8_t *pos, boo
             if(hal.stream.file) {
                 if(!skipping && (status = ngc_eval_expression(line, pos, &value)) == Status_OK) {
                     if((status = stack_push(o_label, operation)) == Status_OK) {
-                        if(!(stack[stack_idx].skip = value == 0.0f)) {
+                        value = nearbyintf(value);
+                        if(!(stack[stack_idx].skip = value <= 0.0f)) {
                             stack[stack_idx].file = hal.stream.file;
                             stack[stack_idx].file_pos = vfs_tell(hal.stream.file);
                             stack[stack_idx].repeats = (uint32_t)value;
@@ -490,20 +570,17 @@ status_code_t ngc_flowctrl (uint32_t o_label, char *line, uint_fast8_t *pos, boo
         case NGCFlowCtrl_Sub:
             if(hal.stream.file) {
                 ngc_sub_t *sub;
-                if((skip_sub = (sub = malloc(sizeof(ngc_sub_t))) != NULL)) {
-                    sub->o_label = o_label;
-                    sub->file = hal.stream.file;
-                    sub->file_pos = vfs_tell(hal.stream.file);
-                    sub->next = NULL;
-                    if(subs == NULL)
-                        subs = sub;
-                    else {
-                        ngc_sub_t *last = subs;
-                        while(last->next)
-                            last = last->next;
-                        last->next = sub;
-                    }
-                } // else out of memory
+                if(o_label > NGC_MAX_PARAM_ID) {
+
+                    if((sub = subs)) do {
+//                        if(sub->o_label == o_label && sub->file == hal.stream.file)
+//                            break;
+                    } while(sub->next && (sub = sub->next));
+
+                    if(sub == NULL || sub->o_label != o_label)
+                        status = Status_FlowControlSyntaxError;
+                } else if(!(skip_sub = (sub = add_sub(o_label, hal.stream.file)) != NULL))
+                    status = Status_FlowControlOutOfMemory;
             } else
                 status = Status_FlowControlNotExecutingMacro;
             break;
@@ -524,11 +601,37 @@ status_code_t ngc_flowctrl (uint32_t o_label, char *line, uint_fast8_t *pos, boo
             break;
 
         case NGCFlowCtrl_Call:
-            if(hal.stream.file) {
+
+            if(hal.stream.file || o_label > NGC_MAX_PARAM_ID) {
+
                 if(!skipping) {
 
                     ngc_sub_t *sub = subs;
-                    do {
+
+                    if(o_label > NGC_MAX_PARAM_ID) {
+#if 0
+                        char *subname;
+                        if((subname = string_register_get_by_id((string_register_id_t)o_label))) {
+                            char filename[60];
+                            vfs_file_t *file;
+
+                            strcpy(filename, "/");
+                            strcat(filename, subname);
+                            strcat(filename, ".macro");
+
+#if LITTLEFS_ENABLE
+                            sprintf(filename, "/littlefs/P%d.macro", macro_id);
+
+                            if((file = vfs_open(filename, "r")) == NULL)
+#endif
+                            if((file = stream_redirect_read(filename, onError, onFileEnd))) {
+                                if((sub = add_sub(o_label, file)) == NULL)
+                                    status = Status_FlowControlOutOfMemory;
+                            } else
+                                status = Status_FlowControlOutOfMemory; // file not found...
+                       }
+#endif
+                    } else do {
                         if(sub->o_label == o_label && sub->file == hal.stream.file)
                             break;
                     } while((sub = sub->next));

ngc_params.h

@@ -34,6 +34,8 @@
 #define NGC_MAX_PARAM_LENGTH 20
 #endif
 
+#define NGC_MAX_PARAM_ID 65535
+
 typedef uint16_t ngc_param_id_t;
 
 typedef struct {

protocol.c

@@ -240,6 +240,9 @@ bool protocol_main_loop (void)
                 if (state_get() == STATE_JOG) // Block all other states from invoking motion cancel.
                     system_set_exec_state_flag(EXEC_MOTION_CANCEL);
 
+            } else if(c == ASCII_EOF) {
+                if(grbl.on_file_end)
+                    grbl.on_file_end(hal.stream.file, gc_state.last_error);
             } else if ((c == '\n') || (c == '\r')) { // End of line reached
 
                 // Check for possible secondary end of line character, do not process as empty line
@@ -279,7 +282,8 @@ bool protocol_main_loop (void)
                 else { // Parse and execute g-code block.
 
 #endif
-                    gc_state.last_error = gc_execute_block(line);
+                    if((gc_state.last_error = gc_execute_block(line)) != Status_OK)
+                        eol = '\0';
                 }
 
                 // Add a short delay for each block processed in Check Mode to

stream_file.c

@@ -0,0 +1,191 @@
+/*
+  stream_file.c - stream redirector for file input
+
+  Part of grblHAL
+
+  Copyright (c) 2024 Terje Io
+
+  grblHAL is free software: you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation, either version 3 of the License, or
+  (at your option) any later version.
+
+  grblHAL is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with grblHAL. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "hal.h"
+#include "stream_file.h"
+
+typedef struct rd_stream {
+    vfs_file_t *file_new;
+    vfs_file_t *file;
+    stream_read_ptr read;
+    stream_type_t type;
+    status_message_ptr status_handler;
+    on_file_end_ptr eof_handler;
+    struct rd_stream *next;
+} rd_stream_t;
+
+static rd_stream_t *rd_streams = NULL;
+static status_message_ptr status_message;
+static on_file_end_ptr on_file_end;
+static on_report_handlers_init_ptr on_report_handlers_init;
+
+// File stream input function.
+// Reads character by character from a file and returns them when
+// requested by the foreground process.
+static int16_t stream_read_file (void)
+{
+    static bool eol_ok = false;
+
+    char c;
+
+    if(hal.stream.file) {
+        if(vfs_read(&c, 1, 1, hal.stream.file) == 1) {
+            if(c == ASCII_CR || c == ASCII_LF) {
+                if(eol_ok)
+                    return SERIAL_NO_DATA;
+                eol_ok = true;
+            } else
+                eol_ok = false;
+        } else if(eol_ok) {
+            eol_ok = false;
+            return ASCII_EOF;   // Return end-of-file. grbl.on_file_end() event generated in protocol.c.
+        } else {
+            eol_ok = true;
+            return ASCII_LF;    // Return a linefeed if the last character was not a linefeed.
+        }
+    } else
+        c = SERIAL_NO_DATA; // TODO: close all streams?
+
+    return (int16_t)c;
+}
+
+static status_code_t onFileEnd (vfs_file_t *file, status_code_t status)
+{
+    rd_stream_t *stream;
+
+    if((stream = rd_streams)) {
+
+        while(stream->next && (stream = stream->next));
+
+        if(stream->eof_handler)
+            stream->eof_handler(file, status);
+    }
+
+    if(on_file_end)
+        on_file_end(file, status);
+
+    return true;
+}
+
+// This code will be executed after each command is sent to the parser,
+// If an error is detected reading from file(s) will be stopped and the
+// status_code reported, if not a "ok" status reply will not be output.
+static status_code_t trap_status_messages (status_code_t status)
+{
+    gc_state.last_error = status;
+
+    if(rd_streams == NULL)
+        status = status_message(status);
+
+    else if(!(status == Status_OK || status == Status_Unhandled)) {
+
+        rd_stream_t *stream;
+
+        if((stream = rd_streams)) {
+
+            while(stream->next && (stream = stream->next));
+
+            if(stream->status_handler)
+                status = stream->status_handler(status);
+        }
+    }
+
+    return status;
+}
+
+static void onReportHandlersInit (void)
+{
+    if(on_report_handlers_init)
+        on_report_handlers_init();
+
+    status_message = grbl.report.status_message;
+    grbl.report.status_message = trap_status_messages;
+}
+
+vfs_file_t *stream_redirect_read (char *filename, status_message_ptr status_handler, on_file_end_ptr eof_handler)
+{
+    static bool error_handler_ok = false;
+
+    vfs_file_t *file;
+
+    if((file = vfs_open(filename, "r"))) {
+        rd_stream_t *rd_stream, *streams = rd_streams;
+        if((rd_stream = malloc(sizeof(rd_stream_t)))) {
+            rd_stream->file = hal.stream.file;
+            rd_stream->type = hal.stream.type;
+            rd_stream->file_new = file;
+            rd_stream->read = hal.stream.read;
+            rd_stream->eof_handler = eof_handler;
+            rd_stream->status_handler = status_handler;
+            rd_stream->next = NULL;
+            hal.stream.file = file;
+            hal.stream.type = StreamType_File;
+            hal.stream.read = stream_read_file;
+            if(streams == NULL)
+                rd_streams = rd_stream;
+            else do {
+                if(streams->next == NULL) {
+                    streams->next = rd_stream;
+                    break;
+                }
+            } while((streams == streams->next));
+        } else {
+            vfs_close(file);
+            file = NULL;
+        }
+    }
+
+    if(file && !error_handler_ok) {
+        error_handler_ok = true;
+
+        on_report_handlers_init = grbl.on_report_handlers_init;
+        grbl.on_report_handlers_init = onReportHandlersInit;
+
+        status_message = grbl.report.status_message;
+        grbl.report.status_message = trap_status_messages;
+
+        on_file_end = grbl.on_file_end;
+        grbl.on_file_end = onFileEnd;
+    }
+
+    return file;
+}
+
+void stream_redirect_close (vfs_file_t *file)
+{
+    rd_stream_t *stream = rd_streams, *prev_stream;
+
+    if(stream) do {
+        if(stream->file_new == file) {
+            vfs_close(file);
+            hal.stream.file = stream->file;
+            hal.stream.read = stream->read;
+            hal.stream.type = stream->type;
+            if(stream == rd_streams)
+                rd_streams = stream->next;
+            else
+                prev_stream->next = stream->next;
+            free(stream);
+            break;
+        }
+        prev_stream = stream;
+    } while((stream = stream->next));
+}

stream_file.h

@@ -0,0 +1,28 @@
+/*
+  stream_file.h - stream redirector for file input
+
+  Part of grblHAL
+
+  Copyright (c) 2024 Terje Io
+
+  grblHAL is free software: you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation, either version 3 of the License, or
+  (at your option) any later version.
+
+  grblHAL is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with grblHAL. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#pragma once
+
+#include "vfs.h"
+#include "core_handlers.h"
+
+void stream_redirect_close (vfs_file_t *file);
+vfs_file_t *stream_redirect_read (char *filename, status_message_ptr status_handler, on_file_end_ptr eof_handler);