diff --git a/src/frontend/Makefile.am b/src/frontend/Makefile.am index c6dbcac37..d6bb1899f 100644 --- a/src/frontend/Makefile.am +++ b/src/frontend/Makefile.am @@ -119,6 +119,7 @@ libfte_la_SOURCES = \ dotcards.c \ dotcards.h \ dvec.c \ + encodedet.c \ error.c \ evaluate.c \ evaluate.h \ diff --git a/src/frontend/encodedet.c b/src/frontend/encodedet.c new file mode 100644 index 000000000..24ff88647 --- /dev/null +++ b/src/frontend/encodedet.c @@ -0,0 +1,151 @@ +/* Copyright Holger Vogt, 2026 + License 3-clause BSD + with help from Mistral Small 4 119B */ + +#include +#include +#include + +#include "encodedet.h" + +FileEncoding detect_file_encoding(FILE *file) { + unsigned char buffer[4]; + size_t bytes_read = fread(buffer, 1, 4, file); + rewind(file); // Reset file pointer + + // Check for BOMs first + if (bytes_read >= 3 && + buffer[0] == 0xEF && buffer[1] == 0xBB && buffer[2] == 0xBF) { + return ENCODING_UTF8_WITH_BOM; + } + + if (bytes_read >= 2) { + if (buffer[0] == 0xFF && buffer[1] == 0xFE) { + return ENCODING_UTF16LE_WITH_BOM; + } + if (buffer[0] == 0xFE && buffer[1] == 0xFF) { + // UTF-16BE with BOM (not in your requested list) + return ENCODING_UTF16BE_WITH_BOM; + } + } + + /* Check for UTF-16LE without BOM (look for null bytes) + if (bytes_read >= 2) { + int likely_utf16le = 1; + for (size_t i = 0; i < bytes_read; i++) { + if (i + 1 >= bytes_read) { + likely_utf16le = 0; + break; + } + if (buffer[i] != 0x00 && (buffer[i] < 0x20 || buffer[i] > 0x7E)) { + likely_utf16le = 0; + break; + } + } + if (likely_utf16le && bytes_read % 2 == 0) { + return ENCODING_UTF16LE_NO_BOM; + } + } */ + + // Check for UTF-16LE without BOM (look for null bytes) + if (bytes_read >= 4) { + if (buffer[1] == '\0' && buffer[3] == '\0') { + return ENCODING_UTF16LE_NO_BOM; + } + } + else if (bytes_read >= 2) { + if (buffer[1] == '\0') { + return ENCODING_UTF16LE_NO_BOM; + } + } + + // Check for UTF-8 without BOM + int is_utf8 = 1; + for (size_t i = 0; i < bytes_read; i++) { + if (buffer[i] == 0x00) { // Null byte found (likely binary) + is_utf8 = 0; + break; + } + if ((buffer[i] & 0x80) == 0x00) { // 0xxxxxxx - ASCII + continue; + } else if ((buffer[i] & 0xE0) == 0xC0) { // 110xxxxx + if (i + 1 >= bytes_read || (buffer[i+1] & 0xC0) != 0x80) { + is_utf8 = 0; + break; + } + i++; // Skip next byte + } else if ((buffer[i] & 0xF0) == 0xE0) { // 1110xxxx + if (i + 2 >= bytes_read || (buffer[i+1] & 0xC0) != 0x80 || + (buffer[i+2] & 0xC0) != 0x80) { + is_utf8 = 0; + break; + } + i += 2; // Skip next 2 bytes + } else if ((buffer[i] & 0xF8) == 0xF0) { // 11110xxx + if (i + 3 >= bytes_read || (buffer[i+1] & 0xC0) != 0x80 || + (buffer[i+2] & 0xC0) != 0x80 || (buffer[i+3] & 0xC0) != 0x80) { + is_utf8 = 0; + break; + } + i += 3; // Skip next 3 bytes + } else { + is_utf8 = 0; + break; + } + } + + if (is_utf8) { + return ENCODING_UTF8_NO_BOM; + } + + // Check for ASCII + int is_ascii = 1; + for (size_t i = 0; i < bytes_read; i++) { + if (buffer[i] > 0x7F) { + is_ascii = 0; + break; + } + } + + if (is_ascii) { + return ENCODING_ASCII; + } + + // Check for binary (contains null bytes or non-text characters) + for (size_t i = 0; i < bytes_read; i++) { + if (buffer[i] == 0x00 || buffer[i] < 0x09 || buffer[i] == 0x0B || + buffer[i] == 0x0C || (buffer[i] > 0x0E && buffer[i] < 0x20) || + buffer[i] > 0x7E) { + return ENCODING_BINARY; + } + } + + return ENCODING_UNKNOWN; +} + +const char* encoding_to_string(FileEncoding encoding) { + switch (encoding) { + case ENCODING_ASCII: return "ASCII"; + case ENCODING_UTF8_NO_BOM: return "UTF-8 without BOM"; + case ENCODING_UTF8_WITH_BOM: return "UTF-8 with BOM"; + case ENCODING_UTF16LE_WITH_BOM: return "UTF-16LE with BOM"; + case ENCODING_UTF16LE_NO_BOM: return "UTF-16LE without BOM"; + case ENCODING_BINARY: return "Binary"; + default: return "Unknown"; + } +} +/* +int main() { + FILE *file = fopen("test.txt", "rb"); + if (!file) { + perror("Failed to open file"); + return 1; + } + + FileEncoding encoding = detect_file_encoding(file); + printf("File encoding: %s\n", encoding_to_string(encoding)); + + fclose(file); + return 0; +} +*/ diff --git a/src/frontend/encodedet.h b/src/frontend/encodedet.h new file mode 100644 index 000000000..7f6182dcf --- /dev/null +++ b/src/frontend/encodedet.h @@ -0,0 +1,15 @@ + +typedef enum { + ENCODING_ASCII, + ENCODING_UTF8_NO_BOM, + ENCODING_UTF8_WITH_BOM, + ENCODING_UTF16LE_WITH_BOM, + ENCODING_UTF16LE_NO_BOM, + ENCODING_UTF16BE_WITH_BOM, + ENCODING_BINARY, + ENCODING_UNKNOWN +} FileEncoding; + +FileEncoding detect_file_encoding(FILE* file); +const char* encoding_to_string(FileEncoding encoding); + diff --git a/src/frontend/inpcom.c b/src/frontend/inpcom.c index d42ded786..854c1b287 100644 --- a/src/frontend/inpcom.c +++ b/src/frontend/inpcom.c @@ -30,6 +30,7 @@ Author: 1985 Wayne A. Christopher #include "numparam/general.h" #include "com_set.h" +#include "encodedet.h" #include #include @@ -122,7 +123,7 @@ wordlist* sourceinfo = NULL; static bool has_if = FALSE; /* if we have an .if ... .endif pair */ -static char *readline(FILE *fd); +static char *readline(FILE *fd, FileEncoding encoding); int get_number_terminals(char *c); static void inp_stripcomments_deck(struct card *deck, bool cs); static void inp_stripcomments_line(char *s, bool cs, bool inc); @@ -209,7 +210,7 @@ struct inp_read_t { }; static struct inp_read_t inp_read( FILE *fp, int call_depth, const char *dir_name, - const char* file_name, bool comfile, bool intfile); + const char* file_name, bool comfile, bool intfile, FileEncoding enc); #ifdef XSPICE @@ -595,7 +596,7 @@ static struct library *read_a_lib(const char *y, const char *dir_name) lib->habitat = ngdirname(yy); lib->deck = - inp_read(newfp, 1 /*dummy*/, lib->habitat, lib->realpath, FALSE, FALSE).cc; + inp_read(newfp, 1 /*dummy*/, lib->habitat, lib->realpath, FALSE, FALSE, ENCODING_UNKNOWN).cc; struct card* tmpdeck; int cnumber = 1; @@ -1059,7 +1060,7 @@ struct card *inp_readall(FILE *fp, const char *dir_name, const char* file_name, This is the next major step: Reading the netlist line by line, handle .include and .lib, line continuation and upper/lower casing */ - rv = inp_read(fp, 0, dir_name, file_name, comfile, intfile); + rv = inp_read(fp, 0, dir_name, file_name, comfile, intfile, ENCODING_UNKNOWN); cc = rv.cc; /* skip all pre-processing for expanded input files created by 'listing r', @@ -1318,7 +1319,7 @@ static char *skip_token(char *s) static struct inp_read_t inp_read(FILE* fp, int call_depth, const char* dir_name, - const char* file_name, bool comfile, bool intfile) + const char* file_name, bool comfile, bool intfile, FileEncoding encoding) /* fp: in, pointer to file to be read, call_depth: in, nested call to fcn dir_name: in, name of directory of file to be read @@ -1379,14 +1380,14 @@ static struct inp_read_t inp_read(FILE* fp, int call_depth, const char* dir_name buffer = copy(big_buff); } else { - buffer = readline(fp); + buffer = readline(fp, encoding); if (!buffer) break; } #else - buffer = readline(fp); + buffer = readline(fp, encoding); if (!buffer) { break; } @@ -1403,23 +1404,23 @@ static struct inp_read_t inp_read(FILE* fp, int call_depth, const char* dir_name continue; } - /* Strip a UTF-8 byte order mark at the start of a line -- i.e. at - the start of a file written by a Windows editor. A BOM is never - legal SPICE syntax; left in place it glues to the first token - (fatal ".subckt/.ends mismatch" class errors in included libs). */ - if (!intfile && strncmp(buffer, "\xEF\xBB\xBF", 3) == 0) - memmove(buffer, buffer + 3, strlen(buffer + 3) + 1); - - /* OK -- now we have loaded the next line into 'buffer'. Process it. - */ + /* after reading the first line */ if (first) { /* Files starting *ng_script are user supplied command files. */ - if (ciprefix("*ng_script", buffer)) comfile = TRUE; + /* Strip a UTF-8 or UTF-16 byte order mark at the start + of a file, e.g. written by a Windows editor. A BOM is never + legal SPICE syntax; left in place it glues to the first token + (fatal ".subckt/.ends mismatch" class errors in included libs). */ + if (!intfile && (encoding == ENCODING_UTF8_WITH_BOM)) + memmove(buffer, buffer + 3, strlen(buffer + 3) + 1); + else if (!intfile && (encoding == ENCODING_UTF16LE_WITH_BOM)) + memmove(buffer, buffer + 2, strlen(buffer + 2) + 1); first = FALSE; } - + /* OK -- now we have loaded the next line into 'buffer'. Process it. + * /* If input line is blank, ignore it & continue looping. */ if ((strcmp(buffer, "\n") == 0) || (strcmp(buffer, "\r\n") == 0)) if (call_depth != 0 || (call_depth == 0 && cc != NULL)) { @@ -1578,6 +1579,26 @@ static struct inp_read_t inp_read(FILE* fp, int call_depth, const char* dir_name } } + /* test input file for its encoding */ + encoding = ENCODING_UNKNOWN; + FILE* enfile = fopen(y_resolved, "rb"); + if (enfile) { + encoding = detect_file_encoding(enfile); + switch (encoding) { + case ENCODING_UTF8_WITH_BOM: + case ENCODING_UTF16LE_WITH_BOM: + case ENCODING_UTF16LE_NO_BOM: + fprintf(stdout, "Note: .include file '%s' encoding: %s\n", y, encoding_to_string(encoding)); + break; + case ENCODING_BINARY: + fprintf(stderr, "Warning: Binary .inlude file '%s' is not supported, skipped!\n", y); + fclose(enfile); + continue; + default: /* do nothing */; + } + fclose(enfile); + } + newfp = fopen(y_resolved, "r"); if (!newfp) { @@ -1597,7 +1618,7 @@ static struct inp_read_t inp_read(FILE* fp, int call_depth, const char* dir_name y_dir_name = ngdirname(y_resolved); newcard = inp_read( - newfp, call_depth + 1, y_dir_name, y_resolved, FALSE, FALSE) + newfp, call_depth + 1, y_dir_name, y_resolved, FALSE, FALSE, encoding) .cc; /* read stuff in include file into netlist */ @@ -2301,7 +2322,7 @@ static char *inp_pathresolve_at(const char *name, const char *dir) #define STRGROW 256 -static char *readline(FILE *fd) +static char *readline(FILE *fd, FileEncoding encoding) { int c; int memlen; @@ -2320,6 +2341,11 @@ static char *readline(FILE *fd) if (c == '\r') continue; + + if (encoding == ENCODING_UTF16LE_WITH_BOM || encoding == ENCODING_UTF16LE_NO_BOM) + if (c == 0) + continue; + strptr[strlen++] = (char) c; if (strlen >= memlen) { @@ -9760,7 +9786,8 @@ utf8_syntax_check(struct card *deck) if (s) { fprintf(stderr, "Error: UTF-8 syntax error in input deck,\n line %d at token/word %s\n", card->linenum_orig, s); fprintf(stderr, " input file %s\n", card->linesource); - controlled_exit(1); + if (ft_stricterror) + controlled_exit(1); } } } diff --git a/visualc/sharedspice.vcxproj b/visualc/sharedspice.vcxproj index 894540844..f7605ed87 100644 --- a/visualc/sharedspice.vcxproj +++ b/visualc/sharedspice.vcxproj @@ -647,6 +647,7 @@ lib /machine:x64 /def:..\..\fftw-3.3-dll64\libfftw3-3.def /out:$(IntDir)libfftw3 + @@ -1261,6 +1262,7 @@ lib /machine:x64 /def:..\..\fftw-3.3-dll64\libfftw3-3.def /out:$(IntDir)libfftw3 + diff --git a/visualc/vngspice-fftw.vcxproj b/visualc/vngspice-fftw.vcxproj index 33827de87..4098767e1 100644 --- a/visualc/vngspice-fftw.vcxproj +++ b/visualc/vngspice-fftw.vcxproj @@ -893,6 +893,7 @@ lib /machine:x64 /def:..\..\fftw-3.3-dll64\libfftw3-3.def /out:$(IntDir)libfftw3 + @@ -1509,6 +1510,7 @@ lib /machine:x64 /def:..\..\fftw-3.3-dll64\libfftw3-3.def /out:$(IntDir)libfftw3 + diff --git a/visualc/vngspice.vcxproj b/visualc/vngspice.vcxproj index 5698d934b..09bd574e7 100644 --- a/visualc/vngspice.vcxproj +++ b/visualc/vngspice.vcxproj @@ -908,6 +908,7 @@ + @@ -1524,6 +1525,7 @@ +