luke
/
ngspice
mirror of https://git.code.sf.net/p/ngspice/ngspice
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Input file encoding 

Read, then remove BOM
UTF-16: Read every second character (o.k. if only ASCII involved).
This commit is contained in:
Holger Vogt 2026-07-25 15:58:36 +02:00
parent aec5785238
commit 77259c40d9
7 changed files with 221 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
src/frontend/Makefile.am
View File

@ -119,6 +119,7 @@ libfte_la_SOURCES = \
dotcards.c \
dotcards.h \
dvec.c \
encodedet.c \
error.c \
evaluate.c \
evaluate.h \

151
src/frontend/encodedet.c Normal file
View File

@ -0,0 +1,151 @@
/* Copyright Holger Vogt, 2026
License 3-clause BSD
with help from Mistral Small 4 119B */
#include <stdio.h>
#include <stdint.h>
#include <string.h>
#include "encodedet.h"
FileEncoding detect_file_encoding(FILE *file) {
unsigned char buffer[4];
size_t bytes_read = fread(buffer, 1, 4, file);
rewind(file); // Reset file pointer
// Check for BOMs first
if (bytes_read >= 3 &&
buffer[0] == 0xEF && buffer[1] == 0xBB && buffer[2] == 0xBF) {
return ENCODING_UTF8_WITH_BOM;
}
if (bytes_read >= 2) {
if (buffer[0] == 0xFF && buffer[1] == 0xFE) {
return ENCODING_UTF16LE_WITH_BOM;
}
if (buffer[0] == 0xFE && buffer[1] == 0xFF) {
// UTF-16BE with BOM (not in your requested list)
return ENCODING_UTF16BE_WITH_BOM;
}
}
/* Check for UTF-16LE without BOM (look for null bytes)
if (bytes_read >= 2) {
int likely_utf16le = 1;
for (size_t i = 0; i < bytes_read; i++) {
if (i + 1 >= bytes_read) {
likely_utf16le = 0;
break;
}
if (buffer[i] != 0x00 && (buffer[i] < 0x20 || buffer[i] > 0x7E)) {
likely_utf16le = 0;
break;
}
}
if (likely_utf16le && bytes_read % 2 == 0) {
return ENCODING_UTF16LE_NO_BOM;
}
} */
// Check for UTF-16LE without BOM (look for null bytes)
if (bytes_read >= 4) {
if (buffer[1] == '\0' && buffer[3] == '\0') {
return ENCODING_UTF16LE_NO_BOM;
}
}
else if (bytes_read >= 2) {
if (buffer[1] == '\0') {
return ENCODING_UTF16LE_NO_BOM;
}
}
// Check for UTF-8 without BOM
int is_utf8 = 1;
for (size_t i = 0; i < bytes_read; i++) {
if (buffer[i] == 0x00) { // Null byte found (likely binary)
is_utf8 = 0;
break;
}
if ((buffer[i] & 0x80) == 0x00) { // 0xxxxxxx - ASCII
continue;
} else if ((buffer[i] & 0xE0) == 0xC0) { // 110xxxxx
if (i + 1 >= bytes_read || (buffer[i+1] & 0xC0) != 0x80) {
is_utf8 = 0;
break;
}
i++; // Skip next byte
} else if ((buffer[i] & 0xF0) == 0xE0) { // 1110xxxx
if (i + 2 >= bytes_read || (buffer[i+1] & 0xC0) != 0x80 ||
(buffer[i+2] & 0xC0) != 0x80) {
is_utf8 = 0;
break;
}
i += 2; // Skip next 2 bytes
} else if ((buffer[i] & 0xF8) == 0xF0) { // 11110xxx
if (i + 3 >= bytes_read || (buffer[i+1] & 0xC0) != 0x80 ||
(buffer[i+2] & 0xC0) != 0x80 || (buffer[i+3] & 0xC0) != 0x80) {
is_utf8 = 0;
break;
}
i += 3; // Skip next 3 bytes
} else {
is_utf8 = 0;
break;
}
}
if (is_utf8) {
return ENCODING_UTF8_NO_BOM;
}
// Check for ASCII
int is_ascii = 1;
for (size_t i = 0; i < bytes_read; i++) {
if (buffer[i] > 0x7F) {
is_ascii = 0;
break;
}
}
if (is_ascii) {
return ENCODING_ASCII;
}
// Check for binary (contains null bytes or non-text characters)
for (size_t i = 0; i < bytes_read; i++) {
if (buffer[i] == 0x00 || buffer[i] < 0x09 || buffer[i] == 0x0B ||
buffer[i] == 0x0C || (buffer[i] > 0x0E && buffer[i] < 0x20) ||
buffer[i] > 0x7E) {
return ENCODING_BINARY;
}
}
return ENCODING_UNKNOWN;
}
const char* encoding_to_string(FileEncoding encoding) {
switch (encoding) {
case ENCODING_ASCII: return "ASCII";
case ENCODING_UTF8_NO_BOM: return "UTF-8 without BOM";
case ENCODING_UTF8_WITH_BOM: return "UTF-8 with BOM";
case ENCODING_UTF16LE_WITH_BOM: return "UTF-16LE with BOM";
case ENCODING_UTF16LE_NO_BOM: return "UTF-16LE without BOM";
case ENCODING_BINARY: return "Binary";
default: return "Unknown";
}
}
/*
int main() {
FILE *file = fopen("test.txt", "rb");
if (!file) {
perror("Failed to open file");
return 1;
}
FileEncoding encoding = detect_file_encoding(file);
printf("File encoding: %s\n", encoding_to_string(encoding));
fclose(file);
return 0;
}
*/

15
src/frontend/encodedet.h Normal file
View File

@ -0,0 +1,15 @@
typedef enum {
ENCODING_ASCII,
ENCODING_UTF8_NO_BOM,
ENCODING_UTF8_WITH_BOM,
ENCODING_UTF16LE_WITH_BOM,
ENCODING_UTF16LE_NO_BOM,
ENCODING_UTF16BE_WITH_BOM,
ENCODING_BINARY,
ENCODING_UNKNOWN
} FileEncoding;
FileEncoding detect_file_encoding(FILE* file);
const char* encoding_to_string(FileEncoding encoding);

69
src/frontend/inpcom.c
View File

@ -30,6 +30,7 @@ Author: 1985 Wayne A. Christopher
#include "numparam/general.h"
#include "com_set.h"
#include "encodedet.h"
#include <limits.h>
#include <stdlib.h>
@ -122,7 +123,7 @@ wordlist* sourceinfo = NULL;
static bool has_if = FALSE; /* if we have an .if ... .endif pair */
static char *readline(FILE *fd);
static char *readline(FILE *fd, FileEncoding encoding);
int get_number_terminals(char *c);
static void inp_stripcomments_deck(struct card *deck, bool cs);
static void inp_stripcomments_line(char *s, bool cs, bool inc);
@ -209,7 +210,7 @@ struct inp_read_t {
};
static struct inp_read_t inp_read( FILE *fp, int call_depth, const char *dir_name,
const char* file_name, bool comfile, bool intfile);
const char* file_name, bool comfile, bool intfile, FileEncoding enc);
#ifdef XSPICE
@ -595,7 +596,7 @@ static struct library *read_a_lib(const char *y, const char *dir_name)
lib->habitat = ngdirname(yy);
lib->deck =
inp_read(newfp, 1 /*dummy*/, lib->habitat, lib->realpath, FALSE, FALSE).cc;
inp_read(newfp, 1 /*dummy*/, lib->habitat, lib->realpath, FALSE, FALSE, ENCODING_UNKNOWN).cc;
struct card* tmpdeck;
int cnumber = 1;
@ -1059,7 +1060,7 @@ struct card *inp_readall(FILE *fp, const char *dir_name, const char* file_name,
This is the next major step:
Reading the netlist line by line, handle .include and .lib,
line continuation and upper/lower casing */
rv = inp_read(fp, 0, dir_name, file_name, comfile, intfile);
rv = inp_read(fp, 0, dir_name, file_name, comfile, intfile, ENCODING_UNKNOWN);
cc = rv.cc;
/* skip all pre-processing for expanded input files created by 'listing r',
@ -1318,7 +1319,7 @@ static char *skip_token(char *s)
static struct inp_read_t inp_read(FILE* fp, int call_depth, const char* dir_name,
const char* file_name, bool comfile, bool intfile)
const char* file_name, bool comfile, bool intfile, FileEncoding encoding)
/* fp: in, pointer to file to be read,
call_depth: in, nested call to fcn
dir_name: in, name of directory of file to be read
@ -1379,14 +1380,14 @@ static struct inp_read_t inp_read(FILE* fp, int call_depth, const char* dir_name
buffer = copy(big_buff);
}
else {
buffer = readline(fp);
buffer = readline(fp, encoding);
if (!buffer)
break;
}
#else
buffer = readline(fp);
buffer = readline(fp, encoding);
if (!buffer) {
break;
}
@ -1403,23 +1404,23 @@ static struct inp_read_t inp_read(FILE* fp, int call_depth, const char* dir_name
continue;
}
/* Strip a UTF-8 byte order mark at the start of a line -- i.e. at
the start of a file written by a Windows editor. A BOM is never
legal SPICE syntax; left in place it glues to the first token
(fatal ".subckt/.ends mismatch" class errors in included libs). */
if (!intfile && strncmp(buffer, "\xEF\xBB\xBF", 3) == 0)
memmove(buffer, buffer + 3, strlen(buffer + 3) + 1);
/* OK -- now we have loaded the next line into 'buffer'. Process it.
*/
/* after reading the first line */
if (first) {
/* Files starting *ng_script are user supplied command files. */
if (ciprefix("*ng_script", buffer))
comfile = TRUE;
/* Strip a UTF-8 or UTF-16 byte order mark at the start
of a file, e.g. written by a Windows editor. A BOM is never
legal SPICE syntax; left in place it glues to the first token
(fatal ".subckt/.ends mismatch" class errors in included libs). */
if (!intfile && (encoding == ENCODING_UTF8_WITH_BOM))
memmove(buffer, buffer + 3, strlen(buffer + 3) + 1);
else if (!intfile && (encoding == ENCODING_UTF16LE_WITH_BOM))
memmove(buffer, buffer + 2, strlen(buffer + 2) + 1);
first = FALSE;
}
/* OK -- now we have loaded the next line into 'buffer'. Process it.
*
/* If input line is blank, ignore it & continue looping. */
if ((strcmp(buffer, "\n") == 0) || (strcmp(buffer, "\r\n") == 0))
if (call_depth != 0 || (call_depth == 0 && cc != NULL)) {
@ -1578,6 +1579,26 @@ static struct inp_read_t inp_read(FILE* fp, int call_depth, const char* dir_name
}
}
/* test input file for its encoding */
encoding = ENCODING_UNKNOWN;
FILE* enfile = fopen(y_resolved, "rb");
if (enfile) {
encoding = detect_file_encoding(enfile);
switch (encoding) {
case ENCODING_UTF8_WITH_BOM:
case ENCODING_UTF16LE_WITH_BOM:
case ENCODING_UTF16LE_NO_BOM:
fprintf(stdout, "Note: .include file '%s' encoding: %s\n", y, encoding_to_string(encoding));
break;
case ENCODING_BINARY:
fprintf(stderr, "Warning: Binary .inlude file '%s' is not supported, skipped!\n", y);
fclose(enfile);
continue;
default: /* do nothing */;
}
fclose(enfile);
}
newfp = fopen(y_resolved, "r");
if (!newfp) {
@ -1597,7 +1618,7 @@ static struct inp_read_t inp_read(FILE* fp, int call_depth, const char* dir_name
y_dir_name = ngdirname(y_resolved);
newcard = inp_read(
newfp, call_depth + 1, y_dir_name, y_resolved, FALSE, FALSE)
newfp, call_depth + 1, y_dir_name, y_resolved, FALSE, FALSE, encoding)
.cc; /* read stuff in include file into
netlist */
@ -2301,7 +2322,7 @@ static char *inp_pathresolve_at(const char *name, const char *dir)
#define STRGROW 256
static char *readline(FILE *fd)
static char *readline(FILE *fd, FileEncoding encoding)
{
int c;
int memlen;
@ -2320,6 +2341,11 @@ static char *readline(FILE *fd)
if (c == '\r')
continue;
if (encoding == ENCODING_UTF16LE_WITH_BOM || encoding == ENCODING_UTF16LE_NO_BOM)
if (c == 0)
continue;
strptr[strlen++] = (char) c;
if (strlen >= memlen) {
@ -9760,7 +9786,8 @@ utf8_syntax_check(struct card *deck)
if (s) {
fprintf(stderr, "Error: UTF-8 syntax error in input deck,\n line %d at token/word %s\n", card->linenum_orig, s);
fprintf(stderr, " input file %s\n", card->linesource);
controlled_exit(1);
if (ft_stricterror)
controlled_exit(1);
}
}
}

2
visualc/sharedspice.vcxproj
View File

@ -647,6 +647,7 @@ lib /machine:x64 /def:..\..\fftw-3.3-dll64\libfftw3-3.def /out:$(IntDir)libfftw3
<ClInclude Include="..\src\frontend\dimens.h" />
<ClInclude Include="..\src\frontend\display.h" />
<ClInclude Include="..\src\frontend\dotcards.h" />
<ClInclude Include="..\src\frontend\encodedet.h" />
<ClInclude Include="..\src\frontend\error.h" />
<ClInclude Include="..\src\frontend\evaluate.h" />
<ClInclude Include="..\src\frontend\fourier.h" />
@ -1261,6 +1262,7 @@ lib /machine:x64 /def:..\..\fftw-3.3-dll64\libfftw3-3.def /out:$(IntDir)libfftw3
<ClCompile Include="..\src\frontend\display.c" />
<ClCompile Include="..\src\frontend\dotcards.c" />
<ClCompile Include="..\src\frontend\dvec.c" />
<ClCompile Include="..\src\frontend\encodedet.c" />
<ClCompile Include="..\src\frontend\error.c" />
<ClCompile Include="..\src\frontend\evaluate.c" />
<ClCompile Include="..\src\frontend\fourier.c" />

2
visualc/vngspice-fftw.vcxproj
View File

@ -893,6 +893,7 @@ lib /machine:x64 /def:..\..\fftw-3.3-dll64\libfftw3-3.def /out:$(IntDir)libfftw3
<ClInclude Include="..\src\frontend\dimens.h" />
<ClInclude Include="..\src\frontend\display.h" />
<ClInclude Include="..\src\frontend\dotcards.h" />
<ClInclude Include="..\src\frontend\encodedet.h" />
<ClInclude Include="..\src\frontend\error.h" />
<ClInclude Include="..\src\frontend\evaluate.h" />
<ClInclude Include="..\src\frontend\fourier.h" />
@ -1509,6 +1510,7 @@ lib /machine:x64 /def:..\..\fftw-3.3-dll64\libfftw3-3.def /out:$(IntDir)libfftw3
<ClCompile Include="..\src\frontend\display.c" />
<ClCompile Include="..\src\frontend\dotcards.c" />
<ClCompile Include="..\src\frontend\dvec.c" />
<ClCompile Include="..\src\frontend\encodedet.c" />
<ClCompile Include="..\src\frontend\error.c" />
<ClCompile Include="..\src\frontend\evaluate.c" />
<ClCompile Include="..\src\frontend\fourier.c" />

2
visualc/vngspice.vcxproj
View File

@ -908,6 +908,7 @@
<ClInclude Include="..\src\frontend\dimens.h" />
<ClInclude Include="..\src\frontend\display.h" />
<ClInclude Include="..\src\frontend\dotcards.h" />
<ClInclude Include="..\src\frontend\encodedet.h" />
<ClInclude Include="..\src\frontend\error.h" />
<ClInclude Include="..\src\frontend\evaluate.h" />
<ClInclude Include="..\src\frontend\fourier.h" />
@ -1524,6 +1525,7 @@
<ClCompile Include="..\src\frontend\display.c" />
<ClCompile Include="..\src\frontend\dotcards.c" />
<ClCompile Include="..\src\frontend\dvec.c" />
<ClCompile Include="..\src\frontend\encodedet.c" />
<ClCompile Include="..\src\frontend\error.c" />
<ClCompile Include="..\src\frontend\evaluate.c" />
<ClCompile Include="..\src\frontend\fourier.c" />