Skip to content

feature/charset #8

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
mplucinski wants to merge 5 commits into
base: master
Choose a base branch
from
Open

feature/charset #8

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
493e375
Use YY_CHAR macro instead of `char` type in places where processed te…
mplucinski Jul 25, 2014
80e5cd7
Add test cases for charset support
mplucinski Jun 20, 2014
3f19e6f
Add implementation for charset support
mplucinski Jun 20, 2014
0fe5125
Extend test-charset to check C++ scanner too
mplucinski Jun 24, 2014
2385656
Make charset support working with C++ scanners too
mplucinski Jun 24, 2014
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions configure.ac
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -170,6 +170,8 @@ tests/test-ccl/Makefile
tests/test-extended/Makefile
tests/test-c++-yywrap/Makefile
tests/test-concatenated-options/Makefile
tests/test-tests_strutils/Makefile
tests/test-charset/Makefile
dnl --new-test-here-- This line is processed by tests/create-test.
)

Expand Down
36 changes: 29 additions & 7 deletions src/FlexLexer.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,11 @@
# define FLEX_STD std::
# endif

#ifndef YY_CHAR_DEFINED
#define YY_CHAR_DEFINED
typedef unsigned char YY_CHAR;
#endif

extern "C++" {

struct yy_buffer_state;
Expand All @@ -62,7 +67,7 @@ class FlexLexer {
public:
virtual ~FlexLexer() { }

const char* YYText() const { return yytext; }
const YY_CHAR* YYText() const { return yytext; }
int YYLeng() const { return yyleng; }

virtual void
Expand Down Expand Up @@ -91,8 +96,16 @@ class FlexLexer {
int debug() const { return yy_flex_debug; }
void set_debug( int flag ) { yy_flex_debug = flag; }

#ifdef YY_CHARSET
void set_charset(char *charset);
char* get_charset();
#endif

protected:
char* yytext;
YY_CHAR* yytext;
#ifdef YY_CHARSET
char *yycharset;
#endif
int yyleng;
int yylineno; // only maintained if you use %option yylineno
int yy_flex_debug; // only has effect with -d or "%option debug"
Expand Down Expand Up @@ -134,7 +147,7 @@ class yyFlexLexer : public FlexLexer {
virtual void LexerOutput( const char* buf, int size );
virtual void LexerError( const char* msg );

void yyunput( int c, char* buf_ptr );
void yyunput( int c, YY_CHAR* buf_ptr );
int yyinput();

void yy_load_buffer_state();
Expand All @@ -153,17 +166,26 @@ class yyFlexLexer : public FlexLexer {
yy_state_type yy_try_NUL_trans( yy_state_type current_state );
int yy_get_next_buffer();

#ifdef YY_CHARSET
size_t yycharset_convert(char* source, size_t source_bytes, YY_CHAR* target,
size_t target_length, size_t* converted_bytes);
virtual size_t yycharset_handler(char *charset,
char *source, size_t source_bytes,
YY_CHAR *target, size_t target_length,
size_t *converted_bytes);
#endif

FLEX_STD istream* yyin; // input source for default LexerInput
FLEX_STD ostream* yyout; // output sink for default LexerOutput

// yy_hold_char holds the character lost when yytext is formed.
char yy_hold_char;
YY_CHAR yy_hold_char;

// Number of characters read into yy_ch_buf.
int yy_n_chars;

// Points to current character in buffer.
char* yy_c_buf_p;
YY_CHAR* yy_c_buf_p;

int yy_init; // whether we need to initialize
int yy_start; // start state number
Expand All @@ -182,12 +204,12 @@ class yyFlexLexer : public FlexLexer {
// on use of certain flex features (like REJECT or yymore()).

yy_state_type yy_last_accepting_state;
char* yy_last_accepting_cpos;
YY_CHAR* yy_last_accepting_cpos;

yy_state_type* yy_state_buf;
yy_state_type* yy_state_ptr;

char* yy_full_match;
YY_CHAR* yy_full_match;
int* yy_full_state;
int yy_full_lp;

Expand Down
Loading