Parser: support writing C++ code in the grammar

24 files changed, 77 insertions, 43 deletions

diff --git a/.gitignore b/.gitignore
index 4c665a8..ea74327 100644
--- a/.gitignore
+++ b/.gitignore
@@ -6,6 +6,8 @@
 *.lo
 *.log
 *.class
+*.java
+*.tokens
 .deps
 .libs
 .dirstamp
@@ -26,9 +28,8 @@ instruo
 long.sh
 massif.out
 libbash.g
-libbash.tokens
+java_libbash.g
 libbash.tokens.md5
-bashast.tokens
 bashast/.DS_Store
 bashast/*.ebuild
 bashast/c_runtime
@@ -42,7 +43,6 @@ libbashLexer.*
 libbashParser.*
 libbashWalker.h
 libbashWalker.cpp
-libbashWalker.tokens
 *.pdf
 *.aux
 config.*
diff --git a/Makefile.am b/Makefile.am
index a706c43..49b6392 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -72,7 +72,7 @@ EBUILD_RESULT = scripts/sunpinyin-2.0.3-r1.ebuild.result
 
 TESTS = $(GUNIT_TESTS) $(BASH_TESTS) $(EBUILD_TESTS)
 # these get cleaned so only add autogenerated stuff or modify CLEANFILES
-check_JAVA = bashast/bashastLexer.java bashast/bashastParser.java
+check_JAVA = bashast/java_libbashLexer.java bashast/java_libbashParser.java
 AM_JAVACFLAGS = -source 1.4 -classpath @antlr_cp@
 
 AM_CXXFLAGS = -std=c++0x
@@ -92,7 +92,10 @@ AM_CPPFLAGS += -D_GLIBCXX_DEBUG
 endif
 
 if HAVE_GTEST
-TESTS += cppunittests test/ast_printer_test.sh test/verify_bashs_test.sh test/verify_error_output_test.sh
+TESTS += cppunittests \
+		 test/ast_printer_test.sh \
+		 test/verify_bashs_test.sh \
+		 test/verify_error_output_test.sh
 check_PROGRAMS = cppunittests
 
 cppunittests_SOURCES =  test/run_tests.cpp \
@@ -148,24 +151,31 @@ ast_printer_LDFLAGS = -static
 # linking will fail
 ast_printer_CPPFLAGS = $(AM_CPPFLAGS) -U_GLIBCXX_DEBUG $(BOOST_CPPFLAGS)
 
-GENERATED_PARSER_C = libbashLexer.c libbashParser.c
-GENERATED_PARSER_H = libbashLexer.h libbashParser.h
+GENERATED_LEXER_CPP = libbashLexer.cpp
+GENERATED_LEXER_H = libbashLexer.h
+GENERATED_PARSER_CPP = libbashParser.cpp
+GENERATED_PARSER_H = libbashParser.h
 GENERATED_WALKER_CPP = libbashWalker.cpp
 GENERATED_WALKER_H = libbashWalker.h
-BUILT_SOURCES = $(GENERATED_PARSER_C) \
+BUILT_SOURCES = $(GENERATED_PARSER_CPP) \
 				$(GENERATED_PARSER_H) \
+				$(GENERATED_LEXER_CPP) \
+				$(GENERATED_LEXER_H) \
 				$(GENERATED_WALKER_CPP) \
 				$(GENERATED_WALKER_H) \
 				$(check_JAVA)
-CLEANFILES = $(GENERATED_PARSER_C) \
+CLEANFILES = $(GENERATED_PARSER_CPP) \
 			 $(GENERATED_PARSER_H) \
+			 $(GENERATED_LEXER_CPP) \
+			 $(GENERATED_LEXER_H) \
 			 $(GENERATED_WALKER_CPP) \
 			 $(GENERATED_WALKER_H) \
 			 $(check_JAVA) \
 			 libbash.g \
+			 java_libbash.g \
 			 libbash.tokens \
 			 libbash.tokens.md5 \
-			 bashast/bashast.tokens \
+			 bashast/java_libbash.tokens \
 			 libbashWalker.tokens \
 			 javagrammar.run \
 			 cgrammar.run \
@@ -203,8 +213,6 @@ libcppbash_la_SOURCES = src/common.h \
 						src/builtins/unset_builtin.h \
 						src/builtins/unset_builtin.cpp \
 						src/builtins/builtin_exceptions.h \
-						$(GENERATED_PARSER_C) \
-						$(GENERATED_PARSER_H) \
 						src/core/interpreter_exception.h \
 						src/core/unset_exception.h \
 						src/core/interpreter.cpp \
@@ -220,7 +228,16 @@ libcppbash_la_SOURCES = src/common.h \
 # separate library because we need per file CXXFLAGS
 # as antlr generated code does not pass our strict developer
 # warning settings
-noinst_LTLIBRARIES = libwalker.la
+noinst_LTLIBRARIES = libparser.la libwalker.la
+libparser_la_SOURCES = $(GENERATED_PARSER_CPP) \
+					   $(GENERATED_PARSER_H) \
+					   $(GENERATED_LEXER_CPP) \
+					   $(GENERATED_LEXER_H)
+
+libparser_la_CXXFLAGS = -std=c++0x \
+						-fvisibility=hidden \
+						-fvisibility-inlines-hidden
+
 libwalker_la_SOURCES = $(GENERATED_WALKER_CPP) \
 					   $(GENERATED_WALKER_H)
 
@@ -237,7 +254,7 @@ if DEVELOPER_MODE
 libcppbash_la_CXXFLAGS += -Wconversion -Wsign-conversion
 endif
 libcppbash_la_CFLAGS = $(AM_CFLAGS) -fvisibility=hidden
-libcppbash_la_LIBADD = libwalker.la
+libcppbash_la_LIBADD = libparser.la libwalker.la
 
 EXTRA_DIST = bashast/bashast.g \
 			 bashast/libbashWalker.g \
@@ -274,12 +291,14 @@ ANTLR = @JAVA@ -classpath @antlr_cp@ org.antlr.Tool
 cgrammar.run: libbash.g
 	$(AM_V_at)touch $@.tmp
 	$(AM_V_GEN)$(ANTLR) -fo . $<
+	$(AM_V_at)mv libbashParser.{c,cpp}
+	$(AM_V_at)mv libbashLexer.{c,cpp}
 	$(AM_V_at)mv $@.tmp $@
 
 libbash.g: bashast/bashast.g
-	$(AM_V_at)sed -e 's/ASTLabelType	= CommonTree;/ASTLabelType	= pANTLR3_BASE_TREE;/g' \
-				  -e 's/language	= Java;/language	= C;/g' \
-				  -e 's/grammar bashast;/grammar libbash;/g' $< > $@
+	$(AM_V_at)$(CC) -E -x c++ -D OUTPUT_C $< > $@
+	$(AM_V_at)sed -i '/^#.*/d' $@
+	$(AM_V_at)sed -i 's/C_INCLUDE \(.*\)/\1/' $@
 
 # http://www.kolpackov.net/pipermail/notes/2004-September.txt
 libbash.tokens.md5: libbash.tokens
@@ -294,14 +313,20 @@ walker.run: bashast/libbashWalker.g libbash.tokens.md5
 	$(AM_V_at)mv libbashWalker.{c,cpp}
 	$(AM_V_at)mv $@.tmp $@
 
-$(GENERATED_PARSER_C): cgrammar.run
+$(GENERATED_PARSER_CPP): cgrammar.run
 $(GENERATED_PARSER_H): cgrammar.run
+$(GENERATED_LEXER_CPP): cgrammar.run
+$(GENERATED_LEXER_H): cgrammar.run
 libbash.tokens: cgrammar.run
 
 $(GENERATED_WALKER_CPP): walker.run
 $(GENERATED_WALKER_H): walker.run
 
-javagrammar.run: bashast/bashast.g
+java_libbash.g: bashast/bashast.g
+	$(AM_V_at)$(CC) -E -x c++ -D OUTPUT_JAVA $< > $@
+	$(AM_V_at)sed -i '/^#.*/d' $@
+
+javagrammar.run: java_libbash.g
 	$(AM_V_at)touch $@.tmp
 	$(AM_V_GEN)$(ANTLR) -fo bashast $<
 	$(AM_V_at)mv $@.tmp $@
diff --git a/bashast/bashast.g b/bashast/bashast.g
index 0f297f6..995e2af 100644
--- a/bashast/bashast.g
+++ b/bashast/bashast.g
@@ -16,14 +16,23 @@
    You should have received a copy of the GNU General Public License
    along with libbash.  If not, see <http://www.gnu.org/licenses/>.
 */
-grammar bashast;
+#ifdef OUTPUT_C
+grammar libbash;
+#else
+grammar java_libbash;
+#endif
 options
 {
 	backtrack	= true;
 	output	= AST;
+	memoize		= true;
+#ifdef OUTPUT_C
+	language  = C;
+	ASTLabelType  = pANTLR3_BASE_TREE;
+#else
 	language	= Java;
 	ASTLabelType	= CommonTree;
-	memoize		= true;
+#endif
 }
 tokens{
 	ARG;
diff --git a/bashast/gunit/arith_main.gunit b/bashast/gunit/arith_main.gunit
index 2a440fc..38c8e2e 100644
--- a/bashast/gunit/arith_main.gunit
+++ b/bashast/gunit/arith_main.gunit
@@ -16,7 +16,7 @@
    You should have received a copy of the GNU General Public License
    along with libbash.  If not, see <http://www.gnu.org/licenses/>.
 */
-gunit bashast;
+gunit java_libbash;
 
 //for this set of unittests, we'll start from the
 //top of the order of ops
diff --git a/bashast/gunit/array.gunit b/bashast/gunit/array.gunit
index 3a15c20..ce0b344 100644
--- a/bashast/gunit/array.gunit
+++ b/bashast/gunit/array.gunit
@@ -16,7 +16,7 @@
    You should have received a copy of the GNU General Public License
    along with libbash.  If not, see <http://www.gnu.org/licenses/>.
 */
-gunit bashast;
+gunit java_libbash;
 
 var_def:
 "asdf=(a b c d)"->(= asdf (ARRAY (STRING a) (STRING b) (STRING c) (STRING d)))
diff --git a/bashast/gunit/assoc_array.gunit b/bashast/gunit/assoc_array.gunit
index a3d1508..3bdca9a 100644
--- a/bashast/gunit/assoc_array.gunit
+++ b/bashast/gunit/assoc_array.gunit
@@ -16,7 +16,7 @@
    You should have received a copy of the GNU General Public License
    along with libbash.  If not, see <http://www.gnu.org/licenses/>.
 */
-gunit bashast;
+gunit java_libbash;
 
 var_def:
 "arr[foo]=\"asdf\"" -> (= (arr (VAR_REF foo)) (STRING (DOUBLE_QUOTED_STRING asdf)))
diff --git a/bashast/gunit/brace.gunit b/bashast/gunit/brace.gunit
index 4d5d6a5..990da6a 100644
--- a/bashast/gunit/brace.gunit
+++ b/bashast/gunit/brace.gunit
@@ -16,7 +16,7 @@
    You should have received a copy of the GNU General Public License
    along with libbash.  If not, see <http://www.gnu.org/licenses/>.
 */
-gunit bashast;
+gunit java_libbash;
 
 brace_expansion:
 "{a,b}"-> (BRACE_EXP (STRING a) (STRING b))
diff --git a/bashast/gunit/command_sub.gunit b/bashast/gunit/command_sub.gunit
index 83f4026..8e9ae2e 100644
--- a/bashast/gunit/command_sub.gunit
+++ b/bashast/gunit/command_sub.gunit
@@ -16,7 +16,7 @@
    You should have received a copy of the GNU General Public License
    along with libbash.  If not, see <http://www.gnu.org/licenses/>.
 */
-gunit bashast;
+gunit java_libbash;
 
 command_sub:
 "$(echo \"foo\")" -> (COMMAND_SUB (LIST (COMMAND (STRING echo) (STRING (DOUBLE_QUOTED_STRING foo)))))
diff --git a/bashast/gunit/comment.gunit b/bashast/gunit/comment.gunit
index cf955a4..5245b93 100644
--- a/bashast/gunit/comment.gunit
+++ b/bashast/gunit/comment.gunit
@@ -16,7 +16,7 @@
    You should have received a copy of the GNU General Public License
    along with libbash.  If not, see <http://www.gnu.org/licenses/>.
 */
-gunit bashast;
+gunit java_libbash;
 
 start:
 "./hello #comment" -> (LIST (COMMAND (STRING . / hello)))
diff --git a/bashast/gunit/compound.gunit b/bashast/gunit/compound.gunit
index 3ea38a6..75996fb 100644
--- a/bashast/gunit/compound.gunit
+++ b/bashast/gunit/compound.gunit
@@ -16,7 +16,7 @@
    You should have received a copy of the GNU General Public License
    along with libbash.  If not, see <http://www.gnu.org/licenses/>.
 */
-gunit bashast;
+gunit java_libbash;
 
 cond_comparison:
 "[[ -a this/is.afile ]]" -> (COMPOUND_COND (KEYWORD_TEST (a (STRING this / is . afile))))
diff --git a/bashast/gunit/cond_main.gunit b/bashast/gunit/cond_main.gunit
index eeef052..5d6d8c2 100644
--- a/bashast/gunit/cond_main.gunit
+++ b/bashast/gunit/cond_main.gunit
@@ -16,7 +16,7 @@
    You should have received a copy of the GNU General Public License
    along with libbash.  If not, see <http://www.gnu.org/licenses/>.
 */
-gunit bashast;
+gunit java_libbash;
 
 cond_expr:
 "[[ -a this/is.afile ]]" -> (KEYWORD_TEST (a (STRING this / is . afile)))
diff --git a/bashast/gunit/continued_lines.gunit b/bashast/gunit/continued_lines.gunit
index 96f6ee4..4d1f3d7 100644
--- a/bashast/gunit/continued_lines.gunit
+++ b/bashast/gunit/continued_lines.gunit
@@ -16,7 +16,7 @@
    You should have received a copy of the GNU General Public License
    along with libbash.  If not, see <http://www.gnu.org/licenses/>.
 */
-gunit bashast;
+gunit java_libbash;
 
 start:
 
diff --git a/bashast/gunit/expansions.gunit b/bashast/gunit/expansions.gunit
index 3cc5632..386aef3 100644
--- a/bashast/gunit/expansions.gunit
+++ b/bashast/gunit/expansions.gunit
@@ -16,7 +16,7 @@
    You should have received a copy of the GNU General Public License
    along with libbash.  If not, see <http://www.gnu.org/licenses/>.
 */
-gunit bashast;
+gunit java_libbash;
 
 clist:
 "echo a{b,c,d}" -> (LIST (COMMAND (STRING echo) (STRING a (BRACE_EXP (STRING b) (STRING c) (STRING d)))))
diff --git a/bashast/gunit/fname.gunit b/bashast/gunit/fname.gunit
index 4627b46..dc66da7 100644
--- a/bashast/gunit/fname.gunit
+++ b/bashast/gunit/fname.gunit
@@ -16,7 +16,7 @@
    You should have received a copy of the GNU General Public License
    along with libbash.  If not, see <http://www.gnu.org/licenses/>.
 */
-gunit bashast;
+gunit java_libbash;
 
 fname:
 "+%Y%m%d" -> (STRING + % Y % m % d)
diff --git a/bashast/gunit/function.gunit b/bashast/gunit/function.gunit
index 55decb2..a046bd1 100644
--- a/bashast/gunit/function.gunit
+++ b/bashast/gunit/function.gunit
@@ -16,7 +16,7 @@
    You should have received a copy of the GNU General Public License
    along with libbash.  If not, see <http://www.gnu.org/licenses/>.
 */
-gunit bashast;
+gunit java_libbash;
 
 function:
 "function quit {
diff --git a/bashast/gunit/list.gunit b/bashast/gunit/list.gunit
index 354fc1e..3417557 100644
--- a/bashast/gunit/list.gunit
+++ b/bashast/gunit/list.gunit
@@ -16,7 +16,7 @@
    You should have received a copy of the GNU General Public License
    along with libbash.  If not, see <http://www.gnu.org/licenses/>.
 */
-gunit bashast;
+gunit java_libbash;
 
 start:
 "make" -> (LIST (COMMAND (STRING make)))
diff --git a/bashast/gunit/param_main.gunit b/bashast/gunit/param_main.gunit
index 67b26cf..4324dc1 100644
--- a/bashast/gunit/param_main.gunit
+++ b/bashast/gunit/param_main.gunit
@@ -16,7 +16,7 @@
    You should have received a copy of the GNU General Public License
    along with libbash.  If not, see <http://www.gnu.org/licenses/>.
 */
-gunit bashast;
+gunit java_libbash;
 
 var_ref:
 "$asdf" -> (VAR_REF asdf)
diff --git a/bashast/gunit/pipeline.gunit b/bashast/gunit/pipeline.gunit
index 3a0af5b..ec6deda 100644
--- a/bashast/gunit/pipeline.gunit
+++ b/bashast/gunit/pipeline.gunit
@@ -16,7 +16,7 @@
    You should have received a copy of the GNU General Public License
    along with libbash.  If not, see <http://www.gnu.org/licenses/>.
 */
-gunit bashast;
+gunit java_libbash;
 
 pipeline:
 "cat asdf" -> (COMMAND (STRING cat) (STRING asdf))
diff --git a/bashast/gunit/process_substitution.gunit b/bashast/gunit/process_substitution.gunit
index 0c2aa70..b0d2754 100644
--- a/bashast/gunit/process_substitution.gunit
+++ b/bashast/gunit/process_substitution.gunit
@@ -16,7 +16,7 @@
    You should have received a copy of the GNU General Public License
    along with libbash.  If not, see <http://www.gnu.org/licenses/>.
 */
-gunit bashast;
+gunit java_libbash;
 
 process_substitution:
 "<( true )" -> (PROCESS_SUBSTITUTION < (LIST (COMMAND (STRING true))))
diff --git a/bashast/gunit/redir.gunit b/bashast/gunit/redir.gunit
index 5b0a0c7..a34b910 100644
--- a/bashast/gunit/redir.gunit
+++ b/bashast/gunit/redir.gunit
@@ -16,7 +16,7 @@
    You should have received a copy of the GNU General Public License
    along with libbash.  If not, see <http://www.gnu.org/licenses/>.
 */
-gunit bashast;
+gunit java_libbash;
 
 redirect:
 ">output_file" -> (REDIR > (STRING output_file))
diff --git a/bashast/gunit/simp_command.gunit b/bashast/gunit/simp_command.gunit
index 63c5372..bf78aa4 100644
--- a/bashast/gunit/simp_command.gunit
+++ b/bashast/gunit/simp_command.gunit
@@ -16,7 +16,7 @@
    You should have received a copy of the GNU General Public License
    along with libbash.  If not, see <http://www.gnu.org/licenses/>.
 */
-gunit bashast;
+gunit java_libbash;
 
 simple_command:
 "./command" -> (STRING . / command)
diff --git a/bashast/gunit/simp_prog.gunit b/bashast/gunit/simp_prog.gunit
index 9a1d88e..64aa001 100644
--- a/bashast/gunit/simp_prog.gunit
+++ b/bashast/gunit/simp_prog.gunit
@@ -16,7 +16,7 @@
    You should have received a copy of the GNU General Public License
    along with libbash.  If not, see <http://www.gnu.org/licenses/>.
 */
-gunit bashast;
+gunit java_libbash;
 
 start:
 "echo \"finding file\"
diff --git a/test/post_check.cpp b/test/post_check.cpp
index 2a411f8..3f0531e 100644
--- a/test/post_check.cpp
+++ b/test/post_check.cpp
@@ -35,12 +35,12 @@ static void check_file_size(const char *path, uintmax_t size_max)
 
 TEST(post_check, lexer_size)
 {
-  check_file_size("libbashLexer.c", 512 * 1024);
+  check_file_size("libbashLexer.cpp", 512 * 1024);
 }
 
 TEST(post_check, parser_size)
 {
-  check_file_size("libbashParser.c", 3*1024*1024);
+  check_file_size("libbashParser.cpp", 3*1024*1024);
 }
 
 TEST(post_check, walker_size)
diff --git a/test/verify_error_output_test.sh b/test/verify_error_output_test.sh
index ebc5b3c..fa6b50d 100755
--- a/test/verify_error_output_test.sh
+++ b/test/verify_error_output_test.sh
@@ -2,4 +2,4 @@
 
 illegal="${srcdir}/scripts/illegal_script.sh"
 output=$(./variable_printer "$illegal" 2>&1)
-[[ $output == "${illegal}(1)  : error 3 : 130:1: command_atom : ( compound_command | function | simple_command );, at offset 3"* ]]
+[[ $output == "${illegal}(1)  : error 3 : 120:1: command_atom : ( compound_command | function | simple_command );, at offset 3"* ]]