From dcae8bd329fa98359031d4fc7c93d6c6e4b93fd9 Mon Sep 17 00:00:00 2001
From: Bikram Chatterjee <razorpeak@gmail.com>
Date: Wed, 18 Apr 2018 16:26:27 +0200
Subject: [PATCH] #189 macro processing

---
 rebar.config          |  3 ++-
 src/imem.app.src      |  3 ++-
 src/imem_compiler.erl | 22 ++++++++++++++++++----
 3 files changed, 22 insertions(+), 6 deletions(-)

diff --git a/rebar.config b/rebar.config
index 56fe197..7436307 100755
--- a/rebar.config
+++ b/rebar.config
@@ -42,7 +42,8 @@
     {lager, {git, "https://github.com/K2InformaticsGmbH/lager.git", {tag, "3.4.2"}}},
     {ranch, {git, "https://github.com/K2InformaticsGmbH/ranch.git", {tag, "1.3.2"}}},
     {sext, {git, "https://github.com/K2InformaticsGmbH/sext.git", {tag, "1.4.1"}}},
-    {sqlparse, {git, "https://github.com/K2InformaticsGmbH/sqlparse.git", {branch, "master"}}}
+    {sqlparse, {git, "https://github.com/K2InformaticsGmbH/sqlparse.git", {branch, "master"}}},
+    {aleppo, {git, "https://github.com/K2InformaticsGmbH/aleppo", {tag, "v0.9.3"}}}
 ]}.
 
 {erl_first_files, ["src/imem_rec_pretty_pt.erl"]}.
diff --git a/src/imem.app.src b/src/imem.app.src
index 1094b59..67ba589 100755
--- a/src/imem.app.src
+++ b/src/imem.app.src
@@ -16,7 +16,8 @@
         lager,
         ranch,
         sext,
-        sqlparse
+        sqlparse,
+        aleppo
     ]},
     {env, [
         {erl_cluster_mgrs, []},
diff --git a/src/imem_compiler.erl b/src/imem_compiler.erl
index 598d40a..d3fb9dd 100644
--- a/src/imem_compiler.erl
+++ b/src/imem_compiler.erl
@@ -138,9 +138,8 @@ nonLocalHFun({Mod, Fun} = FSpec, Args, SafeFuns) ->
 compile_mod(ModuleCodeBinStr) -> compile_mod(ModuleCodeBinStr, [], []).
 compile_mod(ModuleCodeBinStr, Opts) -> compile_mod(ModuleCodeBinStr, [], Opts).
 compile_mod(ModuleCodeBinStr, Restrict, Opts) when is_binary(ModuleCodeBinStr) ->
-    case erl_scan:string(binary_to_list(ModuleCodeBinStr)) of
-        {ok, Tokens, _} ->
-            TokenGroups = cut_dot(Tokens),
+    case tokenize(ModuleCodeBinStr) of
+        {ok, TokenGroups} ->
             case lists:foldl(
                     fun(TokenGroup, Acc) when is_list(Acc) ->
                             case erl_parse:parse_form(TokenGroup) of
@@ -169,6 +168,20 @@ compile_mod(ModuleCodeBinStr, Restrict, Opts) when is_binary(ModuleCodeBinStr) -
                     end;
                 Error -> Error
             end;
+        Error -> Error
+    end.
+
+tokenize(ModuleCodeBinStr) ->
+    case erl_scan:string(binary_to_list(ModuleCodeBinStr), {0,1}) of
+        {ok, RawTokens, _} ->
+            case aleppo:process_tokens(RawTokens) of
+                {ok, TokensEOF} ->
+                    [{eof,_} | RevTokens] = lists:reverse(TokensEOF),
+                    Tokens = lists:reverse(RevTokens),
+                    {ok, cut_dot(Tokens)};
+                {error, Error} ->
+                    {error, {preprocess, {{0, 1}, ?MODULE, Error}, {0, 1}}}
+            end;
         {error, ErrorInfo, ErrorLocation} ->
             {error, {scan, ErrorInfo, ErrorLocation}}
     end.
@@ -185,10 +198,11 @@ error_info(Type, [{_, _, _} = ErrorInfo | ErrorInfos]) ->
     [error_info(Type, ErrorInfo) | error_info(Type, ErrorInfos)];
 error_info(Type, [{_,ErrorInfos}|Tail]) ->
     error_info(Type, ErrorInfos) ++ error_info(Type, Tail);
-error_info(Type, {Line, Module, ErrorDesc}) ->
+error_info(Type, {{Line, Column}, Module, ErrorDesc}) ->
     #{  
         type => Type,
         row => Line,
+        col => Column,
         text => list_to_binary(Module:format_error(ErrorDesc))
     }.