adding the tokenizer to the repo for historical purposes

diff --git a/tokeniser.js b/tokeniser.js
new file mode 100644 (file)
index 0000000..749b95f
--- /dev/null
+++ b/tokeniser.js
@@ -0,0 +1,48 @@
+var mys = "1 this is <?njs  \n\n\n\n\n calling:asdf \n\n\n\n   ?> for \n\n\n the second one its <?njs this.fucku ?> end here <?njs asdfasdf asdf?>";
+
+
+//var tool = mys.split("<?njs");
+
+/*
+for(var i=0; i < tool.length; i++) {
+       var thist = tool[i];
+       console.log("this t: '%s'", thist.trim());
+       var calling = thist.split("?>")[0].trim();
+       console.log("calling: '%s'", calling);
+}
+*/
+
+function splitandcall(thisstring) {
+       var tool = thisstring.split("<?njs");
+
+       var thisoffset=1;
+
+       //console.log("output: '%s'", tool[thisoffset-1].replace("/^start://", ""));
+
+       function calloffset() {
+               var thisca = tool[thisoffset];
+               var calling = thisca.split("?>");
+               dummy_call(calling[0].trim(), function() {
+                       thisoffset++;
+                       if(thisoffset < tool.length) {
+                               console.log("output: '%s'", calling[1]);
+                               calloffset();
+                       } else {
+                               console.log("output: '%s'", calling[1]);
+                               console.log("at end");
+                               return;
+                       }
+               });
+       }
+
+       calloffset();
+}
+
+function dummy_call(name, callback)
+{
+       console.log("in '%s' for name", name);
+       callback();
+}
+
+console.log("mys: '%s'", mys);
+splitandcall("start:" + mys + ":end");