From 890af6013a76a72fe8849c99f69ed876e7239322 Mon Sep 17 00:00:00 2001
From: paulr <paulr@tv.pjr.cc>
Date: Sun, 30 Dec 2012 21:32:28 +1100
Subject: [PATCH] adding the tokenizer to the repo for historical purposes

---
 tokeniser.js |   48 ++++++++++++++++++++++++++++++++++++++++++++++++
 1 files changed, 48 insertions(+), 0 deletions(-)
 create mode 100644 tokeniser.js

diff --git a/tokeniser.js b/tokeniser.js
new file mode 100644
index 0000000..749b95f
--- /dev/null
+++ b/tokeniser.js
@@ -0,0 +1,48 @@
+var mys = "1 this is <?njs  \n\n\n\n\n calling:asdf \n\n\n\n   ?> for \n\n\n the second one its <?njs this.fucku ?> end here <?njs asdfasdf asdf?>";
+
+
+//var tool = mys.split("<?njs");
+
+/*
+for(var i=0; i < tool.length; i++) {
+	var thist = tool[i];
+	console.log("this t: '%s'", thist.trim());
+	var calling = thist.split("?>")[0].trim();
+	console.log("calling: '%s'", calling);
+}
+*/
+
+function splitandcall(thisstring) {
+	var tool = thisstring.split("<?njs");
+
+	var thisoffset=1;
+
+	//console.log("output: '%s'", tool[thisoffset-1].replace("/^start://", ""));
+
+	function calloffset() {
+		var thisca = tool[thisoffset];
+		var calling = thisca.split("?>");
+		dummy_call(calling[0].trim(), function() {
+			thisoffset++;
+			if(thisoffset < tool.length) {
+				console.log("output: '%s'", calling[1]);
+				calloffset();
+			} else {
+				console.log("output: '%s'", calling[1]);
+				console.log("at end");
+				return;
+			}
+		});
+	}
+
+	calloffset();
+}
+
+function dummy_call(name, callback)
+{
+	console.log("in '%s' for name", name);
+	callback();
+}
+
+console.log("mys: '%s'", mys);
+splitandcall("start:" + mys + ":end");
-- 
1.7.0.4