From d9c7eb8248208029df200a897d680914cd0f337f Mon Sep 17 00:00:00 2001 From: Paul J R Date: Sun, 20 Jan 2013 05:22:58 +1100 Subject: [PATCH] lots of work on the request pipeline --- lib/cache.js | 97 ++++++++++++++++++++++++++++------ lib/config.js | 13 ++++- lib/router.js | 148 +++++++++++++++++++++++++++++----------------------- unittests/unify.js | 49 +++++++++++++++++ 4 files changed, 222 insertions(+), 85 deletions(-) create mode 100644 unittests/unify.js diff --git a/lib/cache.js b/lib/cache.js index 18d0045..bf1a3b0 100644 --- a/lib/cache.js +++ b/lib/cache.js @@ -1,5 +1,6 @@ var fs = require("fs"); - +var http = require("http"); +var url = require("url"); function maintainCache() { // TODO i should check that im already running here and exit if i am @@ -14,31 +15,93 @@ exports.startTimer = function() { setInterval(maintainCache, cacheTimer); } +function upstreamRequest(unify, callback) { + // first do a head request + console.log("upsteram as ", unify.requestFor); + + var splpath = unify.requestFor.split("/"); + var topdir = splpath[1]; + var toppath = ""; + for(var i=2; i < splpath.length; i++) { + if(toppath == "") { + toppath = splpath[i]; + } else { + toppath += "/" + splpath[i]; + } + } + console.log("uppath = '%s' and '%s'", topdir, toppath); + if(typeof global.repoproxy.repo[topdir] != "undefined") { + console.log("which is ", global.repoproxy.repo[topdir]); + console.log("so upstream is, ", global.repoproxy.repo[topdir].url + toppath); + } + +} + +exports.upstreamRequest = upstreamRequest; // the service file routine .... PLEASE KILL ME! -function serviceFile(reqpath, res, range) { +function serviceFile(unify) { // for now, ignore range. + + // file should already exist, so we just poop it out + var inp = fs.createReadStream(unify.fullFilePath); + inp.setEncoding("utf8"); + inp.on("data", function(data) { + unify.b.write(data); + }); - fs.exists(reqpath, function(exists) { - if(exists) { - var inp = fs.createReadStream(reqpath); - inp.setEncoding("utf8"); - inp.on("data", function(data) { - res.write(data); - }); + inp.on("end", function(closed) { + unify.b.end(); + }); +} + +exports.serviceFile = serviceFile; + +function serviceDirectory(unify) { + var nfiles = 0; + var res = unify.b; + + res.write("

Directory listing for " + unify.originalReq + "

");
+	if(unify.fullFilePath != "/") res.write("Parent\n\n");
+	fs.readdir(unify.fullFilePath, function(err, files) {
+		console.log("doing directory listing on: ", unify.fullFilePath);
+		if(err == null) {
+			
+			// TODO: make this work asynchronously...
+			for(var i=0; i"+files[i]+"/\n");
+						nfiles++;
+					} else if(stats.isFile()) {
+						var padlength = 80 - (files[i].length) - stats.size.toString().length;
+						var padding = "";
+						if(padlength > 0) {
+							padding = new Array(padlength).join(" ");
+						}
+						res.write("File:      "+files[i]+""+padding+stats.size+" bytes\n");
+						nfiles++;
+					}
+				} else {
+					console.log("ignoring file, ", files[i]);
+				}
+			}
+			
+			if(nfiles == 0) res.write("Empty directory....\n");
 			
-			// TODO, we need to send this upstream, if its upstream we go up.
-			res.writeHead(404, {"Content-Type": "text/plain"});
-			res.write("404 Not Found\n");
+			res.write("
"); + res.end(); + } else { + res.write("we have entered bizaro world...\n"); + res.write(""); res.end(); } }); } -exports.serviceFile = serviceFile; +exports.serviceDirectory = serviceDirectory; \ No newline at end of file diff --git a/lib/config.js b/lib/config.js index dd13be4..441caed 100644 --- a/lib/config.js +++ b/lib/config.js @@ -33,9 +33,9 @@ exports.loadConfig = function (conffile) { break; case "cachedir": - console.log("Cache dir set to: ", line_real[1]); - global.repoproxy.cacheDir = line_real[1]; - + var tmppath = line_real[1].replace(/\/+/g, "/"); + console.log("Cache dir set to: ", tmppath); + global.repoproxy.cacheDir = tmppath; break; case "listenport": console.log("Port set to: ", line_real[1]); @@ -67,11 +67,18 @@ function createCacheStructure() { } catch(e) { try { fs.mkdirSync(global.repoproxy.cacheDir); + fs.mkdirSync(global.repoproxy.cacheDir + "/.cleanup"); } catch(ex) { console.log("ERROR: failure to create cache directory, '%s'", global.repoproxy.cacheDir); } } + try { + fs.mkdirSync(global.repoproxy.cacheDir + "/.cleanup"); + } catch(ex) { + console.log("ERROR: cant create cleanup directory, '%s'", global.repoproxy.cacheDir + "/.cleanup"); + } + for(var index in global.repoproxy.repo) { var fullDir = global.repoproxy.cacheDir + "/" + index; try { diff --git a/lib/router.js b/lib/router.js index 96ca844..71fddb1 100644 --- a/lib/router.js +++ b/lib/router.js @@ -1,83 +1,101 @@ var url = require("url"); var fs = require("fs"); var cache = require("./cache.js"); +var path = require("path"); exports.routeRequest = function(req, res) { - // first, strip a /pub/ off the front if it exists - var originalurl = url.parse(req.url); - var range = 0; - - thisurl = originalurl.pathname.replace(/^\/pub/, ""); - - console.log("pathname now: ", thisurl); - - //if(thisurl.pathname == "") thisurl.pathname = "/"; - - var reqpath = global.repoproxy.cacheDir + "/" + thisurl; - - console.log("request on '%s'", reqpath); - - // see what we're dealing with - fs.stat(reqpath, function(err, stat) { - console.log("err is ", err); - console.log("stat is ", stat); - console.log("fs.stats is ", fs.stats); - - if(err == null) { - if(stat.isDirectory()) { - if(originalurl.pathname.charAt(originalurl.pathname.length-1) != "/") { - // redirect to url + "/" - res.writeHead("302", { "Location": originalurl.pathname+"/" }); - res.end(); - } else { - writeDirectoryListing(reqpath, originalurl.pathname, res); - } + // first, unify the request + var thisQuery = unifyRequest(req, res, function(unified) { + if(unified.exists) { + if(unified.isFile) { + cache.serviceFile(unified); + } else if(unified.isDirectory) { + cache.serviceDirectory(unified); } else { - if(stat.isFile()) { - cache.serviceFile(reqpath, res, range); - } + console.log("ERROR: something went majorly wrong with something, ", unified); } } else { - // go upstream.. - cache.serviceFile(reqpath, res, range); + // it doesnt exist yet, so we send it to the cache service + console.log("file doesnt exist, upstream we go: ", unified); + cache.upstreamRequest(unified, function(err) { + if(err == null) { + cache.watchAndService(unfied); + } // if upstream sends anything other then a 200, cache.upstreamRequest will handle it (i.e. 302, 404, etc) + }); } - }); + }); } -function writeDirectoryListing(reqpath, requesturi, res) { - res.write("

Directory listing for " + requesturi + "

");
-	if(requesturi != "/") res.write("Parent\n\n");
-	fs.readdir(reqpath, function(err, files) {
-		console.log("doing directory listing on: ", reqpath);
+function unifyRequest(req, res, callback, testing) {
+	var unified = new Object();
+	var originalurl = url.parse(req.url);
+	
+	// create the base unified object
+	unified.a = req;
+	unified.b = res;
+
+	// create the request url
+	// remove /pub if it exists
+	unified.requestFor = originalurl.pathname.replace(/^\/pub/, "");
+	unified.originalReq = originalurl.pathname;
+	
+	// create the full file path by spanning the cachedir
+	unified.fullFilePath = (global.repoproxy.cacheDir + "/" + originalurl.pathname.replace(/^\/pub/, "")).replace(/\/+/g, "/");
+	
+	// determine if the request is for a directory
+	if(unified.requestFor.match(/\/$/) != null) {
+		unified.isDirectoryRequest = true;
+		unified.fullPathDirName = unified.fullFilePath;
+	} else {
+		unified.isDirectoryRequest = false;
+		unified.fullPathDirName = path.dirname(unified.fullFilePath);
+	}
+	
+	// determine the topPath, subpath etc.
+	var spl = unified.requestFor.split("/");
+	unified.topPath = spl[1];
+	unified.subPath = "";
+	if(spl.length > 2) {
+		for(var i=2; i < spl.length; i++) {
+			if(unified.subPath == "") unified.subPath = spl[i];
+			else unified.subPath += "/" + spl[i];
+		}
+	} else {
+		unified.subPath = null;
+	}
+	
+	
+	fs.stat(unified.fullFilePath, function(err, stats) {
 		if(err == null) {
+			unified.exists = true;
+			if(stats.isDirectory() && !unified.isDirectoryRequest) {
+				//send a 302 and call it a day
+				res.writeHead("302", { 'Location': unified.originalReq+"/" });
+				res.end();
+				
+				// TODO: remove this after testing
+				if(testing) callback(null);
+				return 302;
+			}
 			
-			// TODO: make this work asynchronously...
-			if(files.length == 0) {
-				res.write("Empty Directory....\b");
+			if(stats.isDirectory()) {
+				unified.isDirectory = true;
+				unified.isFile = false;
+			} else if(stats.isFile()) {
+				unified.isDirectory = false;
+				unified.isFile = true;				
 			} else {
-				for(var i=0; i"+files[i]+"/\n");
-					} else if(stats.isFile()) {
-						var padlength = 80 - (files[i].length) - stats.size.toString().length;
-						var padding = "";
-						if(padlength > 0) {
-							padding = new Array(padlength).join(" ");
-						}
-						res.write("File:      "+files[i]+""+padding+stats.size+" bytes\n");
-					}
-				}
+				unified.isDirectory = false;
+				unified.isFile = false;
 			}
-			res.write("
"); - res.end(); } else { - res.write("we have entered bizaro world...\n"); - res.write(""); - res.end(); + unified.exists = false; } + + callback(unified); }); -} \ No newline at end of file + + return 0; +} + +exports.unifyRequest = unifyRequest; \ No newline at end of file diff --git a/unittests/unify.js b/unittests/unify.js new file mode 100644 index 0000000..7eabb9a --- /dev/null +++ b/unittests/unify.js @@ -0,0 +1,49 @@ +var router = require("../lib/router.js"); + +global.repoproxy = new Object(); +global.repoproxy.cacheDir = "./cache"; + +var paths = new Array(); +var i = 0; +paths[0] ="/pub/something/"; +paths[1] ="/pub/something"; +paths[2] ="/pub/fedora"; +paths[3] ="/pub/fedora/"; +paths[4] ="/pub/ts.js"; +paths[5] ="/ts.js"; +paths[6] ="/fedora"; +paths[7] = "/fedora/"; +paths[8] = "/"; +paths[9] = "/fedora/some/directory/in/here/"; +paths[10] = "/fedora/some/directory/in/here/file"; + +trial(paths[i]); + +function trial(path) { + + console.log("\n\n\n\n\nBEGIN TEST on '%s'", path); + + var req = new Object(); + var res = new Object(); + req.url = path; + + res.writeHead = function(n, o) { + console.log("write head for: ", n); + console.log("and o: ",o); + } + + res.end = function() { + return; + } + + var result = router.unifyRequest(req, res, function(uni) { + console.log("I is ", i); + console.log("Uni is ", uni); + console.log("from path: ", path); + + i++; + + if(typeof paths[i] != "undefined") trial(paths[i]); + }, true); +} + -- 1.7.0.4