From 68fbf923faf99f86cc36dc5776451f89925df8bd Mon Sep 17 00:00:00 2001 From: Paul J R Date: Sun, 20 Jan 2013 16:42:10 +1100 Subject: [PATCH] upstream now appears to work just fine --- lib/cache.js | 135 +++++++++++++++++++++++++++++++++++++++++++++++++++------ lib/config.js | 10 +++- lib/router.js | 26 +++++++---- 3 files changed, 144 insertions(+), 27 deletions(-) diff --git a/lib/cache.js b/lib/cache.js index bf1a3b0..e065a2a 100644 --- a/lib/cache.js +++ b/lib/cache.js @@ -19,26 +19,116 @@ function upstreamRequest(unify, callback) { // first do a head request console.log("upsteram as ", unify.requestFor); - var splpath = unify.requestFor.split("/"); - var topdir = splpath[1]; - var toppath = ""; - for(var i=2; i < splpath.length; i++) { - if(toppath == "") { - toppath = splpath[i]; - } else { - toppath += "/" + splpath[i]; - } - } - console.log("uppath = '%s' and '%s'", topdir, toppath); - if(typeof global.repoproxy.repo[topdir] != "undefined") { - console.log("which is ", global.repoproxy.repo[topdir]); - console.log("so upstream is, ", global.repoproxy.repo[topdir].url + toppath); + var endData = false; + var xpath = ""; + var filefd = null; + if(unify.topPath !=null) if(unify.topPath != "") if(typeof global.repoproxy.repo[unify.topPath] != "undefined") { + var uplink = global.repoproxy.repo[unify.topPath].url; + xpath = uplink + unify.subPath; } + //unify.b.write("would send to '" + xpath + "'"); + //unify.b.end(); + + console.log("sending off to '%s'", xpath); + + var headReq = url.parse(xpath); + headReq["method"] = "HEAD"; + + getup = http.request(xpath, function(res) { + res.setEncoding("utf8"); + + if(!endData) { + console.log("status code is ", typeof res.statusCode); + switch(res.statusCode) { + // TODO: this 301 directory redirect thing needs to work better + case 301: + case 302: + + var loc = res.headers.location.substr(res.headers.location.length-4); + var against_t = xpath + "/"; + var against = against_t.substr(against_t.length-4); + + if(loc == against) { + console.log("got a redirect, upstream for loc => loc/ assuming its a directory"); + makeCacheDir(unify); + unify.b.writeHead(302, { "Location": unify.originalReq + "/" }); + } else { + console.log("checked '%s' against '%s', was false, sending 404", loc, against); + unify.b.writeHead(404, {"Content-Type": "text/plain"}); + unify.b.write("404 Not Found\n"); + } + unify.b.end(); + endData = true; + break; + + case 404: + unify.b.writeHead(404, {"Content-Type": "text/plain"}); + unify.b.write("404 Not Found\n"); + unify.b.end(); + endData = true; + break; + case 200: + makeCacheDir(unify); + if(unify.isDirectoryRequest) { + serviceDirectory(unify); + endData = true; + } else { + // this is where it gets ugly + console.log("do ugly write: ", unify); + //unify.b.write(data); + getAndService(unify, xpath); + + } + break; + default: + console.log(".... data"); + //unify.b.write(data); + } + } + //console.log("res is now ", res); + }); + + getup.end(); + + //console.log("getup: ", getup); } exports.upstreamRequest = upstreamRequest; +function getAndService(unify, xpath) { + + if(global.repoproxy.downloads[unify.fullFilePath] == 1) { + + unify.b.write("trying to service inline"); + unify.b.end(); + } else { + global.repoproxy.downloads[unify.fullFilePath] = 1; + + http.get(xpath, function(res) { + + var file = fs.createWriteStream(unify.fullFilePath); + + console.log("res: ", res); + + //res.setEncoding("utf8"); + + res.on("data", function(data) { + //console.log("chunk"); + file.write(data); + unify.b.write(data); + }); + + res.on("end", function() { + console.log("end..."); + unify.b.end(); + file.end(); + global.repoproxy.downloads[unify.fullFilePath] = 0; + }); + }); + } +} + // the service file routine .... PLEASE KILL ME! function serviceFile(unify) { @@ -58,6 +148,23 @@ function serviceFile(unify) { exports.serviceFile = serviceFile; +function makeCacheDir(path) { + console.log("attempting to create... '%s' as '%s'", path.fullPathDirName, path.subPathDirName); + + var startAt = path.topFullPath; + var nextbits = path.subPathDirName.split("/"); + for(var i=0; i < nextbits.length; i++) { + startAt += "/" + nextbits[i]; + console.log("attempt mkdir on '%s'", startAt); + try { + fs.mkdirSync(startAt); + } catch(e) { + //console.log("e in mkdir, ", e); + } + } + //process.exit(0); +} + function serviceDirectory(unify) { var nfiles = 0; var res = unify.b; diff --git a/lib/config.js b/lib/config.js index 441caed..7cab68a 100644 --- a/lib/config.js +++ b/lib/config.js @@ -6,7 +6,8 @@ exports.loadConfig = function (conffile) { global.repoproxy.listenPort = 8008; global.repoproxy.cacheDir = "./cache"; global.repoproxy.repo = new Object(); - global.repoproxy.scancache = 1; + global.repoproxy.scancache = 1; + global.repoproxy.downloads = new Object(); var confFileData = fs.readFileSync(conffile, "utf8"); @@ -79,13 +80,16 @@ function createCacheStructure() { console.log("ERROR: cant create cleanup directory, '%s'", global.repoproxy.cacheDir + "/.cleanup"); } + console.log("next: ", global.repoproxy.repo); for(var index in global.repoproxy.repo) { var fullDir = global.repoproxy.cacheDir + "/" + index; + console.log("on end, ", fullDir); try { - var state = fs.statSync(global.repoproxy.cacheDir); - //console.log("state is:", state); + var state = fs.statSync(fullDir); + console.log("state is:", state); } catch(e) { try { + console.log("attempted to create cache dir, ", fullDir); fs.mkdirSync(fullDir); } catch(ex) { console.log("ERROR: failed to create cache directory, '%s' for '%s'", fullDir, index); diff --git a/lib/router.js b/lib/router.js index 71fddb1..8a7dca2 100644 --- a/lib/router.js +++ b/lib/router.js @@ -6,7 +6,10 @@ var path = require("path"); exports.routeRequest = function(req, res) { // first, unify the request var thisQuery = unifyRequest(req, res, function(unified) { - if(unified.exists) { + if(unified.requestFor == "/favicon.ico") { + unified.b.writeHead(404, {"Content-Type": "text/plain"}); + unified.b.write("404 Not Found\n"); + } else if(unified.exists) { if(unified.isFile) { cache.serviceFile(unified); } else if(unified.isDirectory) { @@ -42,18 +45,10 @@ function unifyRequest(req, res, callback, testing) { // create the full file path by spanning the cachedir unified.fullFilePath = (global.repoproxy.cacheDir + "/" + originalurl.pathname.replace(/^\/pub/, "")).replace(/\/+/g, "/"); - // determine if the request is for a directory - if(unified.requestFor.match(/\/$/) != null) { - unified.isDirectoryRequest = true; - unified.fullPathDirName = unified.fullFilePath; - } else { - unified.isDirectoryRequest = false; - unified.fullPathDirName = path.dirname(unified.fullFilePath); - } - // determine the topPath, subpath etc. var spl = unified.requestFor.split("/"); unified.topPath = spl[1]; + unified.topFullPath = (global.repoproxy.cacheDir + "/" + spl[1]).replace(/\/+/g, "/"); unified.subPath = ""; if(spl.length > 2) { for(var i=2; i < spl.length; i++) { @@ -64,6 +59,17 @@ function unifyRequest(req, res, callback, testing) { unified.subPath = null; } + // determine if the request is for a directory + if(unified.requestFor.match(/\/$/) != null) { + unified.isDirectoryRequest = true; + unified.fullPathDirName = unified.fullFilePath; + unified.subPathDirName = unified.subPath; + } else { + unified.isDirectoryRequest = false; + unified.fullPathDirName = path.dirname(unified.fullFilePath); + unified.subPathDirName = path.dirname(unified.subPath); + } + fs.stat(unified.fullFilePath, function(err, stats) { if(err == null) { -- 1.7.0.4