X-Git-Url: http://git.pjr.cc/?p=nodejs-repoproxy.git;a=blobdiff_plain;f=lib%2Fcache.js;h=9e5e8daf2cb5b630d86f9a91cc25ecf9a1c0ce86;hp=18d0045aa4759ac27fe51d47cdf52ceafa6ea67d;hb=43d51752ca1aafe7212ee3805ba1b33e398c91ce;hpb=c3e6676d533e875b020d231075aac04e4b885677
diff --git a/lib/cache.js b/lib/cache.js
index 18d0045..9e5e8da 100644
--- a/lib/cache.js
+++ b/lib/cache.js
@@ -1,5 +1,7 @@
var fs = require("fs");
-
+var http = require("http");
+var url = require("url");
+var path = require("path");
function maintainCache() {
// TODO i should check that im already running here and exit if i am
@@ -14,31 +16,333 @@ exports.startTimer = function() {
setInterval(maintainCache, cacheTimer);
}
+function upstreamRequest(unify) {
+ // first do a head request
+ console.log("upsteram as ", unify.requestFor);
+
+ var endData = false;
+ var xpath = "";
+ var filefd = null;
+ if(unify.topPath !=null) if(unify.topPath != "") if(typeof global.repoproxy.repo[unify.topPath] != "undefined") {
+ var uplink = global.repoproxy.repo[unify.topPath].url;
+ xpath = uplink + unify.subPath;
+ }
+
+ //unify.b.write("would send to '" + xpath + "'");
+ //unify.b.end();
+
+ // not doing this properly yet...
+ if(typeof global.repoproxy.downloads[unify.fullFilePath] != undefined && global.repoproxy.downloads[unify.fullFilePath] == 1) {
+ console.log("request for file thats being downloaded already, doing inline request");
+ inlineService(unify);
+ return;
+ }
+
+ console.log("sending off to '%s'", xpath);
+
+ var headReq = url.parse(xpath);
+ headReq["method"] = "HEAD";
+
+ getup = http.request(headReq, function(res) {
+ //res.setEncoding("utf8");
+
+ if(!endData) {
+ console.log("status code is ", typeof res.statusCode);
+ switch(res.statusCode) {
+ // TODO: this 301 directory redirect thing needs to work better
+ case 301:
+ case 302:
+
+ var loc = res.headers.location.substr(res.headers.location.length-4);
+ var against_t = xpath + "/";
+ var against = against_t.substr(against_t.length-4);
+
+ if(loc == against) {
+ console.log("got a redirect, upstream for loc => loc/ assuming its a directory");
+ makeCacheDir(unify);
+ unify.b.writeHead(302, { "Location": unify.originalReq + "/" });
+ } else {
+ console.log("checked '%s' against '%s', was false, sending 404", loc, against);
+ unify.b.writeHead(404, {"Content-Type": "text/plain"});
+ unify.b.write("404 Not Found\n");
+ }
+ unify.b.end();
+ endData = true;
+ break;
+
+ case 404:
+ unify.b.writeHead(404, {"Content-Type": "text/plain"});
+ unify.b.write("404 Not Found\n");
+ unify.b.end();
+ endData = true;
+ break;
+ case 200:
+ makeCacheDir(unify);
+ if(unify.isDirectoryRequest) {
+ serviceDirectory(unify);
+ endData = true;
+ } else {
+ // this is where it gets ugly
+ var filesize = res.headers["content-length"];
+ console.log("do ugly write: ", unify);
+ //unify.b.write(data);
+ var metafilename = unify.fullPathDirName + "/.meta."+ path.basename(unify.requestFor) +".filesize";
+ var metafile = fs.createWriteStream(metafilename);
+ metafile.write(filesize);
+ metafile.end();
+ getAndService(unify, xpath, filesize);
+
+ }
+ break;
+ default:
+ console.log(".... data");
+ //unify.b.write(data);
+ }
+ }
+ //console.log("res is now ", res);
+ });
+
+ getup.end();
+
+ //console.log("getup: ", getup);
+}
+
+exports.upstreamRequest = upstreamRequest;
+
+function getAndService(unify, xpath, filesize) {
+
+ console.log("calling in here with filesize, ", filesize)
+ unify.b.writeHead(200, {'Content-Length' : filesize});
+
+
+ global.repoproxy.downloads[unify.fullFilePath] = 1;
+
+
+ http.get(xpath, function(res) {
+
+ var file = fs.createWriteStream(unify.fullFilePath);
+
+ //console.log("res: ", res);
+
+ //res.setEncoding("utf8");
+
+ res.on("data", function(data) {
+ //console.log("chunk");
+ file.write(data);
+ unify.b.write(data);
+ });
+
+ res.on("end", function() {
+ console.log("end...");
+ unify.b.end();
+ file.end();
+ global.repoproxy.downloads[unify.fullFilePath] = 0;
+ });
+
+ res.on("error", function(err) {
+ console.log("res threw error... ", err);
+ });
+ });
+}
+
+// this is nasty nasty thing that can go horribly wrong in some ways, but currently works...
+function inlineService(unify) {
+ // this method is called when we need to service a file thats being downloaded by something else
+ var metafilename = unify.fullPathDirName + "/.meta."+ path.basename(unify.requestFor) +".filesize";
+ var fsizef = fs.createReadStream(metafilename);
+ var fsize = "";
+ var lastchunk = 0;
+ fsizef.on("data", function(data) {
+ fsize += data;
+ });
+
+ fsizef.on("end", function() {
+ var sentSoFar = 0;
+ unify.b.writeHead(200, {"Content-Length" : fsize });
+
+ // now we go into the file reading loop.
+ console.log("start of inline services");
+ // we loop every 0.5s and do our thing
+
+ function sendPieces() {
+ // this is going to be so fun i want to play real life frogger in real life traffic...
+ fs.stat(unify.fullFilePath, function(err, stats) {
+ if(err == null) {
+ if(stats["size"] > sentSoFar) {
+ // if file size changed between last chunk and this chunk, send the chunks
+
+ lastChunk = 0;
+ // open the file, send the data
+ var rs = fs.createReadStream(unify.fullFilePath, {start: sentSoFar, end: stats["size"]});
+
+ rs.on("data", function(thisdata) {
+ //console.log("inline chunk: ", thisdata.length);
+ unify.b.write(thisdata);
+ });
+
+ rs.on("end", function() {
+ sentSoFar = stats["size"];
+ // every second, we start again
+ if(sentSoFar != fsize) {
+ setTimeout(sendPieces, 1000);
+ } else {
+ // we're done!
+ unify.b.end();
+ }
+ });
+ } else {
+ // if file size did not change between last timeout and this one, incremement the chunk counter
+ // if we reach 60, we had a problem, and so we bomb out
+
+ lastChunk++;
+
+ // we bombed out somehow
+ if(lastChunk > 60) {
+ unify.b.end();
+ } else {
+ setTimeout(sendPieces, 1000);
+ }
+ }
+ } else {
+ console.log("inline service - we're in a very bad place");
+ }
+ });
+
+ }
+
+ setTimeout(sendPieces, 100);
+ });
+}
// the service file routine .... PLEASE KILL ME!
-function serviceFile(reqpath, res, range) {
+function serviceFile(unify) {
// for now, ignore range.
+ // however we need to check if a metadata file exists describing the filesize, check if its all correct
+ // and if not, erase the file (and metafile) and forward the request back to upstream request
+
- fs.exists(reqpath, function(exists) {
- if(exists) {
- var inp = fs.createReadStream(reqpath);
- inp.setEncoding("utf8");
- inp.on("data", function(data) {
- res.write(data);
- });
-
- inp.on("end", function(closed) {
- res.end();
+ checkFile(unify, function() {
+
+ // file should already exist, so we just poop it out
+ var inp = fs.createReadStream(unify.fullFilePath);
+ //inp.setEncoding("utf8");
+ inp.on("data", function(data) {
+ unify.b.write(data);
+ });
+
+ inp.on("end", function(closed) {
+ unify.b.end();
+ });
+ });
+}
+
+exports.serviceFile = serviceFile;
+
+
+function checkFile(unify, callback) {
+ // in here we do the metadata checks
+ var metafilename = unify.fullPathDirName + "/.meta."+ path.basename(unify.requestFor) +".filesize";
+
+ fs.exists(metafilename, function(existence) {
+ if(existence) {
+ var fsizef = fs.createReadStream(metafilename);
+ var fsize = "";
+ fsizef.on("data", function(data) {
+ fsize += data;
+ });
+
+ fsizef.on("end", function() {
+ fs.stat(unify.fullFilePath, function(err, stats) {
+ var rfsize = stats["size"];
+ if(rfsize != fsize.trim()) {
+ // remove the file and start again
+ console.log("reported filesizes dont match, '%s', '%s', removing file and starting again", rfsize, stats["size"]);
+ try {
+ fs.unlink(metafilename, function(){
+ fs.unlink(unify.fullFilePath, function(){
+ upstreamRequest(unify);
+ })
+ });
+ } catch(e) {
+ upstreamRequest(unify);
+ }
+ } else {
+ // we're good
+ unify.b.writeHead(200, {"Content-Length" : unify.fileSize})
+ callback();
+ }
});
+ });
} else {
+ console.log("file, '%s' exists but has no filesize meta data, assuming it was put here manually and servicing", unify.fullFilePath);
+ unify.b.writeHead(200, {"Content-Length" : unify.fileSize})
+ callback();
+ }
+ });
+}
+
+function makeCacheDir(path) {
+ console.log("attempting to create... '%s' as '%s'", path.fullPathDirName, path.subPathDirName);
+
+ var startAt = path.topFullPath;
+ var nextbits = path.subPathDirName.split("/");
+ for(var i=0; i < nextbits.length; i++) {
+ startAt += "/" + nextbits[i];
+ console.log("attempt mkdir on '%s'", startAt);
+ try {
+ fs.mkdirSync(startAt);
+ } catch(e) {
+ //console.log("e in mkdir, ", e);
+ }
+ }
+ //process.exit(0);
+}
+
+function serviceDirectory(unify) {
+ var nfiles = 0;
+ var res = unify.b;
+
+ res.write("Directory listing for " + unify.originalReq + "
");
+ if(unify.originalReq != "/") res.write("Parent\n\n");
+ fs.readdir(unify.fullFilePath, function(err, files) {
+ console.log("doing directory listing on: ", unify.fullFilePath);
+ if(err == null) {
+
+ // TODO: make this work asynchronously...
+ for(var i=0; i"+files[i]+"/\n");
+ nfiles++;
+ } else if(stats.isFile()) {
+ var padlength = 80 - (files[i].length) - stats.size.toString().length;
+ var padding = "";
+ if(padlength > 0) {
+ padding = new Array(padlength).join(" ");
+ }
+ res.write("File: "+files[i]+""+padding+stats.size+" bytes\n");
+ nfiles++;
+ }
+ } else {
+ console.log("ignoring file, ", files[i]);
+ }
+ }
+
+ if(nfiles == 0) res.write("Empty directory....\n");
- // TODO, we need to send this upstream, if its upstream we go up.
- res.writeHead(404, {"Content-Type": "text/plain"});
- res.write("404 Not Found\n");
+ res.write("
");
+ res.end();
+ } else {
+ res.write("we have entered bizaro world...\n");
+ res.write("");
res.end();
}
});
}
-exports.serviceFile = serviceFile;
+exports.serviceDirectory = serviceDirectory;
\ No newline at end of file