1 var fs = require("fs");
2 var http = require("http");
3 var url = require("url");
4 var path = require("path");
5 var crypto = require("crypto");
6 var log = require("./log.js");
8 function upstreamRequest(unify) {
9 // first do a head request
10 log.debug("upsteram as ", unify.requestFor);
15 if(unify.topPath !=null) if(unify.topPath != "") if(typeof global.repoproxy.repo[unify.topPath] != "undefined") {
16 var uplink = global.repoproxy.repo[unify.topPath].url;
17 xpath = uplink + unify.subPath;
20 //unify.b.write("would send to '" + xpath + "'");
23 // not doing this properly yet...
24 if(typeof global.repoproxy.downloads[unify.fullFilePath] != undefined && global.repoproxy.downloads[unify.fullFilePath] == 1) {
25 log.debug("request for file thats being downloaded already, doing inline request");
30 log.debug("sending off to '%s'", xpath);
32 var headReq = url.parse(xpath);
33 headReq["method"] = "HEAD";
35 getup = http.request(headReq, function(res) {
36 //res.setEncoding("utf8");
39 log.debug("status code is ", typeof res.statusCode);
40 switch(res.statusCode) {
41 // TODO: this 301 directory redirect thing needs to work better
45 var loc = res.headers.location.substr(res.headers.location.length-4);
46 var against_t = xpath + "/";
47 var against = against_t.substr(against_t.length-4);
50 log.debug("got a redirect, upstream for loc => loc/ assuming its a directory");
52 unify.b.writeHead(302, { "Location": unify.originalReq + "/" });
54 log.debug("checked '%s' against '%s', was false, sending 404", loc, against);
55 unify.b.writeHead(404, {"Content-Type": "text/plain"});
56 unify.b.write("404 Not Found\n");
63 unify.b.writeHead(404, {"Content-Type": "text/plain"});
64 unify.b.write("404 Not Found\n");
70 if(unify.isDirectoryRequest) {
71 serviceDirectory(unify);
74 // this is where it gets ugly
75 var filesize = res.headers["content-length"];
76 log.debug("do ugly write: ", unify);
77 //unify.b.write(data);
78 var metafilename = unify.fullPathDirName + "/.meta."+ path.basename(unify.requestFor) +".filesize";
79 var metafile = fs.createWriteStream(metafilename);
80 metafile.write(filesize);
82 getAndService(unify, xpath, filesize);
87 log.debug(".... data");
88 //unify.b.write(data);
91 //log.debug("res is now ", res);
96 //log.debug("getup: ", getup);
99 exports.upstreamRequest = upstreamRequest;
101 function getAndService(unify, xpath, filesize) {
103 log.debug("calling in here with filesize, ", filesize)
104 unify.b.writeHead(200, {'Content-Length' : filesize});
106 global.repoproxy.downloads[unify.fullFilePath] = 1;
108 http.get(xpath, function(res) {
110 var file = fs.createWriteStream(unify.fullFilePath);
112 //log.debug("res: ", res);
114 //res.setEncoding("utf8");
116 res.on("data", function(data) {
117 //log.debug("chunk");
122 res.on("end", function() {
126 global.repoproxy.downloads[unify.fullFilePath] = 0;
129 res.on("error", function(err) {
130 log.debug("res threw error... ", err);
135 // this is nasty nasty thing that can go horribly wrong in some ways, but currently works...
136 function inlineService(unify) {
137 // this method is called when we need to service a file thats being downloaded by something else
138 var metafilename = unify.fullPathDirName + "/.meta."+ path.basename(unify.requestFor) +".filesize";
139 var fsizef = fs.createReadStream(metafilename);
142 fsizef.on("data", function(data) {
146 fsizef.on("end", function() {
148 unify.b.writeHead(200, {"Content-Length" : fsize });
150 // now we go into the file reading loop.
151 log.debug("start of inline services");
152 // we loop every 0.5s and do our thing
154 function sendPieces() {
155 // this is going to be so fun i want to play real life frogger in real life traffic...
156 fs.stat(unify.fullFilePath, function(err, stats) {
158 if(stats["size"] > sentSoFar) {
159 // if file size changed between last chunk and this chunk, send the chunks
162 // open the file, send the data
163 var rs = fs.createReadStream(unify.fullFilePath, {start: sentSoFar, end: stats["size"]});
165 rs.on("data", function(thisdata) {
166 //log.debug("inline chunk: ", thisdata.length);
167 unify.b.write(thisdata);
170 rs.on("end", function() {
171 sentSoFar = stats["size"];
172 // every second, we start again
173 if(sentSoFar != fsize) {
174 setTimeout(sendPieces, 1000);
181 // if file size did not change between last timeout and this one, incremement the chunk counter
182 // if we reach 60, we had a problem, and so we bomb out
186 // we bombed out somehow
190 setTimeout(sendPieces, 1000);
194 log.error("inline service - we're in a very bad place, how we ended up here we dont know, but we need to crash");
201 setTimeout(sendPieces, 100);
205 // the service file routine .... PLEASE KILL ME!
206 function serviceFile(unify) {
208 // for now, ignore range.
209 // however we need to check if a metadata file exists describing the filesize, check if its all correct
210 // and if not, erase the file (and metafile) and forward the request back to upstream request
213 checkFile(unify, function() {
215 // file should already exist, so we just poop it out
216 var inp = fs.createReadStream(unify.fullFilePath);
217 //inp.setEncoding("utf8");
218 inp.on("data", function(data) {
222 inp.on("end", function(closed) {
228 exports.serviceFile = serviceFile;
231 function checkFile(unify, callback) {
232 // in here we do the metadata checks
233 var metafilename = unify.fullPathDirName + "/.meta."+ path.basename(unify.requestFor) +".filesize";
235 fs.exists(metafilename, function(existence) {
237 var fsizef = fs.createReadStream(metafilename);
239 fsizef.on("data", function(data) {
243 fsizef.on("end", function() {
244 fs.stat(unify.fullFilePath, function(err, stats) {
245 var rfsize = stats["size"];
246 if(rfsize != fsize.trim()) {
247 // remove the file and start again
248 log.debug("reported filesizes dont match, '%s', '%s', removing file and starting again", rfsize, stats["size"]);
250 fs.unlink(metafilename, function(){
251 fs.unlink(unify.fullFilePath, function(){
252 upstreamRequest(unify);
256 upstreamRequest(unify);
260 unify.b.writeHead(200, {"Content-Length" : unify.fileSize})
266 log.debug("file, '%s' exists but has no filesize meta data, assuming it was put here manually and servicing", unify.fullFilePath);
267 unify.b.writeHead(200, {"Content-Length" : unify.fileSize})
273 function makeCacheDir(path) {
274 log.debug("attempting to create... '%s' as '%s'", path.fullPathDirName, path.subPathDirName);
276 var startAt = path.topFullPath;
277 var nextbits = path.subPathDirName.split("/");
278 for(var i=0; i < nextbits.length; i++) {
279 startAt += "/" + nextbits[i];
280 log.debug("attempt mkdir on '%s'", startAt);
282 fs.mkdirSync(startAt);
284 //log.debug("e in mkdir, ", e);
290 function serviceDirectory(unify) {
294 res.write("<html><h1>Directory listing for " + unify.originalReq + "</h1><hr><pre>");
295 if(unify.originalReq != "/") res.write("<a href=\"..\">Parent</a>\n\n");
296 fs.readdir(unify.fullFilePath, function(err, files) {
297 log.debug("doing directory listing on: ", unify.fullFilePath);
300 // TODO: make this work asynchronously...
301 for(var i=0; i<files.length; i++) {
302 // avoiding statSync is too hard for now, will fix later TODO: fix this sync bit
303 var stats = fs.statSync(unify.fullFilePath+"/"+files[i]);
305 if(files[i].match(/^\..*/) == null) {
306 if(stats.isDirectory()) {
308 res.write("Directory: <a href=\""+files[i]+"/\">"+files[i]+"/</a>\n");
310 } else if(stats.isFile()) {
311 var padlength = 80 - (files[i].length) - stats.size.toString().length;
314 padding = new Array(padlength).join(" ");
316 res.write("File: <a href=\""+files[i]+"\">"+files[i]+"</a>"+padding+stats.size+" bytes\n");
320 log.debug("ignoring file, ", files[i]);
324 if(nfiles == 0) res.write("Empty directory....\n");
326 res.write("<hr></pre>");
329 res.write("we have entered bizaro world...\n");
336 function moveToCleanup(file_or_dir) {
338 var cleanup = global.repoproxy.cacheDir + "/.cleanup";
339 var ctime = new Date().getTime();
340 var encoded = (++global.repoproxy.fileid).toString();
341 var toloc = cleanup + "/" + ctime.toString() + "." + encoded;
343 //log.debug("Moving %s to %s for cleanup", file_or_dir.replace(/\/$/, ""), toloc);
345 fs.renameSync(file_or_dir.replace(/\/$/, ""), toloc);
348 function cleanupRoutine() {
353 exports.serviceDirectory = serviceDirectory;
354 exports.moveToCleanup = moveToCleanup;