lib/cache.js

   1 var fs = require("fs");
   2 var http = require("http");
   3 var url = require("url");
   4 var path = require("path");
   5 var crypto = require("crypto");
   6 var log = require("./log.js");
   7
   8 function upstreamRequest(unify) {
   9         // first do a head request
  10         log.debug("upsteram as ", unify.requestFor);
  11
  12         var endData = false;
  13         var xpath = "";
  14         var filefd = null;
  15         if(unify.topPath !=null) if(unify.topPath != "") if(typeof global.repoproxy.repo[unify.topPath] != "undefined") {
  16                 var uplink = global.repoproxy.repo[unify.topPath].url;
  17                 xpath = uplink + unify.subPath;
  18         }
  19
  20         //unify.b.write("would send to '" + xpath + "'");
  21         //unify.b.end();
  22
  23         // not doing this properly yet...
  24         if(typeof global.repoproxy.downloads[unify.fullFilePath] != undefined && global.repoproxy.downloads[unify.fullFilePath] == 1) {
  25                 log.debug("request for file thats being downloaded already, doing inline request");
  26                 inlineService(unify);
  27                 return;
  28         }
  29
  30         log.debug("sending off to '%s'", xpath);
  31
  32         var headReq = url.parse(xpath);
  33         headReq["method"] = "HEAD";
  34
  35         getup = http.request(headReq, function(res) {
  36                 //res.setEncoding("utf8");
  37
  38                 if(!endData) {
  39                         log.debug("status code is ", typeof res.statusCode);
  40                         switch(res.statusCode) {
  41                         // TODO: this 301 directory redirect thing needs to work better
  42                         case 301:
  43                         case 302:
  44
  45                                 var loc = res.headers.location.substr(res.headers.location.length-4);
  46                                 var against_t = xpath + "/";
  47                                 var against = against_t.substr(against_t.length-4);
  48
  49                                 if(loc == against) {
  50                                         log.debug("got a redirect, upstream for loc => loc/ assuming its a directory");
  51                                         makeCacheDir(unify);
  52                                         unify.b.writeHead(302, { "Location": unify.originalReq + "/" });
  53                                 } else {
  54                                         log.debug("checked '%s' against '%s', was false, sending 404", loc, against);
  55                                         unify.b.writeHead(404, {"Content-Type": "text/plain"});
  56                                         unify.b.write("404 Not Found\n");
  57                                 }
  58                                 unify.b.end();
  59                                 endData = true;
  60                                 break;
  61
  62                         case 404:
  63                                 unify.b.writeHead(404, {"Content-Type": "text/plain"});
  64                                 unify.b.write("404 Not Found\n");
  65                                 unify.b.end();
  66                                 endData = true;
  67                                 break;
  68                         case 200:
  69                                 makeCacheDir(unify);
  70                                 if(unify.isDirectoryRequest) {
  71                                         serviceDirectory(unify);
  72                                         endData = true;
  73                                 } else {
  74                                         // this is where it gets ugly
  75                                         var filesize = res.headers["content-length"];
  76                                         log.debug("do ugly write: ", unify);
  77                                         //unify.b.write(data);
  78                                         var metafilename = unify.fullPathDirName + "/.meta."+ path.basename(unify.requestFor) +".filesize";
  79                                         var metafile = fs.createWriteStream(metafilename);
  80                                         metafile.write(filesize);
  81                                         metafile.end();
  82                                         getAndService(unify, xpath, filesize);
  83
  84                                 }
  85                                 break;
  86                         default:
  87                                 log.debug(".... data");
  88                                 //unify.b.write(data);
  89                         }
  90                 }
  91                 //log.debug("res is now ", res);
  92         });
  93
  94         getup.end();
  95
  96         //log.debug("getup: ", getup);
  97 }
  98
  99 exports.upstreamRequest = upstreamRequest;
 100
 101 function getAndService(unify, xpath, filesize) {
 102
 103         log.debug("calling in here with filesize, ", filesize)
 104         unify.b.writeHead(200, {'Content-Length' : filesize});
 105
 106
 107         global.repoproxy.downloads[unify.fullFilePath] = 1;
 108
 109
 110         http.get(xpath, function(res) {
 111
 112             var file = fs.createWriteStream(unify.fullFilePath);
 113
 114             //log.debug("res: ", res);
 115
 116             //res.setEncoding("utf8");
 117
 118             res.on("data", function(data) {
 119                     //log.debug("chunk");
 120                     file.write(data);
 121                     unify.b.write(data);
 122             });
 123
 124             res.on("end", function() {
 125                     log.debug("end...");
 126                     unify.b.end();
 127                     file.end();
 128                     global.repoproxy.downloads[unify.fullFilePath] = 0;
 129             });
 130
 131             res.on("error", function(err) {
 132                 log.debug("res threw error... ", err);
 133             });
 134         });
 135 }
 136
 137 // this is nasty nasty thing that can go horribly wrong in some ways, but currently works...
 138 function inlineService(unify) {
 139         // this method is called when we need to service a file thats being downloaded by something else
 140         var metafilename = unify.fullPathDirName + "/.meta."+ path.basename(unify.requestFor) +".filesize";
 141         var fsizef = fs.createReadStream(metafilename);
 142         var fsize = "";
 143         var lastchunk = 0;
 144         fsizef.on("data", function(data) {
 145                 fsize += data;
 146         });
 147
 148         fsizef.on("end", function() {
 149                 var sentSoFar = 0;
 150                 unify.b.writeHead(200, {"Content-Length" : fsize });
 151
 152                 // now we go into the file reading loop.
 153                 log.debug("start of inline services");
 154                 // we loop every 0.5s and do our thing
 155
 156                 function sendPieces() {
 157                         // this is going to be so fun i want to play real life frogger in real life traffic...
 158                         fs.stat(unify.fullFilePath, function(err, stats) {
 159                                 if(err == null) {
 160                                         if(stats["size"] > sentSoFar) {
 161                                                 // if file size changed between last chunk and this chunk, send the chunks
 162
 163                                                 lastChunk = 0;
 164                                                 // open the file, send the data
 165                                                 var rs = fs.createReadStream(unify.fullFilePath, {start: sentSoFar, end: stats["size"]});
 166
 167                                                 rs.on("data", function(thisdata) {
 168                                                         //log.debug("inline chunk: ", thisdata.length);
 169                                                         unify.b.write(thisdata);
 170                                                 });
 171
 172                                                 rs.on("end", function() {
 173                                                         sentSoFar = stats["size"];
 174                                                         // every second, we start again
 175                                                         if(sentSoFar != fsize) {
 176                                                                 setTimeout(sendPieces, 1000);
 177                                                         } else {
 178                                                                 // we're done!
 179                                                                 unify.b.end();
 180                                                         }
 181                                                 });
 182                                         } else {
 183                                                 // if file size did not change between last timeout and this one, incremement the chunk counter
 184                                                 // if we reach 60, we had a problem, and so we bomb out
 185
 186                                                 lastChunk++;
 187
 188                                                 // we bombed out somehow
 189                                                 if(lastChunk > 60) {
 190                                                         unify.b.end();
 191                                                 } else {
 192                                                         setTimeout(sendPieces, 1000);
 193                                                 }
 194                                         }
 195                                 } else {
 196                                         log.debug("inline service - we're in a very bad place");
 197                                 }
 198                         });
 199
 200                 }
 201
 202                 setTimeout(sendPieces, 100);
 203         });
 204 }
 205
 206 // the service file routine .... PLEASE KILL ME!
 207 function serviceFile(unify) {
 208
 209         // for now, ignore range.
 210         // however we need to check if a metadata file exists describing the filesize, check if its all correct
 211         // and if not, erase the file (and metafile) and forward the request back to upstream request
 212
 213
 214         checkFile(unify, function() {
 215
 216                 // file should already exist, so we just poop it out
 217                 var inp = fs.createReadStream(unify.fullFilePath);
 218                 //inp.setEncoding("utf8");
 219                 inp.on("data", function(data) {
 220                         unify.b.write(data);
 221                 });
 222
 223                 inp.on("end", function(closed) {
 224                         unify.b.end();
 225                 });
 226         });
 227 }
 228
 229 exports.serviceFile = serviceFile;
 230
 231
 232 function checkFile(unify, callback) {
 233         // in here we do the metadata checks
 234         var metafilename = unify.fullPathDirName + "/.meta."+ path.basename(unify.requestFor) +".filesize";
 235
 236         fs.exists(metafilename, function(existence) {
 237                 if(existence) {
 238                         var fsizef = fs.createReadStream(metafilename);
 239                         var fsize = "";
 240                         fsizef.on("data", function(data) {
 241                                 fsize += data;
 242                         });
 243
 244                         fsizef.on("end", function() {
 245                                 fs.stat(unify.fullFilePath, function(err, stats) {
 246                                         var rfsize = stats["size"];
 247                                         if(rfsize != fsize.trim()) {
 248                                                 // remove the file and start again
 249                                                 log.debug("reported filesizes dont match, '%s', '%s', removing file and starting again", rfsize, stats["size"]);
 250                                                 try {
 251                                                         fs.unlink(metafilename, function(){
 252                                                                 fs.unlink(unify.fullFilePath, function(){
 253                                                                         upstreamRequest(unify);
 254                                                                 })
 255                                                         });
 256                                                 } catch(e) {
 257                                                         upstreamRequest(unify);
 258                                                 }
 259                                         } else {
 260                                                 // we're good
 261                                                 unify.b.writeHead(200, {"Content-Length" : unify.fileSize})
 262                                                 callback();
 263                                         }
 264                                 });
 265                         });
 266                 } else {
 267                         log.debug("file, '%s' exists but has no filesize meta data, assuming it was put here manually and servicing", unify.fullFilePath);
 268                         unify.b.writeHead(200, {"Content-Length" : unify.fileSize})
 269                         callback();
 270                 }
 271         });
 272 }
 273
 274 function makeCacheDir(path) {
 275         log.debug("attempting to create... '%s' as '%s'", path.fullPathDirName, path.subPathDirName);
 276
 277         var startAt = path.topFullPath;
 278         var nextbits = path.subPathDirName.split("/");
 279         for(var i=0; i < nextbits.length; i++) {
 280                 startAt += "/" + nextbits[i];
 281                 log.debug("attempt mkdir on '%s'", startAt);
 282                 try {
 283                         fs.mkdirSync(startAt);
 284                 } catch(e) {
 285                         //log.debug("e in mkdir, ", e);
 286                 }
 287         }
 288         //process.exit(0);
 289 }
 290
 291 function serviceDirectory(unify) {
 292         var nfiles = 0;
 293         var res = unify.b;
 294
 295         res.write("<html><h1>Directory listing for " + unify.originalReq + "</h1><hr><pre>");
 296         if(unify.originalReq != "/") res.write("<a href=\"..\">Parent</a>\n\n");
 297         fs.readdir(unify.fullFilePath, function(err, files) {
 298                 log.debug("doing directory listing on: ", unify.fullFilePath);
 299                 if(err == null) {
 300
 301                         // TODO: make this work asynchronously...
 302                         for(var i=0; i<files.length; i++) {
 303                                 // avoiding statSync is too hard for now, will fix later TODO: fix this sync bit
 304                                 var stats = fs.statSync(unify.fullFilePath+"/"+files[i]);
 305
 306                                 if(files[i].match(/^\..*/) == null) {
 307                                         if(stats.isDirectory()) {
 308
 309                                                 res.write("Directory: <a href=\""+files[i]+"/\">"+files[i]+"/</a>\n");
 310                                                 nfiles++;
 311                                         } else if(stats.isFile()) {
 312                                                 var padlength = 80 - (files[i].length) - stats.size.toString().length;
 313                                                 var padding = "";
 314                                                 if(padlength > 0) {
 315                                                         padding = new Array(padlength).join(" ");
 316                                                 }
 317                                                 res.write("File:      <a href=\""+files[i]+"\">"+files[i]+"</a>"+padding+stats.size+" bytes\n");
 318                                                 nfiles++;
 319                                         }
 320                                 } else {
 321                                         log.debug("ignoring file, ", files[i]);
 322                                 }
 323                         }
 324
 325                         if(nfiles == 0) res.write("Empty directory....\n");
 326
 327                         res.write("<hr></pre>");
 328                         res.end();
 329                 } else {
 330                         res.write("we have entered bizaro world...\n");
 331                         res.write("</pre>");
 332                         res.end();
 333                 }
 334         });
 335 }
 336
 337 function moveToCleanup(file_or_dir) {
 338         // err..?
 339         var cleanup = global.repoproxy.cacheDir + "/.cleanup";
 340         var ctime = new Date().getTime();
 341         var encoded = (++global.repoproxy.fileid).toString();
 342         var toloc = cleanup + "/" + ctime.toString() + "." + encoded;
 343
 344         //log.debug("Moving %s to %s for cleanup", file_or_dir.replace(/\/$/, ""), toloc);
 345
 346         fs.renameSync(file_or_dir.replace(/\/$/, ""), toloc);
 347 }
 348
 349 function cleanupRoutine() {
 350
 351 }
 352
 353
 354 exports.serviceDirectory = serviceDirectory;
 355 exports.moveToCleanup = moveToCleanup;