lib/cache.js

   1 var fs = require("fs");
   2 var http = require("http");
   3 var url = require("url");
   4 var path = require("path");
   5
   6 function upstreamRequest(unify) {
   7         // first do a head request
   8         console.log("upsteram as ", unify.requestFor);
   9
  10         var endData = false;
  11         var xpath = "";
  12         var filefd = null;
  13         if(unify.topPath !=null) if(unify.topPath != "") if(typeof global.repoproxy.repo[unify.topPath] != "undefined") {
  14                 var uplink = global.repoproxy.repo[unify.topPath].url;
  15                 xpath = uplink + unify.subPath;
  16         }
  17
  18         //unify.b.write("would send to '" + xpath + "'");
  19         //unify.b.end();
  20
  21         // not doing this properly yet...
  22         if(typeof global.repoproxy.downloads[unify.fullFilePath] != undefined && global.repoproxy.downloads[unify.fullFilePath] == 1) {
  23                 console.log("request for file thats being downloaded already, doing inline request");
  24                 inlineService(unify);
  25                 return;
  26         }
  27
  28         console.log("sending off to '%s'", xpath);
  29
  30         var headReq = url.parse(xpath);
  31         headReq["method"] = "HEAD";
  32
  33         getup = http.request(headReq, function(res) {
  34                 //res.setEncoding("utf8");
  35
  36                 if(!endData) {
  37                         console.log("status code is ", typeof res.statusCode);
  38                         switch(res.statusCode) {
  39                         // TODO: this 301 directory redirect thing needs to work better
  40                         case 301:
  41                         case 302:
  42
  43                                 var loc = res.headers.location.substr(res.headers.location.length-4);
  44                                 var against_t = xpath + "/";
  45                                 var against = against_t.substr(against_t.length-4);
  46
  47                                 if(loc == against) {
  48                                         console.log("got a redirect, upstream for loc => loc/ assuming its a directory");
  49                                         makeCacheDir(unify);
  50                                         unify.b.writeHead(302, { "Location": unify.originalReq + "/" });
  51                                 } else {
  52                                         console.log("checked '%s' against '%s', was false, sending 404", loc, against);
  53                                         unify.b.writeHead(404, {"Content-Type": "text/plain"});
  54                                         unify.b.write("404 Not Found\n");
  55                                 }
  56                                 unify.b.end();
  57                                 endData = true;
  58                                 break;
  59
  60                         case 404:
  61                                 unify.b.writeHead(404, {"Content-Type": "text/plain"});
  62                                 unify.b.write("404 Not Found\n");
  63                                 unify.b.end();
  64                                 endData = true;
  65                                 break;
  66                         case 200:
  67                                 makeCacheDir(unify);
  68                                 if(unify.isDirectoryRequest) {
  69                                         serviceDirectory(unify);
  70                                         endData = true;
  71                                 } else {
  72                                         // this is where it gets ugly
  73                                         var filesize = res.headers["content-length"];
  74                                         console.log("do ugly write: ", unify);
  75                                         //unify.b.write(data);
  76                                         var metafilename = unify.fullPathDirName + "/.meta."+ path.basename(unify.requestFor) +".filesize";
  77                                         var metafile = fs.createWriteStream(metafilename);
  78                                         metafile.write(filesize);
  79                                         metafile.end();
  80                                         getAndService(unify, xpath, filesize);
  81
  82                                 }
  83                                 break;
  84                         default:
  85                                 console.log(".... data");
  86                                 //unify.b.write(data);
  87                         }
  88                 }
  89                 //console.log("res is now ", res);
  90         });
  91
  92         getup.end();
  93
  94         //console.log("getup: ", getup);
  95 }
  96
  97 exports.upstreamRequest = upstreamRequest;
  98
  99 function getAndService(unify, xpath, filesize) {
 100
 101         console.log("calling in here with filesize, ", filesize)
 102         unify.b.writeHead(200, {'Content-Length' : filesize});
 103
 104
 105         global.repoproxy.downloads[unify.fullFilePath] = 1;
 106
 107
 108         http.get(xpath, function(res) {
 109
 110             var file = fs.createWriteStream(unify.fullFilePath);
 111
 112             //console.log("res: ", res);
 113
 114             //res.setEncoding("utf8");
 115
 116             res.on("data", function(data) {
 117                     //console.log("chunk");
 118                     file.write(data);
 119                     unify.b.write(data);
 120             });
 121
 122             res.on("end", function() {
 123                     console.log("end...");
 124                     unify.b.end();
 125                     file.end();
 126                     global.repoproxy.downloads[unify.fullFilePath] = 0;
 127             });
 128
 129             res.on("error", function(err) {
 130                 console.log("res threw error... ", err);
 131             });
 132         });
 133 }
 134
 135 // this is nasty nasty thing that can go horribly wrong in some ways, but currently works...
 136 function inlineService(unify) {
 137         // this method is called when we need to service a file thats being downloaded by something else
 138         var metafilename = unify.fullPathDirName + "/.meta."+ path.basename(unify.requestFor) +".filesize";
 139         var fsizef = fs.createReadStream(metafilename);
 140         var fsize = "";
 141         var lastchunk = 0;
 142         fsizef.on("data", function(data) {
 143                 fsize += data;
 144         });
 145
 146         fsizef.on("end", function() {
 147                 var sentSoFar = 0;
 148                 unify.b.writeHead(200, {"Content-Length" : fsize });
 149
 150                 // now we go into the file reading loop.
 151                 console.log("start of inline services");
 152                 // we loop every 0.5s and do our thing
 153
 154                 function sendPieces() {
 155                         // this is going to be so fun i want to play real life frogger in real life traffic...
 156                         fs.stat(unify.fullFilePath, function(err, stats) {
 157                                 if(err == null) {
 158                                         if(stats["size"] > sentSoFar) {
 159                                                 // if file size changed between last chunk and this chunk, send the chunks
 160
 161                                                 lastChunk = 0;
 162                                                 // open the file, send the data
 163                                                 var rs = fs.createReadStream(unify.fullFilePath, {start: sentSoFar, end: stats["size"]});
 164
 165                                                 rs.on("data", function(thisdata) {
 166                                                         //console.log("inline chunk: ", thisdata.length);
 167                                                         unify.b.write(thisdata);
 168                                                 });
 169
 170                                                 rs.on("end", function() {
 171                                                         sentSoFar = stats["size"];
 172                                                         // every second, we start again
 173                                                         if(sentSoFar != fsize) {
 174                                                                 setTimeout(sendPieces, 1000);
 175                                                         } else {
 176                                                                 // we're done!
 177                                                                 unify.b.end();
 178                                                         }
 179                                                 });
 180                                         } else {
 181                                                 // if file size did not change between last timeout and this one, incremement the chunk counter
 182                                                 // if we reach 60, we had a problem, and so we bomb out
 183
 184                                                 lastChunk++;
 185
 186                                                 // we bombed out somehow
 187                                                 if(lastChunk > 60) {
 188                                                         unify.b.end();
 189                                                 } else {
 190                                                         setTimeout(sendPieces, 1000);
 191                                                 }
 192                                         }
 193                                 } else {
 194                                         console.log("inline service - we're in a very bad place");
 195                                 }
 196                         });
 197
 198                 }
 199
 200                 setTimeout(sendPieces, 100);
 201         });
 202 }
 203
 204 // the service file routine .... PLEASE KILL ME!
 205 function serviceFile(unify) {
 206
 207         // for now, ignore range.
 208         // however we need to check if a metadata file exists describing the filesize, check if its all correct
 209         // and if not, erase the file (and metafile) and forward the request back to upstream request
 210
 211
 212         checkFile(unify, function() {
 213
 214                 // file should already exist, so we just poop it out
 215                 var inp = fs.createReadStream(unify.fullFilePath);
 216                 //inp.setEncoding("utf8");
 217                 inp.on("data", function(data) {
 218                         unify.b.write(data);
 219                 });
 220
 221                 inp.on("end", function(closed) {
 222                         unify.b.end();
 223                 });
 224         });
 225 }
 226
 227 exports.serviceFile = serviceFile;
 228
 229
 230 function checkFile(unify, callback) {
 231         // in here we do the metadata checks
 232         var metafilename = unify.fullPathDirName + "/.meta."+ path.basename(unify.requestFor) +".filesize";
 233
 234         fs.exists(metafilename, function(existence) {
 235                 if(existence) {
 236                         var fsizef = fs.createReadStream(metafilename);
 237                         var fsize = "";
 238                         fsizef.on("data", function(data) {
 239                                 fsize += data;
 240                         });
 241
 242                         fsizef.on("end", function() {
 243                                 fs.stat(unify.fullFilePath, function(err, stats) {
 244                                         var rfsize = stats["size"];
 245                                         if(rfsize != fsize.trim()) {
 246                                                 // remove the file and start again
 247                                                 console.log("reported filesizes dont match, '%s', '%s', removing file and starting again", rfsize, stats["size"]);
 248                                                 try {
 249                                                         fs.unlink(metafilename, function(){
 250                                                                 fs.unlink(unify.fullFilePath, function(){
 251                                                                         upstreamRequest(unify);
 252                                                                 })
 253                                                         });
 254                                                 } catch(e) {
 255                                                         upstreamRequest(unify);
 256                                                 }
 257                                         } else {
 258                                                 // we're good
 259                                                 unify.b.writeHead(200, {"Content-Length" : unify.fileSize})
 260                                                 callback();
 261                                         }
 262                                 });
 263                         });
 264                 } else {
 265                         console.log("file, '%s' exists but has no filesize meta data, assuming it was put here manually and servicing", unify.fullFilePath);
 266                         unify.b.writeHead(200, {"Content-Length" : unify.fileSize})
 267                         callback();
 268                 }
 269         });
 270 }
 271
 272 function makeCacheDir(path) {
 273         console.log("attempting to create... '%s' as '%s'", path.fullPathDirName, path.subPathDirName);
 274
 275         var startAt = path.topFullPath;
 276         var nextbits = path.subPathDirName.split("/");
 277         for(var i=0; i < nextbits.length; i++) {
 278                 startAt += "/" + nextbits[i];
 279                 console.log("attempt mkdir on '%s'", startAt);
 280                 try {
 281                         fs.mkdirSync(startAt);
 282                 } catch(e) {
 283                         //console.log("e in mkdir, ", e);
 284                 }
 285         }
 286         //process.exit(0);
 287 }
 288
 289 function serviceDirectory(unify) {
 290         var nfiles = 0;
 291         var res = unify.b;
 292
 293         res.write("<html><h1>Directory listing for " + unify.originalReq + "</h1><hr><pre>");
 294         if(unify.originalReq != "/") res.write("<a href=\"..\">Parent</a>\n\n");
 295         fs.readdir(unify.fullFilePath, function(err, files) {
 296                 console.log("doing directory listing on: ", unify.fullFilePath);
 297                 if(err == null) {
 298
 299                         // TODO: make this work asynchronously...
 300                         for(var i=0; i<files.length; i++) {
 301                                 // avoiding statSync is too hard for now, will fix later TODO: fix this sync bit
 302                                 var stats = fs.statSync(unify.fullFilePath+"/"+files[i]);
 303
 304                                 if(files[i].match(/^\..*/) == null) {
 305                                         if(stats.isDirectory()) {
 306
 307                                                 res.write("Directory: <a href=\""+files[i]+"/\">"+files[i]+"/</a>\n");
 308                                                 nfiles++;
 309                                         } else if(stats.isFile()) {
 310                                                 var padlength = 80 - (files[i].length) - stats.size.toString().length;
 311                                                 var padding = "";
 312                                                 if(padlength > 0) {
 313                                                         padding = new Array(padlength).join(" ");
 314                                                 }
 315                                                 res.write("File:      <a href=\""+files[i]+"\">"+files[i]+"</a>"+padding+stats.size+" bytes\n");
 316                                                 nfiles++;
 317                                         }
 318                                 } else {
 319                                         console.log("ignoring file, ", files[i]);
 320                                 }
 321                         }
 322
 323                         if(nfiles == 0) res.write("Empty directory....\n");
 324
 325                         res.write("<hr></pre>");
 326                         res.end();
 327                 } else {
 328                         res.write("we have entered bizaro world...\n");
 329                         res.write("</pre>");
 330                         res.end();
 331                 }
 332         });
 333 }
 334
 335 exports.serviceDirectory = serviceDirectory;