lib/cache.js

   1 var fs = require("fs");
   2 var http = require("http");
   3 var url = require("url");
   4 var path = require("path");
   5
   6 function maintainCache() {
   7         // TODO i should check that im already running here and exit if i am
   8         console.log("Cache maintainence routine starting...");
   9         console.log("Cache maintainence routine ended...");
  10 }
  11
  12 exports.startTimer = function() {
  13         // our once-a-day cache maintainer
  14         var cacheTimer = global.repoproxy.scancache*3600*1000;
  15         //var cacheTimer = global.repoproxy.scancache*100;
  16         setInterval(maintainCache, cacheTimer);
  17 }
  18
  19 function upstreamRequest(unify) {
  20         // first do a head request
  21         console.log("upsteram as ", unify.requestFor);
  22
  23         var endData = false;
  24         var xpath = "";
  25         var filefd = null;
  26         if(unify.topPath !=null) if(unify.topPath != "") if(typeof global.repoproxy.repo[unify.topPath] != "undefined") {
  27                 var uplink = global.repoproxy.repo[unify.topPath].url;
  28                 xpath = uplink + unify.subPath;
  29         }
  30
  31         //unify.b.write("would send to '" + xpath + "'");
  32         //unify.b.end();
  33
  34         // not doing this properly yet...
  35         if(typeof global.repoproxy.downloads[unify.fullFilePath] != undefined && global.repoproxy.downloads[unify.fullFilePath] == 1) {
  36                 console.log("request for file thats being downloaded already, doing inline request");
  37                 inlineService(unify);
  38                 return;
  39         }
  40
  41         console.log("sending off to '%s'", xpath);
  42
  43         var headReq = url.parse(xpath);
  44         headReq["method"] = "HEAD";
  45
  46         getup = http.request(headReq, function(res) {
  47                 //res.setEncoding("utf8");
  48
  49                 if(!endData) {
  50                         console.log("status code is ", typeof res.statusCode);
  51                         switch(res.statusCode) {
  52                         // TODO: this 301 directory redirect thing needs to work better
  53                         case 301:
  54                         case 302:
  55
  56                                 var loc = res.headers.location.substr(res.headers.location.length-4);
  57                                 var against_t = xpath + "/";
  58                                 var against = against_t.substr(against_t.length-4);
  59
  60                                 if(loc == against) {
  61                                         console.log("got a redirect, upstream for loc => loc/ assuming its a directory");
  62                                         makeCacheDir(unify);
  63                                         unify.b.writeHead(302, { "Location": unify.originalReq + "/" });
  64                                 } else {
  65                                         console.log("checked '%s' against '%s', was false, sending 404", loc, against);
  66                                         unify.b.writeHead(404, {"Content-Type": "text/plain"});
  67                                         unify.b.write("404 Not Found\n");
  68                                 }
  69                                 unify.b.end();
  70                                 endData = true;
  71                                 break;
  72
  73                         case 404:
  74                                 unify.b.writeHead(404, {"Content-Type": "text/plain"});
  75                                 unify.b.write("404 Not Found\n");
  76                                 unify.b.end();
  77                                 endData = true;
  78                                 break;
  79                         case 200:
  80                                 makeCacheDir(unify);
  81                                 if(unify.isDirectoryRequest) {
  82                                         serviceDirectory(unify);
  83                                         endData = true;
  84                                 } else {
  85                                         // this is where it gets ugly
  86                                         var filesize = res.headers["content-length"];
  87                                         console.log("do ugly write: ", unify);
  88                                         //unify.b.write(data);
  89                                         var metafilename = unify.fullPathDirName + "/.meta."+ path.basename(unify.requestFor) +".filesize";
  90                                         var metafile = fs.createWriteStream(metafilename);
  91                                         metafile.write(filesize);
  92                                         metafile.end();
  93                                         getAndService(unify, xpath, filesize);
  94
  95                                 }
  96                                 break;
  97                         default:
  98                                 console.log(".... data");
  99                                 //unify.b.write(data);
 100                         }
 101                 }
 102                 //console.log("res is now ", res);
 103         });
 104
 105         getup.end();
 106
 107         //console.log("getup: ", getup);
 108 }
 109
 110 exports.upstreamRequest = upstreamRequest;
 111
 112 function getAndService(unify, xpath, filesize) {
 113
 114         console.log("calling in here with filesize, ", filesize)
 115         unify.b.writeHead(200, {'Content-Length' : filesize});
 116
 117
 118         global.repoproxy.downloads[unify.fullFilePath] = 1;
 119
 120
 121         http.get(xpath, function(res) {
 122
 123             var file = fs.createWriteStream(unify.fullFilePath);
 124
 125             //console.log("res: ", res);
 126
 127             //res.setEncoding("utf8");
 128
 129             res.on("data", function(data) {
 130                     //console.log("chunk");
 131                     file.write(data);
 132                     unify.b.write(data);
 133             });
 134
 135             res.on("end", function() {
 136                     console.log("end...");
 137                     unify.b.end();
 138                     file.end();
 139                     global.repoproxy.downloads[unify.fullFilePath] = 0;
 140             });
 141
 142             res.on("error", function(err) {
 143                 console.log("res threw error... ", err);
 144             });
 145         });
 146 }
 147
 148 // this is nasty nasty thing that can go horribly wrong in some ways, but currently works...
 149 function inlineService(unify) {
 150         // this method is called when we need to service a file thats being downloaded by something else
 151         var metafilename = unify.fullPathDirName + "/.meta."+ path.basename(unify.requestFor) +".filesize";
 152         var fsizef = fs.createReadStream(metafilename);
 153         var fsize = "";
 154         var lastchunk = 0;
 155         fsizef.on("data", function(data) {
 156                 fsize += data;
 157         });
 158
 159         fsizef.on("end", function() {
 160                 var sentSoFar = 0;
 161                 unify.b.writeHead(200, {"Content-Length" : fsize });
 162
 163                 // now we go into the file reading loop.
 164                 console.log("start of inline services");
 165                 // we loop every 0.5s and do our thing
 166
 167                 function sendPieces() {
 168                         // this is going to be so fun i want to play real life frogger in real life traffic...
 169                         fs.stat(unify.fullFilePath, function(err, stats) {
 170                                 if(err == null) {
 171                                         if(stats["size"] > sentSoFar) {
 172                                                 // if file size changed between last chunk and this chunk, send the chunks
 173
 174                                                 lastChunk = 0;
 175                                                 // open the file, send the data
 176                                                 var rs = fs.createReadStream(unify.fullFilePath, {start: sentSoFar, end: stats["size"]});
 177
 178                                                 rs.on("data", function(thisdata) {
 179                                                         //console.log("inline chunk: ", thisdata.length);
 180                                                         unify.b.write(thisdata);
 181                                                 });
 182
 183                                                 rs.on("end", function() {
 184                                                         sentSoFar = stats["size"];
 185                                                         // every second, we start again
 186                                                         if(sentSoFar != fsize) {
 187                                                                 setTimeout(sendPieces, 1000);
 188                                                         } else {
 189                                                                 // we're done!
 190                                                                 unify.b.end();
 191                                                         }
 192                                                 });
 193                                         } else {
 194                                                 // if file size did not change between last timeout and this one, incremement the chunk counter
 195                                                 // if we reach 60, we had a problem, and so we bomb out
 196
 197                                                 lastChunk++;
 198
 199                                                 // we bombed out somehow
 200                                                 if(lastChunk > 60) {
 201                                                         unify.b.end();
 202                                                 } else {
 203                                                         setTimeout(sendPieces, 1000);
 204                                                 }
 205                                         }
 206                                 } else {
 207                                         console.log("inline service - we're in a very bad place");
 208                                 }
 209                         });
 210
 211                 }
 212
 213                 setTimeout(sendPieces, 100);
 214         });
 215 }
 216
 217 // the service file routine .... PLEASE KILL ME!
 218 function serviceFile(unify) {
 219
 220         // for now, ignore range.
 221         // however we need to check if a metadata file exists describing the filesize, check if its all correct
 222         // and if not, erase the file (and metafile) and forward the request back to upstream request
 223
 224
 225         checkFile(unify, function() {
 226
 227                 // file should already exist, so we just poop it out
 228                 var inp = fs.createReadStream(unify.fullFilePath);
 229                 //inp.setEncoding("utf8");
 230                 inp.on("data", function(data) {
 231                         unify.b.write(data);
 232                 });
 233
 234                 inp.on("end", function(closed) {
 235                         unify.b.end();
 236                 });
 237         });
 238 }
 239
 240 exports.serviceFile = serviceFile;
 241
 242
 243 function checkFile(unify, callback) {
 244         // in here we do the metadata checks
 245         var metafilename = unify.fullPathDirName + "/.meta."+ path.basename(unify.requestFor) +".filesize";
 246
 247         fs.exists(metafilename, function(existence) {
 248                 if(existence) {
 249                         var fsizef = fs.createReadStream(metafilename);
 250                         var fsize = "";
 251                         fsizef.on("data", function(data) {
 252                                 fsize += data;
 253                         });
 254
 255                         fsizef.on("end", function() {
 256                                 fs.stat(unify.fullFilePath, function(err, stats) {
 257                                         var rfsize = stats["size"];
 258                                         if(rfsize != fsize.trim()) {
 259                                                 // remove the file and start again
 260                                                 console.log("reported filesizes dont match, '%s', '%s', removing file and starting again", rfsize, stats["size"]);
 261                                                 try {
 262                                                         fs.unlink(metafilename, function(){
 263                                                                 fs.unlink(unify.fullFilePath, function(){
 264                                                                         upstreamRequest(unify);
 265                                                                 })
 266                                                         });
 267                                                 } catch(e) {
 268                                                         upstreamRequest(unify);
 269                                                 }
 270                                         } else {
 271                                                 // we're good
 272                                                 unify.b.writeHead(200, {"Content-Length" : unify.fileSize})
 273                                                 callback();
 274                                         }
 275                                 });
 276                         });
 277                 } else {
 278                         console.log("file, '%s' exists but has no filesize meta data, assuming it was put here manually and servicing", unify.fullFilePath);
 279                         unify.b.writeHead(200, {"Content-Length" : unify.fileSize})
 280                         callback();
 281                 }
 282         });
 283 }
 284
 285 function makeCacheDir(path) {
 286         console.log("attempting to create... '%s' as '%s'", path.fullPathDirName, path.subPathDirName);
 287
 288         var startAt = path.topFullPath;
 289         var nextbits = path.subPathDirName.split("/");
 290         for(var i=0; i < nextbits.length; i++) {
 291                 startAt += "/" + nextbits[i];
 292                 console.log("attempt mkdir on '%s'", startAt);
 293                 try {
 294                         fs.mkdirSync(startAt);
 295                 } catch(e) {
 296                         //console.log("e in mkdir, ", e);
 297                 }
 298         }
 299         //process.exit(0);
 300 }
 301
 302 function serviceDirectory(unify) {
 303         var nfiles = 0;
 304         var res = unify.b;
 305
 306         res.write("<html><h1>Directory listing for " + unify.originalReq + "</h1><hr><pre>");
 307         if(unify.originalReq != "/") res.write("<a href=\"..\">Parent</a>\n\n");
 308         fs.readdir(unify.fullFilePath, function(err, files) {
 309                 console.log("doing directory listing on: ", unify.fullFilePath);
 310                 if(err == null) {
 311
 312                         // TODO: make this work asynchronously...
 313                         for(var i=0; i<files.length; i++) {
 314                                 // avoiding statSync is too hard for now, will fix later TODO: fix this sync bit
 315                                 var stats = fs.statSync(unify.fullFilePath+"/"+files[i]);
 316
 317                                 if(files[i].match(/^\..*/) == null) {
 318                                         if(stats.isDirectory()) {
 319
 320                                                 res.write("Directory: <a href=\""+files[i]+"/\">"+files[i]+"/</a>\n");
 321                                                 nfiles++;
 322                                         } else if(stats.isFile()) {
 323                                                 var padlength = 80 - (files[i].length) - stats.size.toString().length;
 324                                                 var padding = "";
 325                                                 if(padlength > 0) {
 326                                                         padding = new Array(padlength).join(" ");
 327                                                 }
 328                                                 res.write("File:      <a href=\""+files[i]+"\">"+files[i]+"</a>"+padding+stats.size+" bytes\n");
 329                                                 nfiles++;
 330                                         }
 331                                 } else {
 332                                         console.log("ignoring file, ", files[i]);
 333                                 }
 334                         }
 335
 336                         if(nfiles == 0) res.write("Empty directory....\n");
 337
 338                         res.write("<hr></pre>");
 339                         res.end();
 340                 } else {
 341                         res.write("we have entered bizaro world...\n");
 342                         res.write("</pre>");
 343                         res.end();
 344                 }
 345         });
 346 }
 347
 348 exports.serviceDirectory = serviceDirectory;