lib/cache.js

   1 var fs = require("fs");
   2 var http = require("http");
   3 var url = require("url");
   4 var path = require("path");
   5
   6 function maintainCache() {
   7         // TODO i should check that im already running here and exit if i am
   8         console.log("Cache maintainence routine starting...");
   9         console.log("Cache maintainence routine ended...");
  10 }
  11
  12 exports.startTimer = function() {
  13         // our once-a-day cache maintainer
  14         var cacheTimer = global.repoproxy.scancache*3600*1000;
  15         //var cacheTimer = global.repoproxy.scancache*100;
  16         setInterval(maintainCache, cacheTimer);
  17 }
  18
  19 function upstreamRequest(unify) {
  20         // first do a head request
  21         console.log("upsteram as ", unify.requestFor);
  22
  23         var endData = false;
  24         var xpath = "";
  25         var filefd = null;
  26         if(unify.topPath !=null) if(unify.topPath != "") if(typeof global.repoproxy.repo[unify.topPath] != "undefined") {
  27                 var uplink = global.repoproxy.repo[unify.topPath].url;
  28                 xpath = uplink + unify.subPath;
  29         }
  30
  31         //unify.b.write("would send to '" + xpath + "'");
  32         //unify.b.end();
  33
  34         console.log("sending off to '%s'", xpath);
  35
  36         var headReq = url.parse(xpath);
  37         headReq["method"] = "HEAD";
  38
  39         getup = http.request(headReq, function(res) {
  40                 //res.setEncoding("utf8");
  41
  42                 if(!endData) {
  43                         console.log("status code is ", typeof res.statusCode);
  44                         switch(res.statusCode) {
  45                         // TODO: this 301 directory redirect thing needs to work better
  46                         case 301:
  47                         case 302:
  48
  49                                 var loc = res.headers.location.substr(res.headers.location.length-4);
  50                                 var against_t = xpath + "/";
  51                                 var against = against_t.substr(against_t.length-4);
  52
  53                                 if(loc == against) {
  54                                         console.log("got a redirect, upstream for loc => loc/ assuming its a directory");
  55                                         makeCacheDir(unify);
  56                                         unify.b.writeHead(302, { "Location": unify.originalReq + "/" });
  57                                 } else {
  58                                         console.log("checked '%s' against '%s', was false, sending 404", loc, against);
  59                                         unify.b.writeHead(404, {"Content-Type": "text/plain"});
  60                                         unify.b.write("404 Not Found\n");
  61                                 }
  62                                 unify.b.end();
  63                                 endData = true;
  64                                 break;
  65
  66                         case 404:
  67                                 unify.b.writeHead(404, {"Content-Type": "text/plain"});
  68                                 unify.b.write("404 Not Found\n");
  69                                 unify.b.end();
  70                                 endData = true;
  71                                 break;
  72                         case 200:
  73                                 makeCacheDir(unify);
  74                                 if(unify.isDirectoryRequest) {
  75                                         serviceDirectory(unify);
  76                                         endData = true;
  77                                 } else {
  78                                         // this is where it gets ugly
  79                                         var filesize = res.headers["content-length"];
  80                                         console.log("do ugly write: ", unify);
  81                                         //unify.b.write(data);
  82                                         var metafilename = unify.fullPathDirName + "/.meta."+ path.basename(unify.requestFor) +".filesize";
  83                                         var metafile = fs.createWriteStream(metafilename);
  84                                         metafile.write(filesize);
  85                                         metafile.end();
  86                                         getAndService(unify, xpath, filesize);
  87
  88                                 }
  89                                 break;
  90                         default:
  91                                 console.log(".... data");
  92                                 //unify.b.write(data);
  93                         }
  94                 }
  95                 //console.log("res is now ", res);
  96         });
  97
  98         getup.end();
  99
 100         //console.log("getup: ", getup);
 101 }
 102
 103 exports.upstreamRequest = upstreamRequest;
 104
 105 function getAndService(unify, xpath, filesize) {
 106
 107         console.log("calling in here with filesize, ", filesize)
 108         unify.b.writeHead(200, {'Content-Length' : filesize});
 109
 110
 111         if(typeof global.repoproxy.downloads[unify.fullFilePath] != "undefined" && global.repoproxy.downloads[unify.fullFilePath] == 1) {
 112
 113                 console.log("service inline");
 114                 unify.b.write("trying to service inline");
 115                 unify.b.end();
 116         } else {
 117                 global.repoproxy.downloads[unify.fullFilePath] = 1;
 118
 119
 120                 http.get(xpath, function(res) {
 121
 122                     var file = fs.createWriteStream(unify.fullFilePath);
 123
 124                     //console.log("res: ", res);
 125
 126                     //res.setEncoding("utf8");
 127
 128                     res.on("data", function(data) {
 129                             //console.log("chunk");
 130                             file.write(data);
 131                             unify.b.write(data);
 132                     });
 133
 134                     res.on("end", function() {
 135                             console.log("end...");
 136                             unify.b.end();
 137                             file.end();
 138                             global.repoproxy.downloads[unify.fullFilePath] = 0;
 139                     });
 140
 141                     res.on("error", function(err) {
 142                         console.log("res threw error... ", err);
 143                     });
 144                 });
 145         }
 146 }
 147
 148 // the service file routine .... PLEASE KILL ME!
 149 function serviceFile(unify) {
 150
 151         // for now, ignore range.
 152         // however we need to check if a metadata file exists describing the filesize, check if its all correct
 153         // and if not, erase the file (and metafile) and forward the request back to upstream request
 154
 155
 156         checkFile(unify, function() {
 157
 158                 // file should already exist, so we just poop it out
 159                 var inp = fs.createReadStream(unify.fullFilePath);
 160                 //inp.setEncoding("utf8");
 161                 inp.on("data", function(data) {
 162                         unify.b.write(data);
 163                 });
 164
 165                 inp.on("end", function(closed) {
 166                         unify.b.end();
 167                 });
 168         });
 169 }
 170
 171 exports.serviceFile = serviceFile;
 172
 173
 174 function checkFile(unify, callback) {
 175         // in here we do the metadata checks
 176         var metafilename = unify.fullPathDirName + "/.meta."+ path.basename(unify.requestFor) +".filesize";
 177
 178         fs.exists(metafilename, function(existence) {
 179                 if(existence) {
 180                         var fsizef = fs.createReadStream(metafilename);
 181                         var fsize = "";
 182                         fsizef.on("data", function(data) {
 183                                 fsize += data;
 184                         });
 185
 186                         fsizef.on("end", function() {
 187                                 fs.stat(unify.fullFilePath, function(err, stats) {
 188                                         var rfsize = stats["size"];
 189                                         if(rfsize != fsize.trim()) {
 190                                                 // remove the file and start again
 191                                                 console.log("reported filesizes dont match, '%s', '%s', removing file and starting again", rfsize, stats["size"]);
 192                                                 try {
 193                                                         fs.unlink(metafilename, function(){
 194                                                                 fs.unlink(unify.fullFilePath, function(){
 195                                                                         upstreamRequest(unify);
 196                                                                 })
 197                                                         });
 198                                                 } catch(e) {
 199                                                         upstreamRequest(unify);
 200                                                 }
 201                                         } else {
 202                                                 // we're good
 203                                                 unify.b.writeHead(200, {"Content-Length" : unify.fileSize})
 204                                                 callback();
 205                                         }
 206                                 });
 207                         });
 208                 } else {
 209                         console.log("file, '%s' exists but has no filesize meta data, assuming it was put here manually and servicing", unify.fullFilePath);
 210                         unify.b.writeHead(200, {"Content-Length" : unify.fileSize})
 211                         callback();
 212                 }
 213         });
 214 }
 215
 216 function makeCacheDir(path) {
 217         console.log("attempting to create... '%s' as '%s'", path.fullPathDirName, path.subPathDirName);
 218
 219         var startAt = path.topFullPath;
 220         var nextbits = path.subPathDirName.split("/");
 221         for(var i=0; i < nextbits.length; i++) {
 222                 startAt += "/" + nextbits[i];
 223                 console.log("attempt mkdir on '%s'", startAt);
 224                 try {
 225                         fs.mkdirSync(startAt);
 226                 } catch(e) {
 227                         //console.log("e in mkdir, ", e);
 228                 }
 229         }
 230         //process.exit(0);
 231 }
 232
 233 function serviceDirectory(unify) {
 234         var nfiles = 0;
 235         var res = unify.b;
 236
 237         res.write("<html><h1>Directory listing for " + unify.originalReq + "</h1><hr><pre>");
 238         if(unify.originalReq != "/") res.write("<a href=\"..\">Parent</a>\n\n");
 239         fs.readdir(unify.fullFilePath, function(err, files) {
 240                 console.log("doing directory listing on: ", unify.fullFilePath);
 241                 if(err == null) {
 242
 243                         // TODO: make this work asynchronously...
 244                         for(var i=0; i<files.length; i++) {
 245                                 // avoiding statSync is too hard for now, will fix later TODO: fix this sync bit
 246                                 var stats = fs.statSync(unify.fullFilePath+"/"+files[i]);
 247
 248                                 if(files[i].match(/^\..*/) == null) {
 249                                         if(stats.isDirectory()) {
 250
 251                                                 res.write("Directory: <a href=\""+files[i]+"/\">"+files[i]+"/</a>\n");
 252                                                 nfiles++;
 253                                         } else if(stats.isFile()) {
 254                                                 var padlength = 80 - (files[i].length) - stats.size.toString().length;
 255                                                 var padding = "";
 256                                                 if(padlength > 0) {
 257                                                         padding = new Array(padlength).join(" ");
 258                                                 }
 259                                                 res.write("File:      <a href=\""+files[i]+"\">"+files[i]+"</a>"+padding+stats.size+" bytes\n");
 260                                                 nfiles++;
 261                                         }
 262                                 } else {
 263                                         console.log("ignoring file, ", files[i]);
 264                                 }
 265                         }
 266
 267                         if(nfiles == 0) res.write("Empty directory....\n");
 268
 269                         res.write("<hr></pre>");
 270                         res.end();
 271                 } else {
 272                         res.write("we have entered bizaro world...\n");
 273                         res.write("</pre>");
 274                         res.end();
 275                 }
 276         });
 277 }
 278
 279 exports.serviceDirectory = serviceDirectory;