repository maintainence code
[nodejs-repoproxy.git] / lib / cache.js
1 var fs = require("fs");
2 var http = require("http");
3 var url = require("url");
4 var path = require("path");
5
6 function upstreamRequest(unify) {
7         // first do a head request
8         console.log("upsteram as ", unify.requestFor);
9         
10         var endData = false;
11         var xpath = "";
12         var filefd = null;
13         if(unify.topPath !=null) if(unify.topPath != "") if(typeof global.repoproxy.repo[unify.topPath] != "undefined") {
14                 var uplink = global.repoproxy.repo[unify.topPath].url;
15                 xpath = uplink + unify.subPath;
16         }
17         
18         //unify.b.write("would send to '" + xpath + "'");
19         //unify.b.end();
20         
21         // not doing this properly yet...
22         if(typeof global.repoproxy.downloads[unify.fullFilePath] != undefined && global.repoproxy.downloads[unify.fullFilePath] == 1) {
23                 console.log("request for file thats being downloaded already, doing inline request");
24                 inlineService(unify);
25                 return;
26         }
27         
28         console.log("sending off to '%s'", xpath);
29         
30         var headReq = url.parse(xpath);
31         headReq["method"] = "HEAD";
32         
33         getup = http.request(headReq, function(res) {
34                 //res.setEncoding("utf8");
35                 
36                 if(!endData) {
37                         console.log("status code is ", typeof res.statusCode);
38                         switch(res.statusCode) {
39                         // TODO: this 301 directory redirect thing needs to work better
40                         case 301:
41                         case 302:
42                                 
43                                 var loc = res.headers.location.substr(res.headers.location.length-4);
44                                 var against_t = xpath + "/";
45                                 var against = against_t.substr(against_t.length-4);
46                                 
47                                 if(loc == against) {
48                                         console.log("got a redirect, upstream for loc => loc/ assuming its a directory");
49                                         makeCacheDir(unify);
50                                         unify.b.writeHead(302, { "Location": unify.originalReq + "/" });
51                                 } else {
52                                         console.log("checked '%s' against '%s', was false, sending 404", loc, against);
53                                         unify.b.writeHead(404, {"Content-Type": "text/plain"});
54                                         unify.b.write("404 Not Found\n");
55                                 }
56                                 unify.b.end();
57                                 endData = true;
58                                 break;
59                                 
60                         case 404:
61                                 unify.b.writeHead(404, {"Content-Type": "text/plain"});
62                                 unify.b.write("404 Not Found\n");
63                                 unify.b.end();
64                                 endData = true;
65                                 break;
66                         case 200:
67                                 makeCacheDir(unify);
68                                 if(unify.isDirectoryRequest) {
69                                         serviceDirectory(unify);                                        
70                                         endData = true;
71                                 } else {
72                                         // this is where it gets ugly
73                                         var filesize = res.headers["content-length"];
74                                         console.log("do ugly write: ", unify);
75                                         //unify.b.write(data);
76                                         var metafilename = unify.fullPathDirName + "/.meta."+ path.basename(unify.requestFor) +".filesize";
77                                         var metafile = fs.createWriteStream(metafilename);
78                                         metafile.write(filesize);
79                                         metafile.end();
80                                         getAndService(unify, xpath, filesize);
81                                         
82                                 }
83                                 break;
84                         default:
85                                 console.log(".... data");
86                                 //unify.b.write(data);
87                         }
88                 }               
89                 //console.log("res is now ", res);
90         });
91         
92         getup.end();
93         
94         //console.log("getup: ", getup);
95 }
96
97 exports.upstreamRequest = upstreamRequest;
98
99 function getAndService(unify, xpath, filesize) {
100         
101         console.log("calling in here with filesize, ", filesize)
102         unify.b.writeHead(200, {'Content-Length' : filesize});
103
104         
105         global.repoproxy.downloads[unify.fullFilePath] = 1;
106         
107
108         http.get(xpath, function(res) {
109
110             var file = fs.createWriteStream(unify.fullFilePath);
111         
112             //console.log("res: ", res);
113         
114             //res.setEncoding("utf8");
115         
116             res.on("data", function(data) {
117                     //console.log("chunk");
118                     file.write(data);
119                     unify.b.write(data);
120             });
121         
122             res.on("end", function() {
123                     console.log("end...");
124                     unify.b.end();
125                     file.end();
126                     global.repoproxy.downloads[unify.fullFilePath] = 0;
127             });
128             
129             res.on("error", function(err) {
130                 console.log("res threw error... ", err);
131             });
132         });
133 }
134
135 // this is nasty nasty thing that can go horribly wrong in some ways, but currently works...
136 function inlineService(unify) {
137         // this method is called when we need to service a file thats being downloaded by something else
138         var metafilename = unify.fullPathDirName + "/.meta."+ path.basename(unify.requestFor) +".filesize";
139         var fsizef = fs.createReadStream(metafilename);
140         var fsize = "";
141         var lastchunk = 0;
142         fsizef.on("data", function(data) {
143                 fsize += data;
144         });
145         
146         fsizef.on("end", function() {
147                 var sentSoFar = 0;
148                 unify.b.writeHead(200, {"Content-Length" : fsize });
149                 
150                 // now we go into the file reading loop.
151                 console.log("start of inline services");
152                 // we loop every 0.5s and do our thing
153                 
154                 function sendPieces() {
155                         // this is going to be so fun i want to play real life frogger in real life traffic...
156                         fs.stat(unify.fullFilePath, function(err, stats) {
157                                 if(err == null) {
158                                         if(stats["size"] > sentSoFar) {
159                                                 // if file size changed between last chunk and this chunk, send the chunks
160                                                 
161                                                 lastChunk = 0;
162                                                 // open the file, send the data
163                                                 var rs = fs.createReadStream(unify.fullFilePath, {start: sentSoFar, end: stats["size"]});
164                                                 
165                                                 rs.on("data", function(thisdata) {
166                                                         //console.log("inline chunk: ", thisdata.length);
167                                                         unify.b.write(thisdata);
168                                                 });
169                                                 
170                                                 rs.on("end", function() {
171                                                         sentSoFar = stats["size"];
172                                                         // every second, we start again
173                                                         if(sentSoFar != fsize) {
174                                                                 setTimeout(sendPieces, 1000);
175                                                         } else {
176                                                                 // we're done!
177                                                                 unify.b.end();
178                                                         }
179                                                 });
180                                         } else {
181                                                 // if file size did not change between last timeout and this one, incremement the chunk counter
182                                                 // if we reach 60, we had a problem, and so we bomb out
183                                                 
184                                                 lastChunk++;
185                                                 
186                                                 // we bombed out somehow
187                                                 if(lastChunk > 60) {
188                                                         unify.b.end();
189                                                 } else {
190                                                         setTimeout(sendPieces, 1000);
191                                                 }
192                                         }
193                                 } else {
194                                         console.log("inline service - we're in a very bad place");
195                                 }
196                         });
197                         
198                 }
199                 
200                 setTimeout(sendPieces, 100);
201         });
202 }
203
204 // the service file routine .... PLEASE KILL ME!
205 function serviceFile(unify) {
206         
207         // for now, ignore range.
208         // however we need to check if a metadata file exists describing the filesize, check if its all correct
209         // and if not, erase the file (and metafile) and forward the request back to upstream request
210
211         
212         checkFile(unify, function() {
213                 
214                 // file should already exist, so we just poop it out
215                 var inp = fs.createReadStream(unify.fullFilePath);
216                 //inp.setEncoding("utf8");
217                 inp.on("data", function(data) {
218                         unify.b.write(data);
219                 });
220                 
221                 inp.on("end", function(closed) {
222                         unify.b.end();
223                 });
224         });
225 }
226
227 exports.serviceFile = serviceFile;
228
229
230 function checkFile(unify, callback) {
231         // in here we do the metadata checks
232         var metafilename = unify.fullPathDirName + "/.meta."+ path.basename(unify.requestFor) +".filesize";
233         
234         fs.exists(metafilename, function(existence) {
235                 if(existence) {
236                         var fsizef = fs.createReadStream(metafilename);
237                         var fsize = "";
238                         fsizef.on("data", function(data) {
239                                 fsize += data;
240                         });
241                         
242                         fsizef.on("end", function() {
243                                 fs.stat(unify.fullFilePath, function(err, stats) {
244                                         var rfsize = stats["size"];
245                                         if(rfsize != fsize.trim()) {
246                                                 // remove the file and start again
247                                                 console.log("reported filesizes dont match, '%s', '%s', removing file and starting again", rfsize, stats["size"]);
248                                                 try {
249                                                         fs.unlink(metafilename, function(){
250                                                                 fs.unlink(unify.fullFilePath, function(){
251                                                                         upstreamRequest(unify);                                                 
252                                                                 })
253                                                         });
254                                                 } catch(e) {
255                                                         upstreamRequest(unify);
256                                                 }
257                                         } else {
258                                                 // we're good
259                                                 unify.b.writeHead(200, {"Content-Length" : unify.fileSize})
260                                                 callback();
261                                         }
262                                 });
263                         });
264                 } else {
265                         console.log("file, '%s' exists but has no filesize meta data, assuming it was put here manually and servicing", unify.fullFilePath);
266                         unify.b.writeHead(200, {"Content-Length" : unify.fileSize})
267                         callback();
268                 }
269         });
270 }
271
272 function makeCacheDir(path) {
273         console.log("attempting to create... '%s' as '%s'", path.fullPathDirName, path.subPathDirName);
274         
275         var startAt = path.topFullPath;
276         var nextbits = path.subPathDirName.split("/");
277         for(var i=0; i < nextbits.length; i++) {
278                 startAt += "/" + nextbits[i];
279                 console.log("attempt mkdir on '%s'", startAt);
280                 try {
281                         fs.mkdirSync(startAt);
282                 } catch(e) {
283                         //console.log("e in mkdir, ", e);
284                 }
285         }
286         //process.exit(0);
287 }
288
289 function serviceDirectory(unify) {
290         var nfiles = 0;
291         var res = unify.b;
292         
293         res.write("<html><h1>Directory listing for " + unify.originalReq + "</h1><hr><pre>");
294         if(unify.originalReq != "/") res.write("<a href=\"..\">Parent</a>\n\n");
295         fs.readdir(unify.fullFilePath, function(err, files) {
296                 console.log("doing directory listing on: ", unify.fullFilePath);
297                 if(err == null) {
298                         
299                         // TODO: make this work asynchronously...
300                         for(var i=0; i<files.length; i++) {
301                                 // avoiding statSync is too hard for now, will fix later TODO: fix this sync bit
302                                 var stats = fs.statSync(unify.fullFilePath+"/"+files[i]);
303                                 
304                                 if(files[i].match(/^\..*/) == null) {
305                                         if(stats.isDirectory()) {
306                                                 
307                                                 res.write("Directory: <a href=\""+files[i]+"/\">"+files[i]+"/</a>\n");
308                                                 nfiles++;
309                                         } else if(stats.isFile()) {
310                                                 var padlength = 80 - (files[i].length) - stats.size.toString().length;
311                                                 var padding = "";
312                                                 if(padlength > 0) {
313                                                         padding = new Array(padlength).join(" ");
314                                                 }
315                                                 res.write("File:      <a href=\""+files[i]+"\">"+files[i]+"</a>"+padding+stats.size+" bytes\n");
316                                                 nfiles++;
317                                         }
318                                 } else {
319                                         console.log("ignoring file, ", files[i]);
320                                 }
321                         }
322                         
323                         if(nfiles == 0) res.write("Empty directory....\n");
324                         
325                         res.write("<hr></pre>");
326                         res.end();
327                 } else {
328                         res.write("we have entered bizaro world...\n");
329                         res.write("</pre>");
330                         res.end();
331                 }
332         });
333 }
334
335 exports.serviceDirectory = serviceDirectory;