--- /dev/null
+TODO
+====
+
+- fix the log.* entries that need to be errors
+- metadata check on maintenance routines
+-
\ No newline at end of file
var url = require("url");
var path = require("path");
var crypto = require("crypto");
+var log = require("./log.js");
function upstreamRequest(unify) {
// first do a head request
- console.log("upsteram as ", unify.requestFor);
+ log.debug("upsteram as ", unify.requestFor);
var endData = false;
var xpath = "";
// not doing this properly yet...
if(typeof global.repoproxy.downloads[unify.fullFilePath] != undefined && global.repoproxy.downloads[unify.fullFilePath] == 1) {
- console.log("request for file thats being downloaded already, doing inline request");
+ log.debug("request for file thats being downloaded already, doing inline request");
inlineService(unify);
return;
}
- console.log("sending off to '%s'", xpath);
+ log.debug("sending off to '%s'", xpath);
var headReq = url.parse(xpath);
headReq["method"] = "HEAD";
//res.setEncoding("utf8");
if(!endData) {
- console.log("status code is ", typeof res.statusCode);
+ log.debug("status code is ", typeof res.statusCode);
switch(res.statusCode) {
// TODO: this 301 directory redirect thing needs to work better
case 301:
var against = against_t.substr(against_t.length-4);
if(loc == against) {
- console.log("got a redirect, upstream for loc => loc/ assuming its a directory");
+ log.debug("got a redirect, upstream for loc => loc/ assuming its a directory");
makeCacheDir(unify);
unify.b.writeHead(302, { "Location": unify.originalReq + "/" });
} else {
- console.log("checked '%s' against '%s', was false, sending 404", loc, against);
+ log.debug("checked '%s' against '%s', was false, sending 404", loc, against);
unify.b.writeHead(404, {"Content-Type": "text/plain"});
unify.b.write("404 Not Found\n");
}
} else {
// this is where it gets ugly
var filesize = res.headers["content-length"];
- console.log("do ugly write: ", unify);
+ log.debug("do ugly write: ", unify);
//unify.b.write(data);
var metafilename = unify.fullPathDirName + "/.meta."+ path.basename(unify.requestFor) +".filesize";
var metafile = fs.createWriteStream(metafilename);
}
break;
default:
- console.log(".... data");
+ log.debug(".... data");
//unify.b.write(data);
}
}
- //console.log("res is now ", res);
+ //log.debug("res is now ", res);
});
getup.end();
- //console.log("getup: ", getup);
+ //log.debug("getup: ", getup);
}
exports.upstreamRequest = upstreamRequest;
function getAndService(unify, xpath, filesize) {
- console.log("calling in here with filesize, ", filesize)
+ log.debug("calling in here with filesize, ", filesize)
unify.b.writeHead(200, {'Content-Length' : filesize});
var file = fs.createWriteStream(unify.fullFilePath);
- //console.log("res: ", res);
+ //log.debug("res: ", res);
//res.setEncoding("utf8");
res.on("data", function(data) {
- //console.log("chunk");
+ //log.debug("chunk");
file.write(data);
unify.b.write(data);
});
res.on("end", function() {
- console.log("end...");
+ log.debug("end...");
unify.b.end();
file.end();
global.repoproxy.downloads[unify.fullFilePath] = 0;
});
res.on("error", function(err) {
- console.log("res threw error... ", err);
+ log.debug("res threw error... ", err);
});
});
}
unify.b.writeHead(200, {"Content-Length" : fsize });
// now we go into the file reading loop.
- console.log("start of inline services");
+ log.debug("start of inline services");
// we loop every 0.5s and do our thing
function sendPieces() {
var rs = fs.createReadStream(unify.fullFilePath, {start: sentSoFar, end: stats["size"]});
rs.on("data", function(thisdata) {
- //console.log("inline chunk: ", thisdata.length);
+ //log.debug("inline chunk: ", thisdata.length);
unify.b.write(thisdata);
});
}
}
} else {
- console.log("inline service - we're in a very bad place");
+ log.debug("inline service - we're in a very bad place");
}
});
var rfsize = stats["size"];
if(rfsize != fsize.trim()) {
// remove the file and start again
- console.log("reported filesizes dont match, '%s', '%s', removing file and starting again", rfsize, stats["size"]);
+ log.debug("reported filesizes dont match, '%s', '%s', removing file and starting again", rfsize, stats["size"]);
try {
fs.unlink(metafilename, function(){
fs.unlink(unify.fullFilePath, function(){
});
});
} else {
- console.log("file, '%s' exists but has no filesize meta data, assuming it was put here manually and servicing", unify.fullFilePath);
+ log.debug("file, '%s' exists but has no filesize meta data, assuming it was put here manually and servicing", unify.fullFilePath);
unify.b.writeHead(200, {"Content-Length" : unify.fileSize})
callback();
}
}
function makeCacheDir(path) {
- console.log("attempting to create... '%s' as '%s'", path.fullPathDirName, path.subPathDirName);
+ log.debug("attempting to create... '%s' as '%s'", path.fullPathDirName, path.subPathDirName);
var startAt = path.topFullPath;
var nextbits = path.subPathDirName.split("/");
for(var i=0; i < nextbits.length; i++) {
startAt += "/" + nextbits[i];
- console.log("attempt mkdir on '%s'", startAt);
+ log.debug("attempt mkdir on '%s'", startAt);
try {
fs.mkdirSync(startAt);
} catch(e) {
- //console.log("e in mkdir, ", e);
+ //log.debug("e in mkdir, ", e);
}
}
//process.exit(0);
res.write("<html><h1>Directory listing for " + unify.originalReq + "</h1><hr><pre>");
if(unify.originalReq != "/") res.write("<a href=\"..\">Parent</a>\n\n");
fs.readdir(unify.fullFilePath, function(err, files) {
- console.log("doing directory listing on: ", unify.fullFilePath);
+ log.debug("doing directory listing on: ", unify.fullFilePath);
if(err == null) {
// TODO: make this work asynchronously...
nfiles++;
}
} else {
- console.log("ignoring file, ", files[i]);
+ log.debug("ignoring file, ", files[i]);
}
}
var encoded = (++global.repoproxy.fileid).toString();
var toloc = cleanup + "/" + ctime.toString() + "." + encoded;
- //console.log("Moving %s to %s for cleanup", file_or_dir.replace(/\/$/, ""), toloc);
+ //log.debug("Moving %s to %s for cleanup", file_or_dir.replace(/\/$/, ""), toloc);
fs.renameSync(file_or_dir.replace(/\/$/, ""), toloc);
}
global.repoproxy.repo = new Object();
global.repoproxy.scancache = 1;
global.repoproxy.downloads = new Object();
+ global.repoproxy.loglevel = 3;
// set a global file id for file trashing
//global.repoproxy.fileid = new Object();
// parse the line
switch(line_real[0]) {
case "repo":
-
// TODO: VALIDATE!
- console.log("Adding repo: '/%s' type '%s' from '%s', with update interval of '%s' days, and expire time of '%s' days.", line_real[1], line_real[2], line_real[3]+":"+line_real[4], line_real[5], line_real[6]);
+ log.info("Adding repo: '/%s' type '%s' from '%s', with update interval of '%s' days, and expire time of '%s' days.", line_real[1], line_real[2], line_real[3]+":"+line_real[4], line_real[5], line_real[6]);
var thisrepo = { type : line_real[2], url: line_real[3]+":"+line_real[4], updateinterval: line_real[5], expiretime: line_real[6] };
global.repoproxy.repo[line_real[1]] = thisrepo;
-
break;
+
case "cachedir":
var tmppath = line_real[1].replace(/\/+/g, "/");
- console.log("Cache dir set to: ", tmppath);
+ log.info("Cache dir set to: ", tmppath);
global.repoproxy.cacheDir = tmppath;
break;
+
case "listenport":
- console.log("Port set to: ", line_real[1]);
+ log.info("Port set to: ", line_real[1]);
global.repoproxy.listenPort = line_real[1];
break;
+
case "cachescan":
- console.log("Set cache scan rate to: '%s' hours", line_real[1]);
+ log.info("Set cache scan rate to: '%s' hours", line_real[1]);
global.repoproxy.scancache = parseInt(line_real[1]);
if(global.repoproxy.scancache == 0) {
- console.log("Cache scan rate didnt make sense, it was 0, and should be at least 1 - it is set to 24, but you should check this setting");
+ log.info("Cache scan rate didnt make sense, it was 0, and should be at least 1 - it is set to 24, but you should check this setting");
global.repoproxy.scancache = 24;
}
break;
+
case "loggerlevel":
global.repoproxy.loglevel = parseInt(line_real[1]);
- console.log("log level set to: ", global.repoproxy.loglevel);
+ log.info("log level set to: ", global.repoproxy.loglevel);
break;
+
default:
if(line_real[0] != "") {
- console.log("WARNING Invalid line in configuration file ignored: '%s'", line_one);
+ log.info("WARNING Invalid line in configuration file ignored: '%s'", line_one);
}
}
}
createCacheStructure();
- log.testLogSettings();
+ //log.testLogSettings();
}
function createCacheStructure() {
try {
var state = fs.statSync(global.repoproxy.cacheDir);
- //console.log("state is:", state);
+ //log.info("state is:", state);
} catch(e) {
- //console.log("try failure of cache dir stat ", e);
+ //log.info("try failure of cache dir stat ", e);
try {
fs.mkdirSync(global.repoproxy.cacheDir);
} catch(ex) {
- console.log("ERROR: failure to create cache directory, '%s'", global.repoproxy.cacheDir);
+ log.error("failure to create cache directory, '%s'", global.repoproxy.cacheDir);
}
}
try {
var state = fs.statSync(global.repoproxy.cacheDir + "/.cleanup");
- //console.log("state is:", state);
+ //log.info("state is:", state);
} catch(e) {
try {
fs.mkdirSync(global.repoproxy.cacheDir + "/.cleanup");
} catch(ex) {
- console.log("ERROR: cant create cleanup directory, '%s'", global.repoproxy.cacheDir + "/.cleanup");
+ log.error("cant create cleanup directory, '%s'", global.repoproxy.cacheDir + "/.cleanup");
}
}
- //console.log("next: ", global.repoproxy.repo);
+ //log.info("next: ", global.repoproxy.repo);
for(var index in global.repoproxy.repo) {
var fullDir = global.repoproxy.cacheDir + "/" + index;
- //console.log("on end, ", fullDir);
+ //log.info("on end, ", fullDir);
try {
var state = fs.statSync(fullDir);
- //console.log("state is:", state);
+ //log.info("state is:", state);
} catch(e) {
try {
- //console.log("attempted to create cache dir, ", fullDir);
+ //log.info("attempted to create cache dir, ", fullDir);
fs.mkdirSync(fullDir);
} catch(ex) {
- console.log("ERROR: failed to create cache directory, '%s' for '%s'", fullDir, index);
+ log.error("failed to create cache directory, '%s' for '%s'", fullDir, index);
}
}
}
//var ts = ts_t.format("%Y-%m-%d %H:%M:%S");
arguments["0"] = "[??INFO??] ("+ts+"): "+orig;
- if(global.repoproxy.loglevel >= 3) {
+ if(typeof global.repoproxy.loglevel == "undefined" || global.repoproxy.loglevel >= 3) {
sendLog.apply(this, arguments);
}
}
//var ts = ts_t.format("%Y-%m-%d %H:%M:%S");
arguments["0"] = "[^^DEBUG^^] ("+ts+"): "+orig;
- if(global.repoproxy.loglevel >= 4) {
- sendLog.apply(this, arguments);
+ if(typeof global.repoproxy.loglevel != "undefined") {
+ if(global.repoproxy.loglevel >= 3) {
+ sendLog.apply(this, arguments);
+ }
}
}
//var ts = ts_t.format("%Y-%m-%d %H:%M:%S");
arguments["0"] = "[!!WARNING!!] ("+ts+"): "+orig;
- if(global.repoproxy.loglevel >= 2) {
+ if(typeof global.repoproxy.loglevel == "undefined" || global.repoproxy.loglevel >= 2) {
sendLog.apply(this, arguments);
}
}
//var ts = ts_t.format("%Y-%m-%d %H:%M:%S");
arguments["0"] = "[**ERROR**] ("+ts+"): "+orig;
- if(global.repoproxy.loglevel >= 1) {
+ if(typeof global.repoproxy.loglevel == "undefined" || global.repoproxy.loglevel >= 1) {
sendLog.apply(this, arguments);
}
}
var path = require("path");
var repoapt = require("./repo-apt.js");
var repoyum = require("./repo-yum.js");
+var log = require("./log.js");
function maintainCache() {
// TODO i should check that im already running here and exit if i am
- console.log("Cache maintainence routine starting...");
+ log.debug("Cache maintainence routine starting...");
for(var index in global.repoproxy.repo) {
- //console.log("start cleanup in ", index);
- //console.log("cleanup array ", global.repoproxy.repo[index]);
+ //log.debug("start cleanup in ", index);
+ //log.debug("cleanup array ", global.repoproxy.repo[index]);
switch(global.repoproxy.repo[index]["type"]) {
case "apt":
- console.log("Scanning '%s' as apt", index);
+ log.debug("Scanning '%s' as apt", index);
var walkin = path.normalize(global.repoproxy.cacheDir + "/" + index);
walkDir(walkin, function(err, list) {
repoapt.maintain(index, global.repoproxy.repo[index], list);
})
break;
case "yum":
- //console.log("Scanning '%s' as apt", index);
+ //log.debug("Scanning '%s' as apt", index);
//repoyum.maintain(global.repoproxy.repo[index]);
- console.log("Scanning '%s' as yum", index);
+ log.debug("Scanning '%s' as yum", index);
var walkin = path.normalize(global.repoproxy.cacheDir + "/" + index);
walkDir(walkin, function(err, list) {
repoyum.maintain(index, global.repoproxy.repo[index], list);
break;
}
}
- console.log("Cache maintainence routine ended...");
+ log.debug("Cache maintainence routine ended...");
}
exports.startTimer = function() {
var http = require("http");
var url = require("url");
var path = require("path");
+var log = require("./log.js");
function maintain(name, repoinfo, files) {
- //console.log("doing apt clean for ", repoinfo);
+ //log.debug("doing apt clean for ", repoinfo);
var topdir = path.normalize(global.repoproxy.cacheDir + "/" + name + "/");
function fileCheck(i) {
- console.log("checking file: ", files[i]);
+ log.debug("checking file: ", files[i]);
var cfile = files[i].replace(topdir, "");
var ctime_t = new Date();
// file checks go here
if(typeof global.repoproxy.downloads[files[i]] != "undefined" && global.repoproxy.downloads[files[i]] == 1) {
// ignore this file as its being downloaded
- console.log("not checking file because its downloading ", cfile);
+ log.debug("not checking file because its downloading ", cfile);
if(typeof files[i+1] != "undefined") fileCheck(i+1);
} else {
fs.stat(files[i], function(err, stats) {
- //console.log("deep check", cfile);
+ //log.debug("deep check", cfile);
if(cfile.match(/.*dists\/.*/) != null) {
// its a dist file, probably, check age and erase if necessary
if(stats["mtime"].getTime() < time_ui) {
// erase file
- console.log("unlinking file for time_ui: ", files[i])
+ log.debug("unlinking file for time_ui: ", files[i])
//fs.unlink(files[i]);
} else {
- //console.log("times for file '%s', '%s', '%s', '%s'", cfile, time_ui, time_et, stats["mtime"].getTime());
+ //log.debug("times for file '%s', '%s', '%s', '%s'", cfile, time_ui, time_et, stats["mtime"].getTime());
}
} else {
// its just some other file, check the read time
if(stats["atime"].getTime() < time_et) {
- console.log("unlinking file for time_et: ", files[i]);
+ log.debug("unlinking file for time_et: ", files[i]);
}
}
if(typeof files[i+1] != "undefined") fileCheck(i+1);
if(typeof files[0] != 'undefined') fileCheck(0)
- else console.log("Skipping (apt) file check as there are none... apprently?");
+ else log.debug("Skipping (apt) file check as there are none... apprently?");
}
exports.maintain = maintain;
\ No newline at end of file
log.debug("ctime is ", atime);
var age = curtime - atime;
- //console.log("age is (%d) for (%d)", age, expireAge);
+ //log.debug("age is (%d) for (%d)", age, expireAge);
if(age > expireAge) {
// TODO: cleanup singular file
// TODO: cleanup meta too, fuck me
- //console.log("clean up file \n", files[i]);
+ //log.debug("clean up file \n", files[i]);
var metafile = files[i].replace(/(.*)\/(.[^\/]+$)/, "$1/.meta.$2.filesize");
- //console.log("meta for this file is \n", nfile);
+ //log.debug("meta for this file is \n", nfile);
cache.moveToCleanup(files[i]);
cache.moveToCleanup(metafile);
log.info("Sending expired file to trash for cleanup (%s)", files[i]);
exports.routeRequest = function(req, res) {
// first, unify the request
- console.log("request: ", req.url);
+ log.debug("request: ", req.url);
var thisQuery = unifyRequest(req, res, function(unified) {
- console.log("unified request is ", unified);
+ log.debug("unified request is ", unified);
if(unified.requestFor == "/favicon.ico") {
unified.b.writeHead(404, {"Content-Type": "text/plain"});
unified.b.write("404 Not Found\n");
} else if(unified.isDirectory) {
cache.serviceDirectory(unified);
} else {
- console.log("ERROR: something went majorly wrong with something, ", unified);
+ log.debug("ERROR: something went majorly wrong with something, ", unified);
}
} else {
// it doesnt exist yet, so we send it to the cache service if it matches an upstream service
if(typeof global.repoproxy.repo[unified.topPath] != "undefined") {
- console.log("file doesnt exist, upstream we go: ", unified);
+ log.debug("file doesnt exist, upstream we go: ", unified);
cache.upstreamRequest(unified);
} else {
unified.b.writeHead(404, {"Content-Type": "text/plain"});
projects / nodejs-repoproxy.git / commitdiff