added a logger, started working on the maintenance timer routines

diff --git a/lib/cache.js b/lib/cache.js
index b7f111d..588bc73 100644 (file)
--- a/lib/cache.js
+++ b/lib/cache.js
@@ -2,6 +2,7 @@ var fs = require("fs");
 var http = require("http");
 var url = require("url");
 var path = require("path");
+var crypto = require("crypto");
 
 function upstreamRequest(unify) {
        // first do a head request
@@ -332,4 +333,22 @@ function serviceDirectory(unify) {
        });
 }
 
-exports.serviceDirectory = serviceDirectory;
\ No newline at end of file
+function moveToCleanup(file_or_dir) {
+       // err..?
+       var cleanup = global.repoproxy.cacheDir + "/.cleanup";
+       var ctime = new Date().getTime();
+       var encoded = (++global.repoproxy.fileid).toString();
+       var toloc = cleanup + "/" + ctime.toString() + "." + encoded;
+       
+       //console.log("Moving %s to %s for cleanup", file_or_dir.replace(/\/$/, ""), toloc);
+       
+       fs.renameSync(file_or_dir.replace(/\/$/, ""), toloc);
+}
+
+function cleanupRoutine() {
+       
+}
+
+
+exports.serviceDirectory = serviceDirectory;
+exports.moveToCleanup = moveToCleanup;

diff --git a/lib/config.js b/lib/config.js
index 7cab68a..8d6d2fc 100644 (file)
--- a/lib/config.js
+++ b/lib/config.js
@@ -1,4 +1,5 @@
 var fs = require("fs");
+var log = require("./log.js");
 
 exports.loadConfig = function (conffile) {
        
@@ -8,6 +9,12 @@ exports.loadConfig = function (conffile) {
        global.repoproxy.repo = new Object();
        global.repoproxy.scancache = 1;
        global.repoproxy.downloads = new Object();
+
+       // set a global file id for file trashing
+       //global.repoproxy.fileid = new Object();
+       global.repoproxy.fileid = 1;
+
+
        
        var confFileData = fs.readFileSync(conffile, "utf8");
        
@@ -50,6 +57,10 @@ exports.loadConfig = function (conffile) {
                                global.repoproxy.scancache = 24;
                        }
                        break;
+               case "loggerlevel":
+                       global.repoproxy.loglevel = parseInt(line_real[1]);
+                       console.log("log level set to: ", global.repoproxy.loglevel);
+                       break;
                default:
                        if(line_real[0] != "") {
                                console.log("WARNING Invalid line in configuration file ignored: '%s'", line_one);
@@ -58,38 +69,45 @@ exports.loadConfig = function (conffile) {
        }
        
        createCacheStructure();
+       log.testLogSettings();
 }
 
 
+
 function createCacheStructure() {
        try {
                var state = fs.statSync(global.repoproxy.cacheDir);
                //console.log("state is:", state);
        } catch(e) {
+               //console.log("try failure of cache dir stat ", e);
                try {
                        fs.mkdirSync(global.repoproxy.cacheDir);
-                       fs.mkdirSync(global.repoproxy.cacheDir + "/.cleanup");
                } catch(ex) {
                        console.log("ERROR: failure to create cache directory, '%s'", global.repoproxy.cacheDir);
                }
        }
        
        try {
-               fs.mkdirSync(global.repoproxy.cacheDir + "/.cleanup");
-       } catch(ex) {
-               console.log("ERROR: cant create cleanup directory, '%s'", global.repoproxy.cacheDir + "/.cleanup");
+               var state = fs.statSync(global.repoproxy.cacheDir + "/.cleanup");
+               //console.log("state is:", state);
+       } catch(e) {
+               try {
+                       fs.mkdirSync(global.repoproxy.cacheDir + "/.cleanup");
+               } catch(ex) {
+                       console.log("ERROR: cant create cleanup directory, '%s'", global.repoproxy.cacheDir + "/.cleanup");
+               }
        }
        
-       console.log("next: ", global.repoproxy.repo);
+       //console.log("next: ", global.repoproxy.repo);
        for(var index in global.repoproxy.repo) {
                var fullDir = global.repoproxy.cacheDir + "/" + index;
-               console.log("on end, ", fullDir);
+               //console.log("on end, ", fullDir);
                try {
                        var state = fs.statSync(fullDir);
-                       console.log("state is:", state);
+                       //console.log("state is:", state);
                } catch(e) {
                        try {
-                               console.log("attempted to create cache dir, ", fullDir);
+                               //console.log("attempted to create cache dir, ", fullDir);
                                fs.mkdirSync(fullDir);
                        } catch(ex) {
                                console.log("ERROR: failed to create cache directory, '%s' for '%s'", fullDir, index);

diff --git a/lib/log.js b/lib/log.js
new file mode 100644 (file)
index 0000000..24f31d2
--- /dev/null
+++ b/lib/log.js
@@ -0,0 +1,70 @@
+
+
+/*
+ * # level for logging (1 = error, 2 = warning, 3=info, 4=debug);
+loggerlevel:4
+
+ */
+function testLogSettings() {
+       var curtime = new Date().getTime();
+       debug("This is a DEBUG TeST (%d)", curtime);
+       info("This is an Info TEST (%d)", curtime);
+       warning("This is a WARN test (%d)", curtime);
+       error("This is an ERROR test (%d)", curtime);
+}
+
+
+function info() {
+       var orig = arguments["0"];
+       var ts = new Date().toISOString();
+       //var ts = ts_t.format("%Y-%m-%d %H:%M:%S");
+       arguments["0"] = "[??INFO??] ("+ts+"): "+orig;
+       
+       if(global.repoproxy.loglevel >= 3) {
+               sendLog.apply(this, arguments);
+       }
+}
+
+function debug() {
+       var orig = arguments["0"];
+       var ts = new Date().toISOString();
+       //var ts = ts_t.format("%Y-%m-%d %H:%M:%S");
+       arguments["0"] = "[^^DEBUG^^] ("+ts+"): "+orig;
+       
+       if(global.repoproxy.loglevel >= 4) {
+               sendLog.apply(this, arguments); 
+       }
+}
+
+function warning() {
+       var orig = arguments["0"];
+       var ts = new Date().toISOString();
+       //var ts = ts_t.format("%Y-%m-%d %H:%M:%S");
+       arguments["0"] = "[!!WARNING!!] ("+ts+"): "+orig;
+       
+       if(global.repoproxy.loglevel >= 2) {
+               sendLog.apply(this, arguments);
+       }
+}
+
+function error() {
+       var orig = arguments["0"];
+       var ts = new Date().toISOString();
+       //var ts = ts_t.format("%Y-%m-%d %H:%M:%S");
+       arguments["0"] = "[**ERROR**] ("+ts+"): "+orig;
+       
+       if(global.repoproxy.loglevel >= 1) {
+               sendLog.apply(this, arguments);         
+       }       
+}
+
+function sendLog() {
+       console.log.apply(this, arguments);
+}
+
+
+exports.info = info;
+exports.debug = debug;
+exports.warning = warning;
+exports.error = error;
+exports.testLogSettings = testLogSettings;

diff --git a/lib/maintain.js b/lib/maintain.js
index b740a78..46c3ddc 100644 (file)
--- a/lib/maintain.js
+++ b/lib/maintain.js
@@ -68,8 +68,6 @@ function walkDir(dir, done) {
        });
 };
 
-exports.walkDir = walkDir;
 
-function cleanupRoutine() {
-       
-}
\ No newline at end of file
+
+exports.walkDir = walkDir;

diff --git a/lib/repo-yum.js b/lib/repo-yum.js
index 07a1893..25db170 100644 (file)
--- a/lib/repo-yum.js
+++ b/lib/repo-yum.js
@@ -2,61 +2,66 @@ var fs = require("fs");
 var http = require("http");
 var url = require("url");
 var path = require("path");
+var cache = require("./cache.js");
+var log = require("./log.js");
 
 
 
 function maintain(name, repoinfo, files) {
-       console.log("doing yum clean for ", repoinfo);
+       log.info("Starting maintenance routine for yum repo %s (%s)", name, repoinfo.url);
        
        var metaAge = 24*3600*1000*(repoinfo.updateinterval);
        var expireAge = 24*3600*1000*(repoinfo.expiretime);
        
        function fileCheck(i) {
-               console.log("checking file: ", files[i]);
+               log.debug("checking file: ", files[i]);
                
                // we look for repodata/repomd.xml file, if this is past maturity, we clean that whole directory
                if(files[i].match(/.*repodata\/repomd\.xml$/)) {
-                       console.log("Found repomd.xml file: ", files[i]);
+                       log.debug("Found repomd.xml file: ", files[i]);
                        
                        // strip the repomd file to get the dir
                        var repomddir = files[i].replace(/repomd\.xml$/, "");
-                       console.log("repomd dir is:", repomddir);
+                       log.debug("repomd dir is:", repomddir);
 
                        
                        // do the file stat
                        fs.stat(files[i], function(err, stats) {
-                               console.log("stats for file was: ", stats);
+                               log.debug("stats for file was: ", stats);
                                var curtime = new Date().getTime();
                                var ctime = stats.ctime.getTime();
-                               console.log("curtime is ", curtime);
-                               console.log("ctime is ", ctime);
+                               log.debug("curtime is ", curtime);
+                               log.debug("ctime is ", ctime);
                                
                                var age = curtime - ctime;
-                               console.log("age is (%d) for (%d)", age, metaAge);
+                               log.debug("age is (%d) for (%d)", age, metaAge);
                                if(age > metaAge) {
                                        // TODO: cleanup repomd directory
+                                       cache.moveToCleanup(repomddir);
+                                       log.info("Sending repomd directory to trash for cleanup (%s)", repomddir);
                                }
                        })
                        
                } else {
                        // STUFF!!!
                        fs.stat(files[i], function(err, stats) {
-                               console.log("stats for file was: ", stats);
+                               log.debug("stats for file was: ", stats);
                                var curtime = new Date().getTime();
                                var atime = stats.atime.getTime();
-                               console.log("curtime is ", curtime);
-                               console.log("ctime is ", atime);
+                               log.debug("curtime is ", curtime);
+                               log.debug("ctime is ", atime);
                                
                                var age = curtime - atime;
-                               console.log("age is (%d) for (%d)", age, expireAge);
+                               //console.log("age is (%d) for (%d)", age, expireAge);
                                if(age > expireAge) {
                                        // TODO: cleanup singular file
                                        // TODO: cleanup meta too, fuck me
-                                       console.log("clean up file \n", files[i]);
+                                       //console.log("clean up file \n", files[i]);
                                        var metafile = files[i].replace(/(.*)\/(.[^\/]+$)/, "$1/.meta.$2.filesize");
                                        //console.log("meta for this file is \n", nfile);
-                                       fs.unlink(files[i]);
-                                       fs.unlink(metafile);
+                                       cache.moveToCleanup(files[i]);
+                                       cache.moveToCleanup(metafile);
+                                       log.info("Sending expired file to trash for cleanup (%s)", files[i]);
                                }
                        })
                }
@@ -65,7 +70,7 @@ function maintain(name, repoinfo, files) {
        }
        
        if(typeof files[0] != 'undefined') fileCheck(0)
-       else console.log("Skipping (yum) file check as there are none... apprently?");
+       else log.info("Skipping (yum) file check as there are none... apprently?");
 
 
 }

diff --git a/lib/router.js b/lib/router.js
index fcf697c..63622f7 100644 (file)
--- a/lib/router.js
+++ b/lib/router.js
@@ -2,6 +2,7 @@ var url = require("url");
 var fs = require("fs");
 var cache = require("./cache.js");
 var path = require("path");
+var log = require("./log.js");
 
 exports.routeRequest = function(req, res) {
        // first, unify the request

diff --git a/proxy.js b/proxy.js
index dabbdec..61aa8f2 100644 (file)
--- a/proxy.js
+++ b/proxy.js
@@ -9,7 +9,7 @@ var maintain = require("./lib/maintain.js");
 console.log("Loading configuration");
 config.loadConfig("./repos.conf");
 
-
+// start the maintenance timer
 console.log("Starting cache maintenance timer");
 maintain.startTimer();
 

diff --git a/repos.conf b/repos.conf
index f1ab11a..dfa404f 100644 (file)
--- a/repos.conf
+++ b/repos.conf
@@ -18,3 +18,6 @@ listenport:8008
 #              packageage is how long a package will go unread before it gets deleted (days)
 repo:fedora:yum:http://ftp.iinet.net.au/pub/fedora/linux/:7:120
 repo:ubuntu:apt:http://ftp.iinet.net.au/pub/ubuntu/:1:120
+
+# level for logging (1 = error, 2 = warning, 3=info, 4=debug);
+loggerlevel:3

diff --git a/unittests/argtest.js b/unittests/argtest.js
new file mode 100644 (file)
index 0000000..633f3fe
--- /dev/null
+++ b/unittests/argtest.js
@@ -0,0 +1,8 @@
+
+function logtest() {
+       console.log.apply(this, arguments);
+}
+
+var asdf = new Date();
+
+logtest("one", asdf);