From d7478970127408f056b157f18df74fef7db3f892 Mon Sep 17 00:00:00 2001 From: Paul J R Date: Tue, 25 Jun 2013 08:18:02 +1000 Subject: [PATCH] added a logger, started working on the maintenance timer routines --- lib/cache.js | 21 ++++++++++++++- lib/config.js | 34 ++++++++++++++++++----- lib/log.js | 70 ++++++++++++++++++++++++++++++++++++++++++++++++++ lib/maintain.js | 6 +--- lib/repo-yum.js | 37 +++++++++++++++----------- lib/router.js | 1 + proxy.js | 2 +- repos.conf | 3 ++ unittests/argtest.js | 8 +++++ 9 files changed, 152 insertions(+), 30 deletions(-) create mode 100644 lib/log.js create mode 100644 unittests/argtest.js diff --git a/lib/cache.js b/lib/cache.js index b7f111d..588bc73 100644 --- a/lib/cache.js +++ b/lib/cache.js @@ -2,6 +2,7 @@ var fs = require("fs"); var http = require("http"); var url = require("url"); var path = require("path"); +var crypto = require("crypto"); function upstreamRequest(unify) { // first do a head request @@ -332,4 +333,22 @@ function serviceDirectory(unify) { }); } -exports.serviceDirectory = serviceDirectory; \ No newline at end of file +function moveToCleanup(file_or_dir) { + // err..? + var cleanup = global.repoproxy.cacheDir + "/.cleanup"; + var ctime = new Date().getTime(); + var encoded = (++global.repoproxy.fileid).toString(); + var toloc = cleanup + "/" + ctime.toString() + "." + encoded; + + //console.log("Moving %s to %s for cleanup", file_or_dir.replace(/\/$/, ""), toloc); + + fs.renameSync(file_or_dir.replace(/\/$/, ""), toloc); +} + +function cleanupRoutine() { + +} + + +exports.serviceDirectory = serviceDirectory; +exports.moveToCleanup = moveToCleanup; diff --git a/lib/config.js b/lib/config.js index 7cab68a..8d6d2fc 100644 --- a/lib/config.js +++ b/lib/config.js @@ -1,4 +1,5 @@ var fs = require("fs"); +var log = require("./log.js"); exports.loadConfig = function (conffile) { @@ -8,6 +9,12 @@ exports.loadConfig = function (conffile) { global.repoproxy.repo = new Object(); global.repoproxy.scancache = 1; global.repoproxy.downloads = new Object(); + + // set a global file id for file trashing + //global.repoproxy.fileid = new Object(); + global.repoproxy.fileid = 1; + + var confFileData = fs.readFileSync(conffile, "utf8"); @@ -50,6 +57,10 @@ exports.loadConfig = function (conffile) { global.repoproxy.scancache = 24; } break; + case "loggerlevel": + global.repoproxy.loglevel = parseInt(line_real[1]); + console.log("log level set to: ", global.repoproxy.loglevel); + break; default: if(line_real[0] != "") { console.log("WARNING Invalid line in configuration file ignored: '%s'", line_one); @@ -58,38 +69,45 @@ exports.loadConfig = function (conffile) { } createCacheStructure(); + log.testLogSettings(); } + function createCacheStructure() { try { var state = fs.statSync(global.repoproxy.cacheDir); //console.log("state is:", state); } catch(e) { + //console.log("try failure of cache dir stat ", e); try { fs.mkdirSync(global.repoproxy.cacheDir); - fs.mkdirSync(global.repoproxy.cacheDir + "/.cleanup"); } catch(ex) { console.log("ERROR: failure to create cache directory, '%s'", global.repoproxy.cacheDir); } } try { - fs.mkdirSync(global.repoproxy.cacheDir + "/.cleanup"); - } catch(ex) { - console.log("ERROR: cant create cleanup directory, '%s'", global.repoproxy.cacheDir + "/.cleanup"); + var state = fs.statSync(global.repoproxy.cacheDir + "/.cleanup"); + //console.log("state is:", state); + } catch(e) { + try { + fs.mkdirSync(global.repoproxy.cacheDir + "/.cleanup"); + } catch(ex) { + console.log("ERROR: cant create cleanup directory, '%s'", global.repoproxy.cacheDir + "/.cleanup"); + } } - console.log("next: ", global.repoproxy.repo); + //console.log("next: ", global.repoproxy.repo); for(var index in global.repoproxy.repo) { var fullDir = global.repoproxy.cacheDir + "/" + index; - console.log("on end, ", fullDir); + //console.log("on end, ", fullDir); try { var state = fs.statSync(fullDir); - console.log("state is:", state); + //console.log("state is:", state); } catch(e) { try { - console.log("attempted to create cache dir, ", fullDir); + //console.log("attempted to create cache dir, ", fullDir); fs.mkdirSync(fullDir); } catch(ex) { console.log("ERROR: failed to create cache directory, '%s' for '%s'", fullDir, index); diff --git a/lib/log.js b/lib/log.js new file mode 100644 index 0000000..24f31d2 --- /dev/null +++ b/lib/log.js @@ -0,0 +1,70 @@ + + +/* + * # level for logging (1 = error, 2 = warning, 3=info, 4=debug); +loggerlevel:4 + + */ +function testLogSettings() { + var curtime = new Date().getTime(); + debug("This is a DEBUG TeST (%d)", curtime); + info("This is an Info TEST (%d)", curtime); + warning("This is a WARN test (%d)", curtime); + error("This is an ERROR test (%d)", curtime); +} + + +function info() { + var orig = arguments["0"]; + var ts = new Date().toISOString(); + //var ts = ts_t.format("%Y-%m-%d %H:%M:%S"); + arguments["0"] = "[??INFO??] ("+ts+"): "+orig; + + if(global.repoproxy.loglevel >= 3) { + sendLog.apply(this, arguments); + } +} + +function debug() { + var orig = arguments["0"]; + var ts = new Date().toISOString(); + //var ts = ts_t.format("%Y-%m-%d %H:%M:%S"); + arguments["0"] = "[^^DEBUG^^] ("+ts+"): "+orig; + + if(global.repoproxy.loglevel >= 4) { + sendLog.apply(this, arguments); + } +} + +function warning() { + var orig = arguments["0"]; + var ts = new Date().toISOString(); + //var ts = ts_t.format("%Y-%m-%d %H:%M:%S"); + arguments["0"] = "[!!WARNING!!] ("+ts+"): "+orig; + + if(global.repoproxy.loglevel >= 2) { + sendLog.apply(this, arguments); + } +} + +function error() { + var orig = arguments["0"]; + var ts = new Date().toISOString(); + //var ts = ts_t.format("%Y-%m-%d %H:%M:%S"); + arguments["0"] = "[**ERROR**] ("+ts+"): "+orig; + + if(global.repoproxy.loglevel >= 1) { + sendLog.apply(this, arguments); + } +} + +function sendLog() { + console.log.apply(this, arguments); +} + + +exports.info = info; +exports.debug = debug; +exports.warning = warning; +exports.error = error; +exports.testLogSettings = testLogSettings; diff --git a/lib/maintain.js b/lib/maintain.js index b740a78..46c3ddc 100644 --- a/lib/maintain.js +++ b/lib/maintain.js @@ -68,8 +68,6 @@ function walkDir(dir, done) { }); }; -exports.walkDir = walkDir; -function cleanupRoutine() { - -} \ No newline at end of file + +exports.walkDir = walkDir; diff --git a/lib/repo-yum.js b/lib/repo-yum.js index 07a1893..25db170 100644 --- a/lib/repo-yum.js +++ b/lib/repo-yum.js @@ -2,61 +2,66 @@ var fs = require("fs"); var http = require("http"); var url = require("url"); var path = require("path"); +var cache = require("./cache.js"); +var log = require("./log.js"); function maintain(name, repoinfo, files) { - console.log("doing yum clean for ", repoinfo); + log.info("Starting maintenance routine for yum repo %s (%s)", name, repoinfo.url); var metaAge = 24*3600*1000*(repoinfo.updateinterval); var expireAge = 24*3600*1000*(repoinfo.expiretime); function fileCheck(i) { - console.log("checking file: ", files[i]); + log.debug("checking file: ", files[i]); // we look for repodata/repomd.xml file, if this is past maturity, we clean that whole directory if(files[i].match(/.*repodata\/repomd\.xml$/)) { - console.log("Found repomd.xml file: ", files[i]); + log.debug("Found repomd.xml file: ", files[i]); // strip the repomd file to get the dir var repomddir = files[i].replace(/repomd\.xml$/, ""); - console.log("repomd dir is:", repomddir); + log.debug("repomd dir is:", repomddir); // do the file stat fs.stat(files[i], function(err, stats) { - console.log("stats for file was: ", stats); + log.debug("stats for file was: ", stats); var curtime = new Date().getTime(); var ctime = stats.ctime.getTime(); - console.log("curtime is ", curtime); - console.log("ctime is ", ctime); + log.debug("curtime is ", curtime); + log.debug("ctime is ", ctime); var age = curtime - ctime; - console.log("age is (%d) for (%d)", age, metaAge); + log.debug("age is (%d) for (%d)", age, metaAge); if(age > metaAge) { // TODO: cleanup repomd directory + cache.moveToCleanup(repomddir); + log.info("Sending repomd directory to trash for cleanup (%s)", repomddir); } }) } else { // STUFF!!! fs.stat(files[i], function(err, stats) { - console.log("stats for file was: ", stats); + log.debug("stats for file was: ", stats); var curtime = new Date().getTime(); var atime = stats.atime.getTime(); - console.log("curtime is ", curtime); - console.log("ctime is ", atime); + log.debug("curtime is ", curtime); + log.debug("ctime is ", atime); var age = curtime - atime; - console.log("age is (%d) for (%d)", age, expireAge); + //console.log("age is (%d) for (%d)", age, expireAge); if(age > expireAge) { // TODO: cleanup singular file // TODO: cleanup meta too, fuck me - console.log("clean up file \n", files[i]); + //console.log("clean up file \n", files[i]); var metafile = files[i].replace(/(.*)\/(.[^\/]+$)/, "$1/.meta.$2.filesize"); //console.log("meta for this file is \n", nfile); - fs.unlink(files[i]); - fs.unlink(metafile); + cache.moveToCleanup(files[i]); + cache.moveToCleanup(metafile); + log.info("Sending expired file to trash for cleanup (%s)", files[i]); } }) } @@ -65,7 +70,7 @@ function maintain(name, repoinfo, files) { } if(typeof files[0] != 'undefined') fileCheck(0) - else console.log("Skipping (yum) file check as there are none... apprently?"); + else log.info("Skipping (yum) file check as there are none... apprently?"); } diff --git a/lib/router.js b/lib/router.js index fcf697c..63622f7 100644 --- a/lib/router.js +++ b/lib/router.js @@ -2,6 +2,7 @@ var url = require("url"); var fs = require("fs"); var cache = require("./cache.js"); var path = require("path"); +var log = require("./log.js"); exports.routeRequest = function(req, res) { // first, unify the request diff --git a/proxy.js b/proxy.js index dabbdec..61aa8f2 100644 --- a/proxy.js +++ b/proxy.js @@ -9,7 +9,7 @@ var maintain = require("./lib/maintain.js"); console.log("Loading configuration"); config.loadConfig("./repos.conf"); - +// start the maintenance timer console.log("Starting cache maintenance timer"); maintain.startTimer(); diff --git a/repos.conf b/repos.conf index f1ab11a..dfa404f 100644 --- a/repos.conf +++ b/repos.conf @@ -18,3 +18,6 @@ listenport:8008 # packageage is how long a package will go unread before it gets deleted (days) repo:fedora:yum:http://ftp.iinet.net.au/pub/fedora/linux/:7:120 repo:ubuntu:apt:http://ftp.iinet.net.au/pub/ubuntu/:1:120 + +# level for logging (1 = error, 2 = warning, 3=info, 4=debug); +loggerlevel:3 diff --git a/unittests/argtest.js b/unittests/argtest.js new file mode 100644 index 0000000..633f3fe --- /dev/null +++ b/unittests/argtest.js @@ -0,0 +1,8 @@ + +function logtest() { + console.log.apply(this, arguments); +} + +var asdf = new Date(); + +logtest("one", asdf); -- 1.7.0.4