From 85fcaedb62209c356da64b865f981d212ae593d4 Mon Sep 17 00:00:00 2001 From: paulr Date: Tue, 22 Jan 2013 00:11:50 +1100 Subject: [PATCH] repository maintainence code --- lib/maintain.js | 37 +++++++++++++++++++++++++++++++-- lib/repo-apt.js | 54 ++++++++++++++++++++++++++++++++++++++++++++++++- repos.conf | 2 +- unittests/recurse.js | 33 ++++++++++++++++++++++++++++++ 4 files changed, 120 insertions(+), 6 deletions(-) create mode 100644 unittests/recurse.js diff --git a/lib/maintain.js b/lib/maintain.js index e336d4a..cbed4a2 100644 --- a/lib/maintain.js +++ b/lib/maintain.js @@ -14,11 +14,14 @@ function maintainCache() { switch(global.repoproxy.repo[index]["type"]) { case "apt": console.log("Scanning '%s' as apt", index); - repoapt.maintain(global.repoproxy.repo[index]); + var walkin = path.normalize(global.repoproxy.cacheDir + "/" + index); + walkDir(walkin, function(err, list) { + repoapt.maintain(index, global.repoproxy.repo[index], list); + }) break; case "yum": - console.log("Scanning '%s' as apt", index); - repoyum.maintain(global.repoproxy.repo[index]); + //console.log("Scanning '%s' as apt", index); + //repoyum.maintain(global.repoproxy.repo[index]); break; } } @@ -33,6 +36,34 @@ exports.startTimer = function() { setInterval(maintainCache, cacheTimer); } +// this code comes frmo http://stackoverflow.com/questions/5827612/node-js-fs-readdir-recursive-directory-search +function walkDir(dir, done) { + var results = []; + + fs.readdir(dir, function(err, list) { + if (err) return done(err); + var i = 0; + (function next() { + var file = list[i++]; + if (!file) return done(null, results); + file = path.normalize(dir + "/" + file); + fs.stat(file, function(err, stat) { + if (stat && stat.isDirectory()) { + walkDir(file, function(err, res) { + results = results.concat(res); + next(); + }); + } else { + if(!file.match(/.*\.meta.*\.filesize/)) results.push(file); + next(); + } + }); + })(); + }); +}; + +exports.walkDir = walkDir; + function cleanupRoutine() { } \ No newline at end of file diff --git a/lib/repo-apt.js b/lib/repo-apt.js index f37f881..07248a0 100644 --- a/lib/repo-apt.js +++ b/lib/repo-apt.js @@ -1,5 +1,55 @@ -function maintain(details) { - console.log("doing apt clean for ", details); +var fs = require("fs"); +var http = require("http"); +var url = require("url"); +var path = require("path"); + +function maintain(name, repoinfo, files) { + //console.log("doing apt clean for ", repoinfo); + + var topdir = path.normalize(global.repoproxy.cacheDir + "/" + name + "/"); + + function fileCheck(i) { + //console.log("checking file: ", files[i]); + + var cfile = files[i].replace(topdir, ""); + var ctime_t = new Date(); + var ctime = ctime_t.getTime(); + var time_ui = ((ctime) - ((repoinfo["updateinterval"] * 3600 * 24 * 1000) - (12*3600*1000))); + var time_et = ((ctime) - ((repoinfo["expiretime"] * 3600 * 24 * 1000) - (12*3600*1000))); + + // file checks go here + if(typeof global.repoproxy.downloads[files[i]] != "undefined" && global.repoproxy.downloads[files[i]] == 1) { + // ignore this file as its being downloaded + console.log("not checking file because its downloading ", cfile); + if(typeof files[i+1] != "undefined") fileCheck(i+1); + } else { + fs.stat(files[i], function(err, stats) { + + //console.log("deep check", cfile); + if(cfile.match(/.*dists\/.*/) != null) { + // its a dist file, probably, check age and erase if necessary + if(stats["mtime"].getTime() < time_ui) { + // erase file + console.log("unlinking file for time_ui: ", files[i]) + //fs.unlink(files[i]); + } else { + //console.log("times for file '%s', '%s', '%s', '%s'", cfile, time_ui, time_et, stats["mtime"].getTime()); + } + + } else { + // its just some other file, check the read time + if(stats["atime"].getTime() < time_et) { + console.log("unlinking file for time_et: ", files[i]); + } + } + if(typeof files[i+1] != "undefined") fileCheck(i+1); + }); + } + + + } + + fileCheck(0) } exports.maintain = maintain; \ No newline at end of file diff --git a/repos.conf b/repos.conf index 68fa3f7..560d566 100644 --- a/repos.conf +++ b/repos.conf @@ -17,4 +17,4 @@ ploop:asdf # updateinterval is how often repo meta data is refreshed (days) # packageage is how long a package will go unread before it gets deleted (days) repo:fedora:yum:http://ftp.iinet.net.au/pub/fedora/linux/:7:120 -repo:ubuntu:apt:http://ftp.iinet.net.au/pub/ubuntu/:7:120 \ No newline at end of file +repo:ubuntu:apt:http://ftp.iinet.net.au/pub/ubuntu/:1:120 \ No newline at end of file diff --git a/unittests/recurse.js b/unittests/recurse.js new file mode 100644 index 0000000..5819c0a --- /dev/null +++ b/unittests/recurse.js @@ -0,0 +1,33 @@ +var fs = require("fs"); +var path = require("path"); + + +// comes from http://stackoverflow.com/questions/5827612/node-js-fs-readdir-recursive-directory-search +var walk = function(dir, done) { + var results = []; + + fs.readdir(dir, function(err, list) { + if (err) return done(err); + var i = 0; + (function next() { + var file = list[i++]; + if (!file) return done(null, results); + file = path.normalize(dir + "/" + file); + fs.stat(file, function(err, stat) { + if (stat && stat.isDirectory()) { + walk(file, function(err, res) { + results = results.concat(res); + next(); + }); + } else { + if(!file.match(/.*\.meta.*\.filesize/)) results.push(file); + next(); + } + }); + })(); + }); +}; + +walk("/tmp/cache/ubuntu", function(err, res) { + console.log("result: ", res); +}); \ No newline at end of file -- 1.7.0.4