From: paulr <paulr@tv.pjr.cc>
Date: Mon, 21 Jan 2013 13:11:50 +0000 (+1100)
Subject: repository maintainence code
X-Git-Url: http://git.pjr.cc/?p=nodejs-repoproxy.git;a=commitdiff_plain;h=85fcaedb62209c356da64b865f981d212ae593d4

repository maintainence code
---

diff --git a/lib/maintain.js b/lib/maintain.js
index e336d4a..cbed4a2 100644
--- a/lib/maintain.js
+++ b/lib/maintain.js
@@ -14,11 +14,14 @@ function maintainCache() {
 		switch(global.repoproxy.repo[index]["type"]) {
 		case "apt":
 			console.log("Scanning '%s' as apt", index);
-			repoapt.maintain(global.repoproxy.repo[index]);
+			var walkin = path.normalize(global.repoproxy.cacheDir + "/" + index);
+			walkDir(walkin, function(err, list) {
+				repoapt.maintain(index, global.repoproxy.repo[index], list);
+			})
 			break;
 		case "yum":
-			console.log("Scanning '%s' as apt", index);
-			repoyum.maintain(global.repoproxy.repo[index]);
+			//console.log("Scanning '%s' as apt", index);
+			//repoyum.maintain(global.repoproxy.repo[index]);
 			break;
 		}
 	}
@@ -33,6 +36,34 @@ exports.startTimer = function() {
 	setInterval(maintainCache, cacheTimer);
 }
 
+// this code comes frmo http://stackoverflow.com/questions/5827612/node-js-fs-readdir-recursive-directory-search
+function walkDir(dir, done) {
+	var results = [];
+	
+	fs.readdir(dir, function(err, list) {
+		if (err) return done(err);
+		var i = 0;
+		(function next() {
+			var file = list[i++];
+			if (!file) return done(null, results);
+			file = path.normalize(dir + "/" + file);
+			fs.stat(file, function(err, stat) {
+				if (stat && stat.isDirectory()) {
+					walkDir(file, function(err, res) {
+						results = results.concat(res);
+						next();
+					});
+				} else {
+					if(!file.match(/.*\.meta.*\.filesize/)) results.push(file);
+					next();
+				}
+			});
+		})();
+	});
+};
+
+exports.walkDir = walkDir;
+
 function cleanupRoutine() {
 	
 }
\ No newline at end of file
diff --git a/lib/repo-apt.js b/lib/repo-apt.js
index f37f881..07248a0 100644
--- a/lib/repo-apt.js
+++ b/lib/repo-apt.js
@@ -1,5 +1,55 @@
-function maintain(details) {
-	console.log("doing apt clean for ", details);
+var fs = require("fs");
+var http = require("http");
+var url = require("url");
+var path = require("path");
+
+function maintain(name, repoinfo, files) {
+	//console.log("doing apt clean for ", repoinfo);
+	
+	var topdir = path.normalize(global.repoproxy.cacheDir + "/" + name + "/");
+	
+	function fileCheck(i) {
+		//console.log("checking file: ", files[i]);
+		
+		var cfile = files[i].replace(topdir, "");
+		var ctime_t = new Date();
+		var ctime = ctime_t.getTime();
+		var time_ui = ((ctime) - ((repoinfo["updateinterval"] * 3600 * 24 * 1000) - (12*3600*1000)));
+		var time_et = ((ctime) - ((repoinfo["expiretime"] * 3600 * 24 * 1000) - (12*3600*1000)));
+		
+		// file checks go here
+		if(typeof global.repoproxy.downloads[files[i]] != "undefined" && global.repoproxy.downloads[files[i]] == 1) {
+			// ignore this file as its being downloaded
+			console.log("not checking file because its downloading ", cfile);
+			if(typeof files[i+1] != "undefined") fileCheck(i+1);
+		} else {
+			fs.stat(files[i], function(err, stats) {
+				
+				//console.log("deep check", cfile);
+				if(cfile.match(/.*dists\/.*/) != null) {
+					// its a dist file, probably, check age and erase if necessary
+					if(stats["mtime"].getTime() < time_ui) {
+						// erase file
+						console.log("unlinking file for time_ui: ", files[i])
+						//fs.unlink(files[i]);
+					} else {
+						//console.log("times for file '%s', '%s', '%s', '%s'", cfile, time_ui, time_et, stats["mtime"].getTime());
+					}
+					
+				} else {
+					// its just some other file, check the read time
+					if(stats["atime"].getTime() < time_et) {
+						console.log("unlinking file for time_et: ", files[i]);
+					}
+				}
+				if(typeof files[i+1] != "undefined") fileCheck(i+1);				
+			});
+		}
+		
+		
+	}
+	
+	fileCheck(0)
 }
 
 exports.maintain = maintain;
\ No newline at end of file
diff --git a/repos.conf b/repos.conf
index 68fa3f7..560d566 100644
--- a/repos.conf
+++ b/repos.conf
@@ -17,4 +17,4 @@ ploop:asdf
 #		updateinterval is how often repo meta data is refreshed (days)
 #		packageage is how long a package will go unread before it gets deleted (days)
 repo:fedora:yum:http://ftp.iinet.net.au/pub/fedora/linux/:7:120
-repo:ubuntu:apt:http://ftp.iinet.net.au/pub/ubuntu/:7:120
\ No newline at end of file
+repo:ubuntu:apt:http://ftp.iinet.net.au/pub/ubuntu/:1:120
\ No newline at end of file
diff --git a/unittests/recurse.js b/unittests/recurse.js
new file mode 100644
index 0000000..5819c0a
--- /dev/null
+++ b/unittests/recurse.js
@@ -0,0 +1,33 @@
+var fs = require("fs");
+var path = require("path");
+		
+
+// comes from http://stackoverflow.com/questions/5827612/node-js-fs-readdir-recursive-directory-search
+var walk = function(dir, done) {
+	var results = [];
+	
+	fs.readdir(dir, function(err, list) {
+		if (err) return done(err);
+		var i = 0;
+		(function next() {
+			var file = list[i++];
+			if (!file) return done(null, results);
+			file = path.normalize(dir + "/" + file);
+			fs.stat(file, function(err, stat) {
+				if (stat && stat.isDirectory()) {
+					walk(file, function(err, res) {
+						results = results.concat(res);
+						next();
+					});
+				} else {
+					if(!file.match(/.*\.meta.*\.filesize/)) results.push(file);
+					next();
+				}
+			});
+		})();
+	});
+};
+
+walk("/tmp/cache/ubuntu", function(err, res) {
+	console.log("result: ", res);
+});
\ No newline at end of file