From ce6eddc1cf6b4a879b9239fec18258b99f3d739a Mon Sep 17 00:00:00 2001 From: paulr Date: Sun, 7 Aug 2011 02:14:51 +1000 Subject: [PATCH] added a apt repo updater... that doesnt work --- bin/updateaptrepo.php | 36 +++++++ plugins/admin.php | 65 ++++++++++--- plugins/knownos.php | 30 +----- plugins/repo.php | 261 +++++++++++-------------------------------------- www/css/normal.css | 9 ++ 5 files changed, 157 insertions(+), 244 deletions(-) create mode 100644 bin/updateaptrepo.php diff --git a/bin/updateaptrepo.php b/bin/updateaptrepo.php new file mode 100644 index 0000000..38aae68 --- /dev/null +++ b/bin/updateaptrepo.php @@ -0,0 +1,36 @@ + \ No newline at end of file diff --git a/plugins/admin.php b/plugins/admin.php index 13121c3..492ed29 100644 --- a/plugins/admin.php +++ b/plugins/admin.php @@ -162,19 +162,22 @@ class GLCASAdmin { $version = $glt["versions"]; $archs = $glt["arch"]; - echo "
";
-			print_r($glt);
-			echo "
"; + //echo "
";
+			//print_r($glt);
+			//echo "
"; + echo "The joy of apt is that there could be a million distro's under the root of an apt mirror"; + echo "if it doesn't list here, but you know its there, its still usable for glcas"; echo "
"; echo ""; echo ""; + echo ""; echo ""; echo ""; echo ""; - // TODO change this "provides" bit with a bit that allows us to choose which versions - // of ubuntu will be made available from the ones we found - echo ""; @@ -236,9 +239,26 @@ class GLCASAdmin { $init = false; if(isset($_REQUEST["initial"])) $init = true; + $blocklist = null; + $bn = 0; + foreach($_REQUEST as $key => $val) { + if(preg_match("/^blocked-*/", $key)) { + $blocked = preg_replace("/^blocked-/", "", $key); + //echo "
got block for $blocked
"; + $blocklist[$bn] = "dists\/$blocked.*"; + $bn++; + } + } + //echo "
";
+		//print_r($_REQUEST);
+		//echo "
"; - $repo->addRepo($desc, $OS, $version, $arch, "-", $shorturl, $prefix, $repurl, "APT", $init, $expiretime); + //exit(0); + + $repo->addRepo($desc, $OS, $version, $arch, "-", $shorturl, $prefix, $repurl, "APT", $init, $expiretime, $blocklist); + + // next we need to deal with that "blocked" bit global $WEB_ROOT_FS, $URL_HANDLERS, $BASE_URL; header("Location: $BASE_URL/admin/"); @@ -271,6 +291,8 @@ class GLCASAdmin { function doUpdateRepo($url) { $rkey = $_REQUEST["repo"]; + + error_log("would update for repo"); $repo = new GLCASRepo($this->config); @@ -338,13 +360,12 @@ class GLCASAdmin { //echo ""; // wrap all this in a table - echo "
Description
Provides (tick those you wish to BLOCK)"; + + // TODO: deal with block bit + echo "
Provides (tick those you wish to BLOCK - not implemented)"; foreach($glt["knownrepo"] as $key => $val) { $reponame = $kos["apt"][$val["name"]]; $a = ""; @@ -183,7 +186,7 @@ class GLCASAdmin { if($glt["knownrepo"][$key]["amd64"]) $b = "x86_64"; if($a != "" && $b != "") $archs = "$a/$b"; else $archs = "$a$b"; - echo " $reponame ($archs)
"; + echo " $reponame ($archs)
"; } echo "
"; + echo "
"; // now, add a repo echo "

Add A Repo

"; echo ""; echo "Type
"; echo "URL
"; @@ -359,19 +380,37 @@ class GLCASAdmin { echo "Hint
"; echo "URL
"; echo "
"; echo ""; - echo "
"; + echo "		"; + + // now for mirror lists + echo "

Via Mirrolist - not implemented

"; + echo "
"; + echo "Distro
"; + echo "Architecture
"; + echo "Version
"; + echo "Get updates too
"; + echo "
"; + echo "
"; + + + echo "
"; // repo storage location echo "

Storage


"; echo "
"; $storloc = $this->config->getConfigVar("storagelocation"); - echo ""; + echo ""; echo ""; echo "
"; } diff --git a/plugins/knownos.php b/plugins/knownos.php index b6dd89c..1a8b447 100644 --- a/plugins/knownos.php +++ b/plugins/knownos.php @@ -51,42 +51,18 @@ function getKnownOSList() // ubuntu bit for apt // TODO uncomment these ones out of testing - /*$kos["apt"]["hardy"] = "Hardy Heron (8.04 LTS)"; - $kos["apt"]["hardy-backports"] = "Hardy Heron (8.04 LTS) Backports"; - $kos["apt"]["hardy-proposed"] = "Hardy Heron (8.04 LTS) Proposed"; - $kos["apt"]["hardy-security"] = "Hardy Heron (8.04 LTS) Security"; - $kos["apt"]["hardy-updates"] = "Hardy Heron (8.04 LTS) Updates"; + $kos["apt"]["hardy"] = "Hardy Heron (8.04 LTS)"; $kos["apt"]["karmic"] = "Karmic Koala (9.10)"; - $kos["apt"]["karmic-backports"] = "Karmic Koala (9.10) Backports"; - $kos["apt"]["karmic-proposed"] = "Karmic Koala (9.10) Proposed"; - $kos["apt"]["karmic-security"] = "Karmic Koala (9.10) Security"; - $kos["apt"]["karmic-updates"] = "Karmic Koala (9.10) Updates"; $kos["apt"]["lucid"] = "Lucid Lynx (10.04 LTS)"; - $kos["apt"]["lucid-backports"] = "Lucid Lynx (10.04 LTS) Backports"; - $kos["apt"]["lucid-proposed"] = "Lucid Lynx (10.04 LTS) Proposed"; - $kos["apt"]["lucid-security"] = "Lucid Lynx (10.04 LTS) Security"; - $kos["apt"]["lucid-updates"] = "Lucid Lynx (10.04 LTS) Updates"; $kos["apt"]["maverick"] = "Maverick Meerkat (10.10)"; - $kos["apt"]["maverick-backports"] = "Maverick Meerkat (10.10) Backports"; - $kos["apt"]["maverick-proposed"] = "Maverick Meerkat (10.10) Proposed"; - $kos["apt"]["maverick-security"] = "Maverick Meerkat (10.10) Security"; - $kos["apt"]["maverick-updates"] = "Maverick Meerkat (10.10) Updates";*/ $kos["apt"]["natty"] = "Natty Narwhal (11.04)"; - $kos["apt"]["natty-backports"] = "Natty Narwhal (11.04) Backports"; - $kos["apt"]["natty-proposed"] = "Natty Narwhal (11.04) Proposed"; - $kos["apt"]["natty-security"] = "Natty Narwhal (11.04) Security"; - $kos["apt"]["natty-updates"] = "Natty Narwhal (11.04) Updates"; - /*$kos["apt"]["oneiric"] = "Oneiric Ocelot (11.10)"; - $kos["apt"]["oneiric-backports"] = "Oneiric Ocelot (11.10) Backports"; - $kos["apt"]["oneiric-proposed"] = "Oneiric Ocelot (11.10) Proposed"; - $kos["apt"]["oneiric-security"] = "Oneiric Ocelot (11.10) Security"; - $kos["apt"]["oneiric-updates"] = "Oneiric Ocelot (11.10) Updates"; + $kos["apt"]["oneiric"] = "Oneiric Ocelot (11.10)"; $kos["apt"]["lenny"] = "Debian Lenny"; $kos["apt"]["sid"] = "Debian Sid"; $kos["apt"]["squeeze"] = "Debian Squeeze"; $kos["apt"]["wheezy"] = "Debian Wheezy"; $kos["apt"]["stable"] = "Debian Stable"; - $kos["apt"]["unstable"] = "Debian Unstable";*/ + $kos["apt"]["unstable"] = "Debian Unstable"; diff --git a/plugins/repo.php b/plugins/repo.php index ee1f141..9d1de84 100644 --- a/plugins/repo.php +++ b/plugins/repo.php @@ -36,7 +36,10 @@ class GLCASRepo { } if(!isset($repo["repotype"])) { - echo "Repo type not set for repo
"; + echo "Repo type not set for repo, setting to yum
"; + $uconf[$rkey]["repotype"] = "YUM"; + $this->config->setConfigVar("repodata", serialize($uconf)); + $this->config->saveConfig(); } else if($repo["repotype"] == "YUM") { if(file_exists("$repostore/$rkey/repodata/repoupdate.lock")) { echo " - Repo locked for update
"; @@ -55,7 +58,12 @@ class GLCASRepo { echo " - not updating repo
"; } } else if($repo["repotype"] == "APT") { - echo "I have no idea how to spawn an update of an APT repo - apt is crap
"; + if(file_exists("$repostore/$rkey/update.lock")) { + echo " - Repo locaked for update
"; + } + echo " - Expire time is ".$repo["expiretime"]." days, checking repo
"; + $maxtdiff = $repo["expiretime"] * 24 * 3600; + $this->updateRepo($rkey); } } } @@ -169,6 +177,7 @@ class GLCASRepo { header("HTTP/1.0 404 Not Found"); return; } + // something was matched, so now we reconstruct the file component of the url @@ -180,7 +189,20 @@ class GLCASRepo { // so, the ultimate url for the file we need is: $actualfile = "$repostore/$matched/$file"; error_log("Atcualfile is $actualfile"); - + + // now check for a block in that repo + if(isset($uconf[$matched]["blocklist"])) { + foreach($uconf[$matched]["blocklist"] as $blockers) { + $rfile = ltrim($file, "/"); + error_log("checking $blockers against $rfile"); + if(preg_match("/$blockers/", $rfile) > 0) { + error_log("should block"); + header("HTTP/1.0 404 Not Found"); + return; + } + } + } + // if its a directory, lets do a print if(is_dir($actualfile)) { $this->printDir($actualfile, $file, $url); @@ -541,206 +563,7 @@ class GLCASRepo { } // this is a nightmare - function getRepoForUrlOld($url) - { - // the way we breakdown a url is to explode it - $xurl = split("[/,]", $url); - - // we first check if [0] is a prefix - // if now, we check for it being a shorturl (lets just do that for now) - $uconf = unserialize($this->config->getConfigVar("repodata")); - $repostore = $this->config->getConfigVar("storagelocation"); - - $matched = -1; - - // first we check for /repo/repoid as a url - $startat = 0; - if($xurl[0] == "repo") { - $repid = $xurl[1]; - error_log("trying to get repo for repoid, $repid"); - if(isset($uconf[$repid])) { - $matched = ((int)($repid)); - error_log("set matched, $matched, $repid"); - $startat +=2; - } - } - - - $prematch = false; - if($matched < 0) foreach($uconf as $key => $var) { - $pre = $var["prefix"]; - - if($pre!="") { - //echo "Checking pre $pre against ".$xurl[0]."\n"; - if(strcasecmp($pre, $xurl[0])==0) { - //echo "Matched pre\n"; - $prematch = true; - $startat++; - } - } - } - - - if($matched < 0) foreach($uconf as $key => $var) { - // if we matched a pre, then we check against the second url component - - $short = $var["shorturl"]; - - if($short!="") { - //echo "Checking short $short against ".$xurl[$startat]."\n"; - if(strcasecmp($xurl[$startat], $short)==0) { - //echo "Matched\n"; - $matched = $key; - $startat++; - } - } - } - - if($matched < 0) { - echo "No such repo
"; - return; - } - - // now we find an actual file - $file = "/"; - if(count($xurl) > $startat) for($i=$startat; $i < count($xurl); $i++) { - $file .= "/".$xurl[$i]; - } - - // now we want to find repostore/$matched/$file; - $actualfile = "$repostore/$matched/$file"; - error_log("Atcualfile is $actualfile"); - //echo "Start file for $actualfile\n"; - - // first check any directories in $file are in existence - $splfile = explode("/", $file); - if(count($splfile) > 1) { - $tomake = "$repostore/$matched/"; - for($i = 0; $i < count($splfile)-1; $i++) { - $tomake .= "/".$splfile[$i]; - //error_log("making directory $tomake"); - if(!is_dir($tomake)) mkdir($tomake); - } - } - - $reqhead = print_r($_REQUEST, true); - $sevhead = print_r($_SERVER, true); - - error_log("req $reqhead"); - error_log("sev $sevhead"); - - $rangestart = -1; - $rangelength = -1; - $rangesstr = ""; - if(isset($_SERVER["HTTP_RANGE"])) { - // oh shit - $rangesa = explode("=", $_SERVER["HTTP_RANGE"]); - $rangesb = explode(",", $rangesa[1]); - $rangesstr = $rangesb[0]; - $ranges = explode("-", $rangesb[0]); - $rangestart = $ranges[0]; - $rangelength = $ranges[1] - $ranges[0] +1; - error_log("going ranges at $rangestart, $rangelength,".$rangesa[1].",".$rangesb[0]); - } - - // i have to support http_range cause REDHAT/CENTOS IS annoying as all hell. christ, why do this? - if(is_file($actualfile)) { - // file is stored locally, away we go - if($rangelength != -1) { - header("HTTP/1.1 206 Partial Content"); - header("Content-Length: ".$rangelength); - header("Content-Range: bytes $rangesstr/".filesize($actualfile)); - //header("Content-Length: ".filesize($actualfile)); - } else { - header("Content-Length: ".filesize($actualfile)); - } - $type = mime_content_type($actualfile); - header("Content-type: $type"); - $localfile = fopen($actualfile, "r"); - if($rangestart!=-1) fseek($localfile, $rangestart, SEEK_SET); - while(!feof($localfile)) { - // cant make this high cause centos is crap - if($rangelength!=-1) { - $data = fread($localfile, $rangelength); - error_log("data size was ".strlen($data)); - } else { - $data = fread($localfile, 2048); - } - - echo $data; - flush(); - - if($rangelength!=-1) { - fclose($localfile); - exit(0); - } - } - fclose($localfile); - } else if(is_dir($actualfile)) { - //echo "in dir for $actualfile\n"; - // here we print the contents of the directory - $this->printDir($actualfile, $file, $url); - } else { - // ok, get the file - //echo "in getcheck\n"; - $remotefile = $uconf[$matched]["url"]."/$file"; - - // TODO: i should get remote contents with fopen/fread/fwrite as - // it should be more memory conservative and we can push to the end client - // straight away - ignore_user_abort(true); - $rf = fopen($remotefile, "r"); - error_log("attempting to get remote file $remotefile"); - - - // hopefully this works. if we get a 30x message, it means we tried to get a directory - // i cant think of another way of dealing with it - but this is UGLY - // also get content length and content type - $clen = 0; - foreach($http_response_header as $key => $val) { - if(preg_match("/HTTP.*30[1-9].*/", $val)) { - error_log("got a 30x, must be a directory"); - mkdir($actualfile); - header("Location: ".$_SERVER["REQUEST_URI"]."/"); - return; - } - // get content length form upstream and print - if(preg_match("/^Content-Length:.*/", $val)) { - $clen = $val; - header($val); - } - // get content type from upstream and print - if(preg_match("/^Content-Type:.*/", $val)) { - header($val); - } - } - //error_log("repsonse: $http_response_header"); - if(!$rf) { - // return 404 - header("HTTP/1.0 404 Not Found"); - } else { - $localfile = fopen($actualfile.".tmp.data.deleteme", "w"); - $localsizefile = fopen($actualfile.".tmp.data.deleteme.size", "w"); - fwrite($localsizefile, "$clen"); - fclose($localsizefile); - while(!feof($rf)) { - $data = fread($rf, 8192); - echo $data; - fwrite($localfile, $data); - flush(); - } - fclose($localfile); - fclose($rf); - rename($actualfile.".tmp.data.deleteme", $actualfile); - //error_log("got actualfile, tried to save as $actualfile, did it work?"); - } - } - - //echo "got ".$file." for $url which is $actualfile\n"; - - //echo ""; - } function printDir($dir, $localfile, $baseurl) { @@ -906,7 +729,7 @@ class GLCASRepo { } } - function addRepo($desc, $os, $version, $arch, $other, $shorturl, $prefix, $repurl, $repotype, $init, $expiretime) + function addRepo($desc, $os, $version, $arch, $other, $shorturl, $prefix, $repurl, $repotype, $init, $expiretime, $blocklist=null) { $uconf = $this->config->getConfigVar("repodata"); @@ -920,6 +743,9 @@ class GLCASRepo { $cs["url"] = $repurl; $cs["repotype"] = $repotype; $cs["expiretime"] = $expiretime; + if($blocklist != null) { + $cs["blocklist"] = $blocklist; + } $ckey = 0; @@ -950,7 +776,34 @@ class GLCASRepo { function updateRepo($repokey) { // we only do yum yet - $this->updateRepoYum($repokey); + $repod = $this->getRepo($repokey); + + error_log("in update repo"); + + if($repod["repotype"] == "YUM") $this->updateRepoYum($repokey); + if($repod["repotype"] == "APT") $this->updateRepoApt($repokey); + } + + function updateRepoApt($repokey) + { + $repostore = $this->config->getConfigVar("storagelocation"); + + $repod = $this->getRepo($repokey); + + $repourl = $repod["url"]; + + if(!file_exists("$repostore/$repokey")) { + mkdir("$repostore/$repokey"); + } + + error_log("background apt repo update, start"); + global $WEB_ROOT_FS, $URL_HANDLERS, $BASE_URL; + if(file_exists("$WEB_ROOT_FS/../bin/downloadfile.php")) { + $scall = "/usr/bin/php $WEB_ROOT_FS/../bin/updateaptrepo.php '$repourl' '$repostore/$repokey/' > /tmp/updateaptrepo.$repokey.log 2>&1 &"; + system($scall); + } else { + error_log("cant find download apt helper... dieing"); + } } function updateRepoYum($repokey) diff --git a/www/css/normal.css b/www/css/normal.css index 460827d..bf63908 100644 --- a/www/css/normal.css +++ b/www/css/normal.css @@ -126,4 +126,13 @@ div.wrapper { } .tabUnselected { background-color:#dadaf8; +} + +input[type=text] { + width: 200px; + +} + +input.storage { + width: 400px; } \ No newline at end of file -- 1.7.0.4