From fc40a152c4238a9208086bb1203add8322861675 Mon Sep 17 00:00:00 2001 From: paulr Date: Mon, 1 Aug 2011 03:11:35 +1000 Subject: [PATCH] fixed the directory making bits for the downloader... added a cron for doing repo updates added al ock for repo updates --- bin/downloadfile.php | 33 +++++++++++++++- bin/updateyumrepo.php | 62 ++++++++++++++++++++++++++++ libglcas/web.php | 2 +- plugins/admin.php | 6 ++- plugins/cron.php | 37 +++++++++++++++++ plugins/repo.php | 106 +++++++++++++++++++++++++++++++++++++------------ 6 files changed, 216 insertions(+), 30 deletions(-) create mode 100644 bin/updateyumrepo.php create mode 100644 plugins/cron.php diff --git a/bin/downloadfile.php b/bin/downloadfile.php index 4760627..c0cbec8 100644 --- a/bin/downloadfile.php +++ b/bin/downloadfile.php @@ -15,6 +15,17 @@ if($url == "" || $file == "") { exit(0); } +// first check any directories in $file are in existence +$splfile = explode("/", $file); +if(count($splfile) > 1) { + $tomake = "$repostore/$matched/"; + for($i = 0; $i < count($splfile)-1; $i++) { + $tomake .= "/".$splfile[$i]; + echo("making directory $tomake"); + if(!is_dir($tomake)) mkdir($tomake); + } +} + // ok, we kick off a download if(file_exists("$file")) { @@ -33,17 +44,37 @@ if(file_exists("$file")) { } $remotefile = fopen($url, "r"); + +// check if ther mote file started +if(!$remotefile) { + foreach($http_response_header as $key => $val) { + if(preg_match("/HTTP.*404.*/", $val)) { + echo "got a 404 touching 404 file\n"; + touch("$file.404"); + exit(0); + } + } +} else { + if(file_exists("$file.404")) { + unlink("$file.404"); + } +} + $localfile = fopen($file, "w"); $lockres = flock($localfile, LOCK_EX); if(!$localfile) { - erorr_log("something went plop"); + echo "something went plop\n"; return; } + + // get the headers from the remote request and use them to hurt people $contentlen = 0; $contenttype = ""; foreach($http_response_header as $key => $val) { if(preg_match("/HTTP.*30[1-9].*/", $val)) { + fclose($localfile); + unlink($file); mkdir($file); exit(0); } diff --git a/bin/updateyumrepo.php b/bin/updateyumrepo.php new file mode 100644 index 0000000..a4878fb --- /dev/null +++ b/bin/updateyumrepo.php @@ -0,0 +1,62 @@ + $var) { + //echo "for key $key has:\n"; + //print_r($var); + if($key == "data") { + $fileloc = $var->location["href"]; + if(!file_exists("$file/$fileloc")) { + error_log("getting $file/$fileloc on $url/$fileloc"); + $dlfile = file_get_contents("$url/$fileloc"); + file_put_contents("$file/$fileloc", $dlfile); + } else { + error_log("Not getting $fileloc because we already have it"); + } + } +} + +unlink("$file/repodata/repoupdate.lock"); + +?> \ No newline at end of file diff --git a/libglcas/web.php b/libglcas/web.php index 3327d3d..0bdd798 100644 --- a/libglcas/web.php +++ b/libglcas/web.php @@ -44,7 +44,7 @@ function GLCASMessageBuilder() echo "Messages not implemented yet"; } -function GLCASpageBuilder($bodyClass, $bodyFunction, $bodycontent, $title="GLCAS") +function GLCASpageBuilder($bodyClass, $bodyFunction, $bodycontent=null, $title="GLCAS") { global $WEB_ROOT_FS, $BASE_URL; diff --git a/plugins/admin.php b/plugins/admin.php index 845d857..4a99fb7 100644 --- a/plugins/admin.php +++ b/plugins/admin.php @@ -111,7 +111,8 @@ class GLCASAdmin { echo "Other (OS, Updates, etc)"; echo "Short URL"; echo "URL Prefix (blank for none)"; - echo "Do Initial Update (can take a while)"; + echo "Do Initial Update (can take a while, but done in background)"; + echo "Expire time for Meta Data days"; echo ""; echo ""; echo ""; @@ -132,12 +133,13 @@ class GLCASAdmin { $shorturl = $_REQUEST["shorturl"]; $prefix = $_REQUEST["prefix"]; $repurl = $_REQUEST["repourl"]; + $expiretime = $REQUEST["expiretime"]; $init = false; if(isset($_REQUEST["initial"])) $init = true; - $repo->addRepo($desc, $OS, $version, $arch, $other, $shorturl, $prefix, $repurl, "YUM", $init); + $repo->addRepo($desc, $OS, $version, $arch, $other, $shorturl, $prefix, $repurl, "YUM", $init, $expiretime); global $WEB_ROOT_FS, $URL_HANDLERS, $BASE_URL; header("Location: $BASE_URL/admin/"); diff --git a/plugins/cron.php b/plugins/cron.php new file mode 100644 index 0000000..8168e9a --- /dev/null +++ b/plugins/cron.php @@ -0,0 +1,37 @@ +config = $config; + error_log("constructor for GLCASCron"); + + } + + function go($url) + { + global $CRON_CLASSES; + + echo ""; + foreach($CRON_CLASSES as $crons) { + $calling = new $crons($this->config); + if(method_exists($crons, "cron")) { + echo "CRON: Calling $crons"."->cron:
"; + echo "
$crons"; + $calling->cron(); + echo "
"; + } else { + echo "CRON: would call $crons"."->cron but it doesnt exist
"; + } + + } + echo ""; + } + + private $config; +} + + +?> \ No newline at end of file diff --git a/plugins/repo.php b/plugins/repo.php index 48ffae0..54f3557 100644 --- a/plugins/repo.php +++ b/plugins/repo.php @@ -1,6 +1,7 @@ "; + $uconf = unserialize($this->config->getConfigVar("repodata")); + $repostore = $this->config->getConfigVar("storagelocation"); + //print_r($uconf); + //echo ""; + + foreach($uconf as $rkey => $repo) { + echo "Repo $rkey: ".$repo["desc"]."
"; + if(!isset($repo["expiretime"])) { + echo " - Expire time not set, setting to 2 days by default
"; + $uconf[$rkey]["expiretime"] = 2; + $repo["expiretime"] = 2; + $this->config->setConfigVar("repodata", serialize($uconf)); + $this->config->saveConfig(); + } + + if(file_exists("$repostore/$rkey/repodata/repoupdate.lock")) { + echo " - Repo locked for update
"; + + } + + // we still do this next bit, even if its locked because it may be a stray file + echo " - Expire time is ".$repo["expiretime"]." days, checking repo
"; + error_log("checking $repostore/$rkey/repodata/repomd.xml"); + $tdiff = time() - filemtime("$repostore/$rkey/repodata/repomd.xml"); + $maxtdiff = $repo["expiretime"] * 24 * 3600; + if($tdiff > $maxtdiff) { + echo " - updating repo
"; + $this->updateRepo($rkey); + } else { + echo " - not updating repo
"; + } + } + } function go($url) { @@ -278,8 +316,34 @@ class GLCASRepo { // or a range (watching and sending) // TODO: it may be advicable to start the download as a seperate cli process rather then something goin on here // so it definitely cant be interrupted. - - + + // check for a 404 file and wait 2 if it exists - i should really check the timestamp for an updated + // file, but thats too much effort for now: TODO: check timestamp on 404 file + $slept = 0; + while(!file_exists("$filename")) { + clearstatcache(); + sleep(1); + $slept += 1; + error_log("Sleeping waiting for file"); + + // if 404 file exists, we wait much less time + if(file_exists("$filename.404") && $slept > 2) { + header("HTTP/1.0 404 Not Found"); + return; + } + if($slept > 10) { + header("HTTP/1.0 404 Not Found"); + return; + } + } + + clearstatcache(); + if(is_dir($filename)) { + + header("Location: ".$_SERVER["REQUEST_URI"]."/"); + return; + } + // first, getting and sending - this is easy. if (!$ranged) { @@ -463,7 +527,7 @@ class GLCASRepo { error_log("background downloader, start"); global $WEB_ROOT_FS, $URL_HANDLERS, $BASE_URL; if(file_exists("$WEB_ROOT_FS/../bin/downloadfile.php")) { - $scall = "/usr/bin/php $WEB_ROOT_FS/../bin/downloadfile.php '$url' '$file' > /dev/null 2>&1 &"; + $scall = "/usr/bin/php $WEB_ROOT_FS/../bin/downloadfile.php '$url' '$file' > /tmp/dllog 2>&1 &"; system($scall); } else { error_log("cant find download helper... dieing"); @@ -771,7 +835,7 @@ class GLCASRepo { } } - function addRepo($desc, $os, $version, $arch, $other, $shorturl, $prefix, $repurl, $repotype, $init) + function addRepo($desc, $os, $version, $arch, $other, $shorturl, $prefix, $repurl, $repotype, $init, $expiretime) { $uconf = $this->config->getConfigVar("repodata"); @@ -784,6 +848,7 @@ class GLCASRepo { $cs["prefix"] = $prefix; $cs["url"] = $repurl; $cs["repotype"] = $repotype; + $cs["expiretime"] = $expiretime; $ckey = 0; @@ -832,29 +897,18 @@ class GLCASRepo { if(!file_exists("$repostore/$repokey/repodata")) { mkdir("$repostore/$repokey/repodata"); } + + error_log("background yum repo update, start"); + global $WEB_ROOT_FS, $URL_HANDLERS, $BASE_URL; + if(file_exists("$WEB_ROOT_FS/../bin/downloadfile.php")) { + $scall = "/usr/bin/php $WEB_ROOT_FS/../bin/updateyumrepo.php '$repourl' '$repostore/$repokey/' > /tmp/updateyumrepo.$repokey.log 2>&1 &"; + system($scall); + } else { + error_log("cant find download yum helper... dieing"); + } + //ignore_user_abort(true); - $actionurl = "$repourl/repodata/repomd.xml"; - $repomdxml = file_get_contents($actionurl); - file_put_contents("$repostore/$repokey/repodata/repomd.xml", $repomdxml); - - $xml = simplexml_load_file("$repostore/$repokey/repodata/repomd.xml"); - - - foreach($xml as $key => $var) { - //echo "for key $key has:\n"; - //print_r($var); - if($key == "data") { - $fileloc = $var->location["href"]; - if(!file_exists("$repostore/$repokey/$fileloc")) { - error_log("getting $fileloc for $repokey on $repourl"); - $dlfile = file_get_contents("$repourl/$fileloc"); - file_put_contents("$repostore/$repokey/$fileloc", $dlfile); - } else { - error_log("Not getting $fileloc because we already have it"); - } - } - } } function getRepo($id) -- 1.7.0.4